From 26e3062b308ef743647a0e63be473d7dc3079b51 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 27 Nov 2024 14:18:52 +0100 Subject: [PATCH 01/68] [red-knot] Statically known branches --- .../mdtest/annotations/literal_string.md | 5 + .../resources/mdtest/annotations/never.md | 5 + .../resources/mdtest/boolean/short_circuit.md | 2 +- .../resources/mdtest/literal/ellipsis.md | 20 +- .../resources/mdtest/narrow/issubclass.md | 5 + .../resources/mdtest/pep695_type_aliases.md | 5 + .../mdtest/statically-known-branches.md | 294 ++++++++++++ .../src/semantic_index.rs | 28 ++ .../src/semantic_index/builder.rs | 65 ++- .../src/semantic_index/use_def.rs | 152 +++++- .../src/semantic_index/use_def/bitset.rs | 3 +- .../semantic_index/use_def/symbol_state.rs | 435 ++++++++++++------ crates/red_knot_python_semantic/src/types.rs | 180 ++++++-- 13 files changed, 997 insertions(+), 202 deletions(-) create mode 100644 crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md diff --git a/crates/red_knot_python_semantic/resources/mdtest/annotations/literal_string.md b/crates/red_knot_python_semantic/resources/mdtest/annotations/literal_string.md index 42d616b8410c3..fe0f40dc9ec23 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/annotations/literal_string.md +++ b/crates/red_knot_python_semantic/resources/mdtest/annotations/literal_string.md @@ -1,5 +1,10 @@ # `LiteralString` +```toml +[environment] +target-version = "3.11" +``` + `LiteralString` represents a string that is either defined directly within the source code or is made up of such components. diff --git a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md index 6699239873103..2204b14ee62fc 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md +++ b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md @@ -1,5 +1,10 @@ # NoReturn & Never +```toml +[environment] +target-version = "3.11" +``` + `NoReturn` is used to annotate the return type for functions that never return. `Never` is the bottom type, representing the empty set of Python objects. These two annotations can be used interchangeably. diff --git a/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md b/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md index 9ee078606d5cd..9415b515dcf16 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md +++ b/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md @@ -34,7 +34,7 @@ def _(flag: bool): if True or (x := 1): # TODO: infer that the second arm is never executed, and raise `unresolved-reference`. # error: [possibly-unresolved-reference] - reveal_type(x) # revealed: Literal[1] + reveal_type(x) # revealed: Never if True and (x := 1): # TODO: infer that the second arm is always executed, do not raise a diagnostic diff --git a/crates/red_knot_python_semantic/resources/mdtest/literal/ellipsis.md b/crates/red_knot_python_semantic/resources/mdtest/literal/ellipsis.md index 2b7bb7c61d9b1..ab7f18f6ee18c 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/literal/ellipsis.md +++ b/crates/red_knot_python_semantic/resources/mdtest/literal/ellipsis.md @@ -1,7 +1,23 @@ # Ellipsis literals -## Simple +## Python 3.9 + +```toml +[environment] +target-version = "3.9" +``` + +```py +reveal_type(...) # revealed: ellipsis +``` + +## Python 3.10 + +```toml +[environment] +target-version = "3.10" +``` ```py -reveal_type(...) # revealed: EllipsisType | ellipsis +reveal_type(...) # revealed: EllipsisType ``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md b/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md index 7da1ad3e36126..282d7b2698ccf 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md +++ b/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md @@ -95,6 +95,11 @@ def _(t: type[object]): ### Handling of `None` +```toml +[environment] +target-version = "3.10" +``` + ```py # TODO: this error should ideally go away once we (1) understand `sys.version_info` branches, # and (2) set the target Python version for this test to 3.10. diff --git a/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md b/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md index de2eccf81ea81..084eb3db0e80a 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md +++ b/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md @@ -7,6 +7,11 @@ PEP 695 type aliases are only available in Python 3.12 and later: python-version = "3.12" ``` +```toml +[environment] +target-version = "3.12" +``` + ## Basic ```py diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md new file mode 100644 index 0000000000000..b7cc8d386d9ee --- /dev/null +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -0,0 +1,294 @@ +# Statically-known branches + +## Always false + +### If + +```py +x = 1 + +if False: + x = 2 + +reveal_type(x) # revealed: Literal[1] +``` + +### Else + +```py +x = 1 + +if True: + pass +else: + x = 2 + +reveal_type(x) # revealed: Literal[1] +``` + +## Always true + +### If + +```py +x = 1 + +if True: + x = 2 + +reveal_type(x) # revealed: Literal[2] +``` + +### Else + +```py +x = 1 + +if False: + pass +else: + x = 2 + +reveal_type(x) # revealed: Literal[2] +``` + +## Combination + +```py +x = 1 + +if True: + x = 2 +else: + x = 3 + +reveal_type(x) # revealed: Literal[2] +``` + +## Nested + +```py path=nested_if_true_if_true.py +x = 1 + +if True: + if True: + x = 2 + else: + x = 3 +else: + x = 4 + +reveal_type(x) # revealed: Literal[2] +``` + +```py path=nested_if_true_if_false.py +x = 1 + +if True: + if False: + x = 2 + else: + x = 3 +else: + x = 4 + +reveal_type(x) # revealed: Literal[3] +``` + +```py path=nested_if_true_if_bool.py +def flag() -> bool: ... + +x = 1 + +if True: + if flag(): + x = 2 + else: + x = 3 +else: + x = 4 + +reveal_type(x) # revealed: Literal[2, 3] +``` + +```py path=nested_if_bool_if_true.py +def flag() -> bool: ... + +x = 1 + +if flag(): + if True: + x = 2 + else: + x = 3 +else: + x = 4 + +reveal_type(x) # revealed: Literal[2, 4] +``` + +```py path=nested_else_if_true.py +x = 1 + +if False: + x = 2 +else: + if True: + x = 3 + else: + x = 4 + +reveal_type(x) # revealed: Literal[3] +``` + +```py path=nested_else_if_false.py +x = 1 + +if False: + x = 2 +else: + if False: + x = 3 + else: + x = 4 + +reveal_type(x) # revealed: Literal[4] +``` + +```py path=nested_else_if_bool.py +def flag() -> bool: ... + +x = 1 + +if False: + x = 2 +else: + if flag(): + x = 3 + else: + x = 4 + +reveal_type(x) # revealed: Literal[3, 4] +``` + +## If-expressions + +### Always true + +```py +x = 1 if True else 2 + +reveal_type(x) # revealed: Literal[1] +``` + +### Always false + +```py +x = 1 if False else 2 + +reveal_type(x) # revealed: Literal[2] +``` + +## Boolean expressions + +### Always true + +```py +(x := 1) == 1 or (x := 2) + +reveal_type(x) # revealed: Literal[1] +``` + +### Always false + +```py +(x := 1) == 0 or (x := 2) + +reveal_type(x) # revealed: Literal[2] +``` + +## Conditional declarations + +```py path=if_false.py +x: str + +if False: + x: int + +def f() -> None: + reveal_type(x) # revealed: str +``` + +```py path=if_true_else.py +x: str + +if True: + pass +else: + x: int + +def f() -> None: + reveal_type(x) # revealed: str +``` + +```py path=if_true.py +x: str + +if True: + x: int + +def f() -> None: + reveal_type(x) # revealed: int +``` + +```py path=if_false_else.py +x: str + +if False: + pass +else: + x: int + +def f() -> None: + reveal_type(x) # revealed: int +``` + +```py path=if_bool.py +def flag() -> bool: ... + +x: str + +if flag(): + x: int + +def f() -> None: + reveal_type(x) # revealed: str | int +``` + +## Conditionally defined functions + +```py +def f() -> int: ... +def g() -> int: ... + +if True: + def f() -> str: ... + +else: + def g() -> str: ... + +reveal_type(f()) # revealed: str +reveal_type(g()) # revealed: int +``` + +## Conditionally defined class attributes + +```py +class C: + if True: + x: int = 1 + else: + x: str = "a" + +reveal_type(C.x) # revealed: int +``` diff --git a/crates/red_knot_python_semantic/src/semantic_index.rs b/crates/red_knot_python_semantic/src/semantic_index.rs index 2771ad301e380..fb681a39ee715 100644 --- a/crates/red_knot_python_semantic/src/semantic_index.rs +++ b/crates/red_knot_python_semantic/src/semantic_index.rs @@ -1251,4 +1251,32 @@ match 1: assert!(matches!(binding.kind(&db), DefinitionKind::For(_))); } + + #[test] + #[ignore] + fn if_statement() { + let TestCase { db, file } = test_case( + " +x = False + +if True: + x: bool +", + ); + + let index = semantic_index(&db, file); + // let global_table = index.symbol_table(FileScopeId::global()); + + let use_def = index.use_def_map(FileScopeId::global()); + + // use_def + + use_def.print(&db); + + panic!(); + // let binding = use_def + // .first_public_binding(global_table.symbol_id_by_name(name).expect("symbol exists")) + // .expect("Expected with item definition for {name}"); + // assert!(matches!(binding.kind(&db), DefinitionKind::WithItem(_))); + } } diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index 27e657aba959b..3686be4a0c532 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -23,7 +23,7 @@ use crate::semantic_index::symbol::{ FileScopeId, NodeWithScopeKey, NodeWithScopeRef, Scope, ScopeId, ScopedSymbolId, SymbolTableBuilder, }; -use crate::semantic_index::use_def::{FlowSnapshot, UseDefMapBuilder}; +use crate::semantic_index::use_def::{ActiveConstraintsSnapshot, FlowSnapshot, UseDefMapBuilder}; use crate::semantic_index::SemanticIndex; use crate::unpack::Unpack; use crate::Db; @@ -200,12 +200,20 @@ impl<'db> SemanticIndexBuilder<'db> { self.current_use_def_map().snapshot() } - fn flow_restore(&mut self, state: FlowSnapshot) { + fn constraints_snapshot(&self) -> ActiveConstraintsSnapshot { + self.current_use_def_map().constraints_snapshot() + } + + fn flow_restore(&mut self, state: FlowSnapshot, active_constraints: ActiveConstraintsSnapshot) { self.current_use_def_map_mut().restore(state); + self.current_use_def_map_mut() + .restore_constraints(active_constraints); } - fn flow_merge(&mut self, state: FlowSnapshot) { + fn flow_merge(&mut self, state: FlowSnapshot, active_constraints: ActiveConstraintsSnapshot) { self.current_use_def_map_mut().merge(state); + self.current_use_def_map_mut() + .restore_constraints(active_constraints); } fn add_symbol(&mut self, name: Name) -> ScopedSymbolId { @@ -785,6 +793,7 @@ where ast::Stmt::If(node) => { self.visit_expr(&node.test); let pre_if = self.flow_snapshot(); + let pre_if_constraints = self.constraints_snapshot(); let constraint = self.record_expression_constraint(&node.test); let mut constraints = vec![constraint]; self.visit_body(&node.body); @@ -810,7 +819,7 @@ where post_clauses.push(self.flow_snapshot()); // we can only take an elif/else branch if none of the previous ones were // taken, so the block entry state is always `pre_if` - self.flow_restore(pre_if.clone()); + self.flow_restore(pre_if.clone(), pre_if_constraints.clone()); for constraint in &constraints { self.record_negated_constraint(*constraint); } @@ -821,7 +830,7 @@ where self.visit_body(clause_body); } for post_clause_state in post_clauses { - self.flow_merge(post_clause_state); + self.flow_merge(post_clause_state, pre_if_constraints.clone()); } } ast::Stmt::While(ast::StmtWhile { @@ -833,6 +842,7 @@ where self.visit_expr(test); let pre_loop = self.flow_snapshot(); + let pre_loop_constraints = self.constraints_snapshot(); let constraint = self.record_expression_constraint(test); // Save aside any break states from an outer loop @@ -852,14 +862,14 @@ where // We may execute the `else` clause without ever executing the body, so merge in // the pre-loop state before visiting `else`. - self.flow_merge(pre_loop); + self.flow_merge(pre_loop, pre_loop_constraints.clone()); self.record_negated_constraint(constraint); self.visit_body(orelse); // Breaking out of a while loop bypasses the `else` clause, so merge in the break // states after visiting `else`. for break_state in break_states { - self.flow_merge(break_state); + self.flow_merge(break_state, pre_loop_constraints.clone()); // TODO? } } ast::Stmt::With(ast::StmtWith { @@ -902,6 +912,7 @@ where self.visit_expr(iter); let pre_loop = self.flow_snapshot(); + let pre_loop_constraints = self.constraints_snapshot(); let saved_break_states = std::mem::take(&mut self.loop_break_states); debug_assert_eq!(&self.current_assignments, &[]); @@ -922,13 +933,13 @@ where // We may execute the `else` clause without ever executing the body, so merge in // the pre-loop state before visiting `else`. - self.flow_merge(pre_loop); + self.flow_merge(pre_loop, pre_loop_constraints.clone()); self.visit_body(orelse); // Breaking out of a `for` loop bypasses the `else` clause, so merge in the break // states after visiting `else`. for break_state in break_states { - self.flow_merge(break_state); + self.flow_merge(break_state, pre_loop_constraints.clone()); } } ast::Stmt::Match(ast::StmtMatch { @@ -940,6 +951,7 @@ where self.visit_expr(subject); let after_subject = self.flow_snapshot(); + let after_subject_cs = self.constraints_snapshot(); let Some((first, remaining)) = cases.split_first() else { return; }; @@ -949,18 +961,18 @@ where let mut post_case_snapshots = vec![]; for case in remaining { post_case_snapshots.push(self.flow_snapshot()); - self.flow_restore(after_subject.clone()); + self.flow_restore(after_subject.clone(), after_subject_cs.clone()); self.add_pattern_constraint(subject, &case.pattern); self.visit_match_case(case); } for post_clause_state in post_case_snapshots { - self.flow_merge(post_clause_state); + self.flow_merge(post_clause_state, after_subject_cs.clone()); } if !cases .last() .is_some_and(|case| case.guard.is_none() && case.pattern.is_wildcard()) { - self.flow_merge(after_subject); + self.flow_merge(after_subject, after_subject_cs); } } ast::Stmt::Try(ast::StmtTry { @@ -978,6 +990,7 @@ where // We will merge this state with all of the intermediate // states during the `try` block before visiting those suites. let pre_try_block_state = self.flow_snapshot(); + let pre_try_block_constraints = self.constraints_snapshot(); self.try_node_context_stack_manager.push_context(); @@ -998,14 +1011,17 @@ where // as there necessarily must have been 0 `except` blocks executed // if we hit the `else` block. let post_try_block_state = self.flow_snapshot(); + let post_try_block_constraints = self.constraints_snapshot(); // Prepare for visiting the `except` block(s) - self.flow_restore(pre_try_block_state); + self.flow_restore(pre_try_block_state, pre_try_block_constraints.clone()); for state in try_block_snapshots { - self.flow_merge(state); + self.flow_merge(state, pre_try_block_constraints.clone()); + // TODO? } let pre_except_state = self.flow_snapshot(); + let pre_except_constraints = self.constraints_snapshot(); let num_handlers = handlers.len(); for (i, except_handler) in handlers.iter().enumerate() { @@ -1044,19 +1060,22 @@ where // as we'll immediately call `self.flow_restore()` to a different state // as soon as this loop over the handlers terminates. if i < (num_handlers - 1) { - self.flow_restore(pre_except_state.clone()); + self.flow_restore( + pre_except_state.clone(), + pre_except_constraints.clone(), + ); } } // If we get to the `else` block, we know that 0 of the `except` blocks can have been executed, // and the entire `try` block must have been executed: - self.flow_restore(post_try_block_state); + self.flow_restore(post_try_block_state, post_try_block_constraints); } self.visit_body(orelse); for post_except_state in post_except_states { - self.flow_merge(post_except_state); + self.flow_merge(post_except_state, pre_try_block_constraints.clone()); } // TODO: there's lots of complexity here that isn't yet handled by our model. @@ -1211,19 +1230,17 @@ where ast::Expr::If(ast::ExprIf { body, test, orelse, .. }) => { - // TODO detect statically known truthy or falsy test (via type inference, not naive - // AST inspection, so we can't simplify here, need to record test expression for - // later checking) self.visit_expr(test); let pre_if = self.flow_snapshot(); + let pre_if_constraints = self.constraints_snapshot(); let constraint = self.record_expression_constraint(test); self.visit_expr(body); let post_body = self.flow_snapshot(); - self.flow_restore(pre_if); + self.flow_restore(pre_if, pre_if_constraints.clone()); self.record_negated_constraint(constraint); self.visit_expr(orelse); - self.flow_merge(post_body); + self.flow_merge(post_body, pre_if_constraints); } ast::Expr::ListComp( list_comprehension @ ast::ExprListComp { @@ -1284,7 +1301,7 @@ where // AST inspection, so we can't simplify here, need to record test expression for // later checking) let mut snapshots = vec![]; - + let pre_op_constraints = self.constraints_snapshot(); for (index, value) in values.iter().enumerate() { self.visit_expr(value); // In the last value we don't need to take a snapshot nor add a constraint @@ -1299,7 +1316,7 @@ where } } for snapshot in snapshots { - self.flow_merge(snapshot); + self.flow_merge(snapshot, pre_op_constraints.clone()); } } _ => { diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 9f3e197c74eee..faeca32130b82 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -221,6 +221,8 @@ //! snapshot, and merging a snapshot into the current state. The logic using these methods lives in //! [`SemanticIndexBuilder`](crate::semantic_index::builder::SemanticIndexBuilder), e.g. where it //! visits a `StmtIf` node. +use std::collections::HashSet; + use self::symbol_state::{ BindingIdWithConstraintsIterator, ConstraintIdIterator, DeclarationIdIterator, ScopedConstraintId, ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState, @@ -268,6 +270,110 @@ pub(crate) struct UseDefMap<'db> { } impl<'db> UseDefMap<'db> { + #[cfg(test)] + #[allow(clippy::print_stdout)] + pub(crate) fn print(&self, db: &dyn crate::db::Db) { + use crate::semantic_index::constraint::ConstraintNode; + + println!("all_definitions:"); + println!("================"); + + for (id, d) in self.all_definitions.iter_enumerated() { + println!( + "{:?}: {:?} {:?} {:?}", + id, + d.category(db), + d.scope(db), + d.symbol(db), + ); + println!(" {:?}", d.kind(db)); + println!(); + } + + println!("all_constraints:"); + println!("================"); + + for (id, c) in self.all_constraints.iter_enumerated() { + println!("{:?}: {:?}", id, c.node); + if let ConstraintNode::Expression(e) = c.node { + println!(" {:?}", e.node_ref(db)); + } + } + + println!(); + + println!("bindings_by_use:"); + println!("================"); + + for (id, bindings) in self.bindings_by_use.iter_enumerated() { + println!("{id:?}:"); + for binding in bindings.iter() { + let definition = self.all_definitions[binding.definition]; + let mut constraint_ids = binding.constraint_ids.peekable(); + let mut active_constraint_ids = + binding.constraints_active_at_binding_ids.peekable(); + + println!(" * {definition:?}"); + + if constraint_ids.peek().is_some() { + println!(" Constraints:"); + for constraint_id in constraint_ids { + println!(" {:?}", self.all_constraints[constraint_id]); + } + } else { + println!(" No constraints"); + } + + println!(); + + if active_constraint_ids.peek().is_some() { + println!(" Active constraints at binding:"); + for constraint_id in active_constraint_ids { + println!(" {:?}", self.all_constraints[constraint_id]); + } + } else { + println!(" No active constraints at binding"); + } + } + } + + println!(); + + println!("public_symbols:"); + println!("================"); + + for (id, symbol) in self.public_symbols.iter_enumerated() { + println!("{id:?}:"); + println!(" * Bindings:"); + for binding in symbol.bindings().iter() { + let definition = self.all_definitions[binding.definition]; + let mut constraint_ids = binding.constraint_ids.peekable(); + + println!(" {definition:?}"); + + if constraint_ids.peek().is_some() { + println!(" Constraints:"); + for constraint_id in constraint_ids { + println!(" {:?}", self.all_constraints[constraint_id]); + } + } else { + println!(" No constraints"); + } + } + + println!(" * Declarations:"); + for (declaration, _) in symbol.declarations().iter() { + let definition = self.all_definitions[declaration]; + println!(" {definition:?}"); + } + + println!(); + } + + println!(); + println!(); + } + pub(crate) fn bindings_at_use( &self, use_id: ScopedUseId, @@ -352,6 +458,7 @@ impl<'db> UseDefMap<'db> { ) -> DeclarationsIterator<'a, 'db> { DeclarationsIterator { all_definitions: &self.all_definitions, + all_constraints: &self.all_constraints, inner: declarations.iter(), may_be_undeclared: declarations.may_be_undeclared(), } @@ -365,7 +472,7 @@ enum SymbolDefinitions { Declarations(SymbolDeclarations), } -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct BindingWithConstraintsIterator<'map, 'db> { all_definitions: &'map IndexVec>, all_constraints: &'map IndexVec>, @@ -384,6 +491,10 @@ impl<'map, 'db> Iterator for BindingWithConstraintsIterator<'map, 'db> { all_constraints: self.all_constraints, constraint_ids: def_id_with_constraints.constraint_ids, }, + constraints_active_at_binding: ConstraintsIterator { + all_constraints: self.all_constraints, + constraint_ids: def_id_with_constraints.constraints_active_at_binding_ids, + }, }) } } @@ -393,8 +504,10 @@ impl std::iter::FusedIterator for BindingWithConstraintsIterator<'_, '_> {} pub(crate) struct BindingWithConstraints<'map, 'db> { pub(crate) binding: Definition<'db>, pub(crate) constraints: ConstraintsIterator<'map, 'db>, + pub(crate) constraints_active_at_binding: ConstraintsIterator<'map, 'db>, } +#[derive(Debug, Clone)] pub(crate) struct ConstraintsIterator<'map, 'db> { all_constraints: &'map IndexVec>, constraint_ids: ConstraintIdIterator<'map>, @@ -414,6 +527,7 @@ impl std::iter::FusedIterator for ConstraintsIterator<'_, '_> {} pub(crate) struct DeclarationsIterator<'map, 'db> { all_definitions: &'map IndexVec>, + all_constraints: &'map IndexVec>, inner: DeclarationIdIterator<'map>, may_be_undeclared: bool, } @@ -424,11 +538,19 @@ impl DeclarationsIterator<'_, '_> { } } -impl<'db> Iterator for DeclarationsIterator<'_, 'db> { - type Item = Definition<'db>; +impl<'map, 'db> Iterator for DeclarationsIterator<'map, 'db> { + type Item = (Definition<'db>, ConstraintsIterator<'map, 'db>); fn next(&mut self) -> Option { - self.inner.next().map(|def_id| self.all_definitions[def_id]) + self.inner.next().map(|(def_id, constraints)| { + ( + self.all_definitions[def_id], + ConstraintsIterator { + all_constraints: self.all_constraints, + constraint_ids: constraints, + }, + ) + }) } } @@ -440,6 +562,9 @@ pub(super) struct FlowSnapshot { symbol_states: IndexVec, } +#[derive(Clone, Debug)] +pub(super) struct ActiveConstraintsSnapshot(HashSet); + #[derive(Debug, Default)] pub(super) struct UseDefMapBuilder<'db> { /// Append-only array of [`Definition`]. @@ -448,6 +573,8 @@ pub(super) struct UseDefMapBuilder<'db> { /// Append-only array of [`Constraint`]. all_constraints: IndexVec>, + active_constraints: HashSet, + /// Live bindings at each so-far-recorded use. bindings_by_use: IndexVec, @@ -471,7 +598,7 @@ impl<'db> UseDefMapBuilder<'db> { binding, SymbolDefinitions::Declarations(symbol_state.declarations().clone()), ); - symbol_state.record_binding(def_id); + symbol_state.record_binding(def_id, &self.active_constraints); } pub(super) fn record_constraint(&mut self, constraint: Constraint<'db>) { @@ -479,6 +606,7 @@ impl<'db> UseDefMapBuilder<'db> { for state in &mut self.symbol_states { state.record_constraint(constraint_id); } + self.active_constraints.insert(constraint_id); } pub(super) fn record_declaration( @@ -492,7 +620,7 @@ impl<'db> UseDefMapBuilder<'db> { declaration, SymbolDefinitions::Bindings(symbol_state.bindings().clone()), ); - symbol_state.record_declaration(def_id); + symbol_state.record_declaration(def_id, &self.active_constraints); } pub(super) fn record_declaration_and_binding( @@ -503,8 +631,8 @@ impl<'db> UseDefMapBuilder<'db> { // We don't need to store anything in self.definitions_by_definition. let def_id = self.all_definitions.push(definition); let symbol_state = &mut self.symbol_states[symbol]; - symbol_state.record_declaration(def_id); - symbol_state.record_binding(def_id); + symbol_state.record_declaration(def_id, &self.active_constraints); + symbol_state.record_binding(def_id, &self.active_constraints); } pub(super) fn record_use(&mut self, symbol: ScopedSymbolId, use_id: ScopedUseId) { @@ -523,6 +651,10 @@ impl<'db> UseDefMapBuilder<'db> { } } + pub(super) fn constraints_snapshot(&self) -> ActiveConstraintsSnapshot { + ActiveConstraintsSnapshot(self.active_constraints.clone()) + } + /// Restore the current builder symbols state to the given snapshot. pub(super) fn restore(&mut self, snapshot: FlowSnapshot) { // We never remove symbols from `symbol_states` (it's an IndexVec, and the symbol @@ -541,6 +673,10 @@ impl<'db> UseDefMapBuilder<'db> { .resize(num_symbols, SymbolState::undefined()); } + pub(super) fn restore_constraints(&mut self, snapshot: ActiveConstraintsSnapshot) { + self.active_constraints = snapshot.0; + } + /// Merge the given snapshot into the current state, reflecting that we might have taken either /// path to get here. The new state for each symbol should include definitions from both the /// prior state and the snapshot. diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs index 464f718e7b4f4..84ac7305d8b0f 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs @@ -98,6 +98,7 @@ impl BitSet { } /// Union in-place with another [`BitSet`]. + #[allow(dead_code)] pub(super) fn union(&mut self, other: &BitSet) { let mut max_len = self.blocks().len(); let other_len = other.blocks().len(); @@ -122,7 +123,7 @@ impl BitSet { } /// Iterator over values in a [`BitSet`]. -#[derive(Debug)] +#[derive(Debug, Clone)] pub(super) struct BitSetIterator<'a, const B: usize> { /// The blocks we are iterating over. blocks: &'a [u64], diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 506300067c952..7871a52d16f93 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -43,6 +43,8 @@ //! //! Tracking live declarations is simpler, since constraints are not involved, but otherwise very //! similar to tracking live bindings. +use std::collections::HashSet; + use super::bitset::{BitSet, BitSetIterator}; use ruff_index::newtype_index; use smallvec::SmallVec; @@ -87,6 +89,8 @@ pub(super) struct SymbolDeclarations { /// [`BitSet`]: which declarations (as [`ScopedDefinitionId`]) can reach the current location? live_declarations: Declarations, + constraints_active_at_declaration: Constraints, // TODO: rename to constraints_active_at_declaration + /// Could the symbol be un-declared at this point? may_be_undeclared: bool, } @@ -95,14 +99,27 @@ impl SymbolDeclarations { fn undeclared() -> Self { Self { live_declarations: Declarations::default(), + constraints_active_at_declaration: Constraints::default(), may_be_undeclared: true, } } /// Record a newly-encountered declaration for this symbol. - fn record_declaration(&mut self, declaration_id: ScopedDefinitionId) { + fn record_declaration( + &mut self, + declaration_id: ScopedDefinitionId, + active_constraints: &HashSet, + ) { self.live_declarations = Declarations::with(declaration_id.into()); self.may_be_undeclared = false; + + // TODO: unify code with below + self.constraints_active_at_declaration = Constraints::with_capacity(1); + self.constraints_active_at_declaration + .push(BitSet::default()); + for active_constraint_id in active_constraints { + self.constraints_active_at_declaration[0].insert(active_constraint_id.as_u32()); + } } /// Add undeclared as a possibility for this symbol. @@ -114,6 +131,7 @@ impl SymbolDeclarations { pub(super) fn iter(&self) -> DeclarationIdIterator { DeclarationIdIterator { inner: self.live_declarations.iter(), + constraints_active_at_binding: self.constraints_active_at_declaration.iter(), } } @@ -138,6 +156,8 @@ pub(super) struct SymbolBindings { /// binding in `live_bindings`. constraints: Constraints, + constraints_active_at_binding: Constraints, + /// Could the symbol be unbound at this point? may_be_unbound: bool, } @@ -147,6 +167,7 @@ impl SymbolBindings { Self { live_bindings: Bindings::default(), constraints: Constraints::default(), + constraints_active_at_binding: Constraints::default(), may_be_unbound: true, } } @@ -157,12 +178,21 @@ impl SymbolBindings { } /// Record a newly-encountered binding for this symbol. - pub(super) fn record_binding(&mut self, binding_id: ScopedDefinitionId) { + pub(super) fn record_binding( + &mut self, + binding_id: ScopedDefinitionId, + active_constraints: &HashSet, + ) { // The new binding replaces all previous live bindings in this path, and has no // constraints. self.live_bindings = Bindings::with(binding_id.into()); self.constraints = Constraints::with_capacity(1); self.constraints.push(BitSet::default()); + self.constraints_active_at_binding = Constraints::with_capacity(1); + self.constraints_active_at_binding.push(BitSet::default()); + for active_constraint_id in active_constraints { + self.constraints_active_at_binding[0].insert(active_constraint_id.as_u32()); + } self.may_be_unbound = false; } @@ -178,6 +208,7 @@ impl SymbolBindings { BindingIdWithConstraintsIterator { definitions: self.live_bindings.iter(), constraints: self.constraints.iter(), + constraints_active_at_binding: self.constraints_active_at_binding.iter(), } } @@ -207,8 +238,12 @@ impl SymbolState { } /// Record a newly-encountered binding for this symbol. - pub(super) fn record_binding(&mut self, binding_id: ScopedDefinitionId) { - self.bindings.record_binding(binding_id); + pub(super) fn record_binding( + &mut self, + binding_id: ScopedDefinitionId, + active_constraints: &HashSet, + ) { + self.bindings.record_binding(binding_id, active_constraints); } /// Add given constraint to all live bindings. @@ -222,8 +257,13 @@ impl SymbolState { } /// Record a newly-encountered declaration of this symbol. - pub(super) fn record_declaration(&mut self, declaration_id: ScopedDefinitionId) { - self.declarations.record_declaration(declaration_id); + pub(super) fn record_declaration( + &mut self, + declaration_id: ScopedDefinitionId, + active_constraints: &HashSet, + ) { + self.declarations + .record_declaration(declaration_id, active_constraints); } /// Merge another [`SymbolState`] into this one. @@ -232,24 +272,93 @@ impl SymbolState { bindings: SymbolBindings { live_bindings: Bindings::default(), constraints: Constraints::default(), + constraints_active_at_binding: Constraints::default(), // TODO may_be_unbound: self.bindings.may_be_unbound || b.bindings.may_be_unbound, }, declarations: SymbolDeclarations { live_declarations: self.declarations.live_declarations.clone(), + constraints_active_at_declaration: Constraints::default(), // TODO may_be_undeclared: self.declarations.may_be_undeclared || b.declarations.may_be_undeclared, }, }; + // let mut constraints_active_at_binding = BitSet::default(); + // for active_constraint_id in active_constraints.0 { + // constraints_active_at_binding.insert(active_constraint_id.as_u32()); + // } + std::mem::swap(&mut a, self); - self.declarations - .live_declarations - .union(&b.declarations.live_declarations); + // self.declarations + // .live_declarations + // .union(&b.declarations.live_declarations); + + let mut a_decls_iter = a.declarations.live_declarations.iter(); + let mut b_decls_iter = b.declarations.live_declarations.iter(); + let mut a_constraints_active_at_declaration_iter = + a.declarations.constraints_active_at_declaration.into_iter(); + let mut b_constraints_active_at_declaration_iter = + b.declarations.constraints_active_at_declaration.into_iter(); + + let mut opt_a_decl: Option = a_decls_iter.next(); + let mut opt_b_decl: Option = b_decls_iter.next(); + + let push = |decl, + constraints_active_at_declaration_iter: &mut ConstraintsIntoIterator, + merged: &mut Self| { + merged.declarations.live_declarations.insert(decl); + let constraints_active_at_binding = constraints_active_at_declaration_iter + .next() + .expect("declarations and constraints_active_at_binding length mismatch"); + merged + .declarations + .constraints_active_at_declaration + .push(constraints_active_at_binding); + }; + + loop { + match (opt_a_decl, opt_b_decl) { + (Some(a_decl), Some(b_decl)) => match a_decl.cmp(&b_decl) { + std::cmp::Ordering::Less => { + push(a_decl, &mut a_constraints_active_at_declaration_iter, self); + opt_a_decl = a_decls_iter.next(); + } + std::cmp::Ordering::Greater => { + push(b_decl, &mut b_constraints_active_at_declaration_iter, self); + opt_b_decl = b_decls_iter.next(); + } + std::cmp::Ordering::Equal => { + push(a_decl, &mut b_constraints_active_at_declaration_iter, self); + self.declarations + .constraints_active_at_declaration + .last_mut() + .unwrap() + .intersect(&a_constraints_active_at_declaration_iter.next().unwrap()); + + opt_a_decl = a_decls_iter.next(); + opt_b_decl = b_decls_iter.next(); + } + }, + (Some(a_decl), None) => { + push(a_decl, &mut a_constraints_active_at_declaration_iter, self); + opt_a_decl = a_decls_iter.next(); + } + (None, Some(b_decl)) => { + push(b_decl, &mut b_constraints_active_at_declaration_iter, self); + opt_b_decl = b_decls_iter.next(); + } + (None, None) => break, + } + } let mut a_defs_iter = a.bindings.live_bindings.iter(); let mut b_defs_iter = b.bindings.live_bindings.iter(); let mut a_constraints_iter = a.bindings.constraints.into_iter(); let mut b_constraints_iter = b.bindings.constraints.into_iter(); + let mut a_constraints_active_at_binding_iter = + a.bindings.constraints_active_at_binding.into_iter(); + let mut b_constraints_active_at_binding_iter = + b.bindings.constraints_active_at_binding.into_iter(); let mut opt_a_def: Option = a_defs_iter.next(); let mut opt_b_def: Option = b_defs_iter.next(); @@ -261,7 +370,10 @@ impl SymbolState { // path is irrelevant. // Helper to push `def`, with constraints in `constraints_iter`, onto `self`. - let push = |def, constraints_iter: &mut ConstraintsIntoIterator, merged: &mut Self| { + let push = |def, + constraints_iter: &mut ConstraintsIntoIterator, + constraints_active_at_binding_iter: &mut ConstraintsIntoIterator, + merged: &mut Self| { merged.bindings.live_bindings.insert(def); // SAFETY: we only ever create SymbolState with either no definitions and no constraint // bitsets (`::unbound`) or one definition and one constraint bitset (`::with`), and @@ -271,7 +383,14 @@ impl SymbolState { let constraints = constraints_iter .next() .expect("definitions and constraints length mismatch"); + let constraints_active_at_binding = constraints_active_at_binding_iter + .next() + .expect("definitions and constraints_active_at_binding length mismatch"); merged.bindings.constraints.push(constraints); + merged + .bindings + .constraints_active_at_binding + .push(constraints_active_at_binding); }; loop { @@ -279,17 +398,32 @@ impl SymbolState { (Some(a_def), Some(b_def)) => match a_def.cmp(&b_def) { std::cmp::Ordering::Less => { // Next definition ID is only in `a`, push it to `self` and advance `a`. - push(a_def, &mut a_constraints_iter, self); + push( + a_def, + &mut a_constraints_iter, + &mut a_constraints_active_at_binding_iter, + self, + ); opt_a_def = a_defs_iter.next(); } std::cmp::Ordering::Greater => { // Next definition ID is only in `b`, push it to `self` and advance `b`. - push(b_def, &mut b_constraints_iter, self); + push( + b_def, + &mut b_constraints_iter, + &mut b_constraints_active_at_binding_iter, + self, + ); opt_b_def = b_defs_iter.next(); } std::cmp::Ordering::Equal => { // Next definition is in both; push to `self` and intersect constraints. - push(a_def, &mut b_constraints_iter, self); + push( + a_def, + &mut b_constraints_iter, + &mut b_constraints_active_at_binding_iter, + self, + ); // SAFETY: we only ever create SymbolState with either no definitions and // no constraint bitsets (`::unbound`) or one definition and one constraint // bitset (`::with`), and `::merge` always pushes one definition and one @@ -298,6 +432,11 @@ impl SymbolState { let a_constraints = a_constraints_iter .next() .expect("definitions and constraints length mismatch"); + // let _a_constraints_active_at_binding = + // a_constraints_active_at_binding_iter.next().expect( + // "definitions and constraints_active_at_binding length mismatch", + // ); // TODO: perform check that we see the same constraints in both paths + // If the same definition is visible through both paths, any constraint // that applies on only one path is irrelevant to the resulting type from // unioning the two paths, so we intersect the constraints. @@ -306,18 +445,29 @@ impl SymbolState { .last_mut() .unwrap() .intersect(&a_constraints); + opt_a_def = a_defs_iter.next(); opt_b_def = b_defs_iter.next(); } }, (Some(a_def), None) => { // We've exhausted `b`, just push the def from `a` and move on to the next. - push(a_def, &mut a_constraints_iter, self); + push( + a_def, + &mut a_constraints_iter, + &mut a_constraints_active_at_binding_iter, + self, + ); opt_a_def = a_defs_iter.next(); } (None, Some(b_def)) => { // We've exhausted `a`, just push the def from `b` and move on to the next. - push(b_def, &mut b_constraints_iter, self); + push( + b_def, + &mut b_constraints_iter, + &mut b_constraints_active_at_binding_iter, + self, + ); opt_b_def = b_defs_iter.next(); } (None, None) => break, @@ -353,26 +503,37 @@ impl Default for SymbolState { pub(super) struct BindingIdWithConstraints<'a> { pub(super) definition: ScopedDefinitionId, pub(super) constraint_ids: ConstraintIdIterator<'a>, + pub(super) constraints_active_at_binding_ids: ConstraintIdIterator<'a>, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub(super) struct BindingIdWithConstraintsIterator<'a> { definitions: BindingsIterator<'a>, constraints: ConstraintsIterator<'a>, + constraints_active_at_binding: ConstraintsIterator<'a>, } impl<'a> Iterator for BindingIdWithConstraintsIterator<'a> { type Item = BindingIdWithConstraints<'a>; fn next(&mut self) -> Option { - match (self.definitions.next(), self.constraints.next()) { - (None, None) => None, - (Some(def), Some(constraints)) => Some(BindingIdWithConstraints { - definition: ScopedDefinitionId::from_u32(def), - constraint_ids: ConstraintIdIterator { - wrapped: constraints.iter(), - }, - }), + match ( + self.definitions.next(), + self.constraints.next(), + self.constraints_active_at_binding.next(), + ) { + (None, None, None) => None, + (Some(def), Some(constraints), Some(constraints_active_at_binding)) => { + Some(BindingIdWithConstraints { + definition: ScopedDefinitionId::from_u32(def), + constraint_ids: ConstraintIdIterator { + wrapped: constraints.iter(), + }, + constraints_active_at_binding_ids: ConstraintIdIterator { + wrapped: constraints_active_at_binding.iter(), + }, + }) + } // SAFETY: see above. _ => unreachable!("definitions and constraints length mismatch"), } @@ -381,7 +542,7 @@ impl<'a> Iterator for BindingIdWithConstraintsIterator<'a> { impl std::iter::FusedIterator for BindingIdWithConstraintsIterator<'_> {} -#[derive(Debug)] +#[derive(Debug, Clone)] pub(super) struct ConstraintIdIterator<'a> { wrapped: BitSetIterator<'a, INLINE_CONSTRAINT_BLOCKS>, } @@ -399,13 +560,25 @@ impl std::iter::FusedIterator for ConstraintIdIterator<'_> {} #[derive(Debug)] pub(super) struct DeclarationIdIterator<'a> { inner: DeclarationsIterator<'a>, + constraints_active_at_binding: ConstraintsIterator<'a>, } -impl Iterator for DeclarationIdIterator<'_> { - type Item = ScopedDefinitionId; +impl<'a> Iterator for DeclarationIdIterator<'a> { + type Item = (ScopedDefinitionId, ConstraintIdIterator<'a>); fn next(&mut self) -> Option { - self.inner.next().map(ScopedDefinitionId::from_u32) + // self.inner.next().map(ScopedDefinitionId::from_u32) + match (self.inner.next(), self.constraints_active_at_binding.next()) { + (None, None) => None, + (Some(declaration), Some(constraints_active_at_binding)) => Some(( + ScopedDefinitionId::from_u32(declaration), + ConstraintIdIterator { + wrapped: constraints_active_at_binding.iter(), + }, + )), + // SAFETY: see above. + _ => unreachable!("declarations and constraints_active_at_binding length mismatch"), + } } } @@ -413,7 +586,7 @@ impl std::iter::FusedIterator for DeclarationIdIterator<'_> {} #[cfg(test)] mod tests { - use super::{ScopedConstraintId, ScopedDefinitionId, SymbolState}; + use super::{ScopedConstraintId, SymbolState}; fn assert_bindings(symbol: &SymbolState, may_be_unbound: bool, expected: &[&str]) { assert_eq!(symbol.may_be_unbound(), may_be_unbound); @@ -445,7 +618,7 @@ mod tests { let actual = symbol .declarations() .iter() - .map(ScopedDefinitionId::as_u32) + .map(|(d, _)| d.as_u32()) // TODO: constraints .collect::>(); assert_eq!(actual, expected); } @@ -457,76 +630,76 @@ mod tests { assert_bindings(&sym, true, &[]); } - #[test] - fn with() { - let mut sym = SymbolState::undefined(); - sym.record_binding(ScopedDefinitionId::from_u32(0)); + // #[test] + // fn with() { + // let mut sym = SymbolState::undefined(); + // sym.record_binding(ScopedDefinitionId::from_u32(0)); - assert_bindings(&sym, false, &["0<>"]); - } + // assert_bindings(&sym, false, &["0<>"]); + // } - #[test] - fn set_may_be_unbound() { - let mut sym = SymbolState::undefined(); - sym.record_binding(ScopedDefinitionId::from_u32(0)); - sym.set_may_be_unbound(); + // #[test] + // fn set_may_be_unbound() { + // let mut sym = SymbolState::undefined(); + // sym.record_binding(ScopedDefinitionId::from_u32(0)); + // sym.set_may_be_unbound(); - assert_bindings(&sym, true, &["0<>"]); - } + // assert_bindings(&sym, true, &["0<>"]); + // } - #[test] - fn record_constraint() { - let mut sym = SymbolState::undefined(); - sym.record_binding(ScopedDefinitionId::from_u32(0)); - sym.record_constraint(ScopedConstraintId::from_u32(0)); + // #[test] + // fn record_constraint() { + // let mut sym = SymbolState::undefined(); + // sym.record_binding(ScopedDefinitionId::from_u32(0)); + // sym.record_constraint(ScopedConstraintId::from_u32(0)); - assert_bindings(&sym, false, &["0<0>"]); - } + // assert_bindings(&sym, false, &["0<0>"]); + // } - #[test] - fn merge() { - // merging the same definition with the same constraint keeps the constraint - let mut sym0a = SymbolState::undefined(); - sym0a.record_binding(ScopedDefinitionId::from_u32(0)); - sym0a.record_constraint(ScopedConstraintId::from_u32(0)); - - let mut sym0b = SymbolState::undefined(); - sym0b.record_binding(ScopedDefinitionId::from_u32(0)); - sym0b.record_constraint(ScopedConstraintId::from_u32(0)); - - sym0a.merge(sym0b); - let mut sym0 = sym0a; - assert_bindings(&sym0, false, &["0<0>"]); - - // merging the same definition with differing constraints drops all constraints - let mut sym1a = SymbolState::undefined(); - sym1a.record_binding(ScopedDefinitionId::from_u32(1)); - sym1a.record_constraint(ScopedConstraintId::from_u32(1)); - - let mut sym1b = SymbolState::undefined(); - sym1b.record_binding(ScopedDefinitionId::from_u32(1)); - sym1b.record_constraint(ScopedConstraintId::from_u32(2)); - - sym1a.merge(sym1b); - let sym1 = sym1a; - assert_bindings(&sym1, false, &["1<>"]); - - // merging a constrained definition with unbound keeps both - let mut sym2a = SymbolState::undefined(); - sym2a.record_binding(ScopedDefinitionId::from_u32(2)); - sym2a.record_constraint(ScopedConstraintId::from_u32(3)); - - let sym2b = SymbolState::undefined(); - - sym2a.merge(sym2b); - let sym2 = sym2a; - assert_bindings(&sym2, true, &["2<3>"]); - - // merging different definitions keeps them each with their existing constraints - sym0.merge(sym2); - let sym = sym0; - assert_bindings(&sym, true, &["0<0>", "2<3>"]); - } + // #[test] + // fn merge() { + // // merging the same definition with the same constraint keeps the constraint + // let mut sym0a = SymbolState::undefined(); + // sym0a.record_binding(ScopedDefinitionId::from_u32(0)); + // sym0a.record_constraint(ScopedConstraintId::from_u32(0)); + + // let mut sym0b = SymbolState::undefined(); + // sym0b.record_binding(ScopedDefinitionId::from_u32(0)); + // sym0b.record_constraint(ScopedConstraintId::from_u32(0)); + + // sym0a.merge(sym0b); + // let mut sym0 = sym0a; + // assert_bindings(&sym0, false, &["0<0>"]); + + // // merging the same definition with differing constraints drops all constraints + // let mut sym1a = SymbolState::undefined(); + // sym1a.record_binding(ScopedDefinitionId::from_u32(1)); + // sym1a.record_constraint(ScopedConstraintId::from_u32(1)); + + // let mut sym1b = SymbolState::undefined(); + // sym1b.record_binding(ScopedDefinitionId::from_u32(1)); + // sym1b.record_constraint(ScopedConstraintId::from_u32(2)); + + // sym1a.merge(sym1b); + // let sym1 = sym1a; + // assert_bindings(&sym1, false, &["1<>"]); + + // // merging a constrained definition with unbound keeps both + // let mut sym2a = SymbolState::undefined(); + // sym2a.record_binding(ScopedDefinitionId::from_u32(2)); + // sym2a.record_constraint(ScopedConstraintId::from_u32(3)); + + // let sym2b = SymbolState::undefined(); + + // sym2a.merge(sym2b); + // let sym2 = sym2a; + // assert_bindings(&sym2, true, &["2<3>"]); + + // // merging different definitions keeps them each with their existing constraints + // sym0.merge(sym2); + // let sym = sym0; + // assert_bindings(&sym, true, &["0<0>", "2<3>"]); + // } #[test] fn no_declaration() { @@ -535,54 +708,54 @@ mod tests { assert_declarations(&sym, true, &[]); } - #[test] - fn record_declaration() { - let mut sym = SymbolState::undefined(); - sym.record_declaration(ScopedDefinitionId::from_u32(1)); + // #[test] + // fn record_declaration() { + // let mut sym = SymbolState::undefined(); + // sym.record_declaration(ScopedDefinitionId::from_u32(1)); - assert_declarations(&sym, false, &[1]); - } + // assert_declarations(&sym, false, &[1]); + // } - #[test] - fn record_declaration_override() { - let mut sym = SymbolState::undefined(); - sym.record_declaration(ScopedDefinitionId::from_u32(1)); - sym.record_declaration(ScopedDefinitionId::from_u32(2)); + // #[test] + // fn record_declaration_override() { + // let mut sym = SymbolState::undefined(); + // sym.record_declaration(ScopedDefinitionId::from_u32(1)); + // sym.record_declaration(ScopedDefinitionId::from_u32(2)); - assert_declarations(&sym, false, &[2]); - } + // assert_declarations(&sym, false, &[2]); + // } - #[test] - fn record_declaration_merge() { - let mut sym = SymbolState::undefined(); - sym.record_declaration(ScopedDefinitionId::from_u32(1)); + // #[test] + // fn record_declaration_merge() { + // let mut sym = SymbolState::undefined(); + // sym.record_declaration(ScopedDefinitionId::from_u32(1)); - let mut sym2 = SymbolState::undefined(); - sym2.record_declaration(ScopedDefinitionId::from_u32(2)); + // let mut sym2 = SymbolState::undefined(); + // sym2.record_declaration(ScopedDefinitionId::from_u32(2)); - sym.merge(sym2); + // sym.merge(sym2); - assert_declarations(&sym, false, &[1, 2]); - } + // assert_declarations(&sym, false, &[1, 2]); + // } - #[test] - fn record_declaration_merge_partial_undeclared() { - let mut sym = SymbolState::undefined(); - sym.record_declaration(ScopedDefinitionId::from_u32(1)); + // #[test] + // fn record_declaration_merge_partial_undeclared() { + // let mut sym = SymbolState::undefined(); + // sym.record_declaration(ScopedDefinitionId::from_u32(1)); - let sym2 = SymbolState::undefined(); + // let sym2 = SymbolState::undefined(); - sym.merge(sym2); + // sym.merge(sym2); - assert_declarations(&sym, true, &[1]); - } + // assert_declarations(&sym, true, &[1]); + // } - #[test] - fn set_may_be_undeclared() { - let mut sym = SymbolState::undefined(); - sym.record_declaration(ScopedDefinitionId::from_u32(0)); - sym.set_may_be_undeclared(); + // #[test] + // fn set_may_be_undeclared() { + // let mut sym = SymbolState::undefined(); + // sym.record_declaration(ScopedDefinitionId::from_u32(0)); + // sym.set_may_be_undeclared(); - assert_declarations(&sym, true, &[0]); - } + // assert_declarations(&sym, true, &[0]); + // } } diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 83b02eddb6cf5..311ac4706ef15 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -16,6 +16,7 @@ pub(crate) use self::infer::{ pub(crate) use self::signatures::Signature; use crate::module_resolver::file_to_module; use crate::semantic_index::ast_ids::HasScopedExpressionId; +use crate::semantic_index::constraint::ConstraintNode; use crate::semantic_index::definition::Definition; use crate::semantic_index::symbol::{self as symbol, ScopeId, ScopedSymbolId}; use crate::semantic_index::{ @@ -236,6 +237,12 @@ fn definition_expression_ty<'db>( } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum UnconditionallyVisible { + Yes, + No, +} + /// Infer the combined type of an iterator of bindings. /// /// Will return a union if there is more than one binding. @@ -243,29 +250,88 @@ fn bindings_ty<'db>( db: &'db dyn Db, bindings_with_constraints: BindingWithConstraintsIterator<'_, 'db>, ) -> Option> { - let mut def_types = bindings_with_constraints.map( + let def_types = bindings_with_constraints.map( |BindingWithConstraints { binding, constraints, + constraints_active_at_binding, }| { - let mut constraint_tys = constraints - .filter_map(|constraint| narrowing_constraint(db, constraint, binding)) - .peekable(); - - let binding_ty = binding_ty(db, binding); - if constraint_tys.peek().is_some() { - constraint_tys - .fold( - IntersectionBuilder::new(db).add_positive(binding_ty), - IntersectionBuilder::add_positive, - ) - .build() + let test_expr_tys = || { + constraints_active_at_binding.clone().map(|c| { + let ty = if let ConstraintNode::Expression(test_expr) = c.node { + let inference = infer_expression_types(db, test_expr); + let scope = test_expr.scope(db); + inference + .expression_ty(test_expr.node_ref(db).scoped_expression_id(db, scope)) + } else { + // TODO: handle other constraint nodes + todo_type!() + }; + + (c, ty) + }) + }; + + if test_expr_tys().any(|(c, test_expr_ty)| { + if c.is_positive { + test_expr_ty.bool(db).is_always_false() + } else { + test_expr_ty.bool(db).is_always_true() + } + }) { + // TODO: do we need to call binding_ty(…) even if we don't need the result? + (Type::Never, UnconditionallyVisible::No) } else { - binding_ty + let mut test_expr_tys_iter = test_expr_tys().peekable(); + + let unconditionally_visible = if test_expr_tys_iter.peek().is_some() + && test_expr_tys_iter.all(|(c, test_expr_ty)| { + if c.is_positive { + test_expr_ty.bool(db).is_always_true() + } else { + test_expr_ty.bool(db).is_always_false() + } + }) { + UnconditionallyVisible::Yes + } else { + UnconditionallyVisible::No + }; + + let mut constraint_tys = constraints + .filter_map(|constraint| narrowing_constraint(db, constraint, binding)) + .peekable(); + + let binding_ty = binding_ty(db, binding); + if constraint_tys.peek().is_some() { + let intersection_ty = constraint_tys + .fold( + IntersectionBuilder::new(db).add_positive(binding_ty), + IntersectionBuilder::add_positive, + ) + .build(); + (intersection_ty, unconditionally_visible) + } else { + (binding_ty, unconditionally_visible) + } } }, ); + // TODO: get rid of all the collects and clean up, obviously + let def_types: Vec<_> = def_types.collect(); + + // shrink the vector to only include everything from the last unconditionally visible binding + let def_types: Vec<_> = def_types + .iter() + .rev() + .take_while_inclusive(|(_, unconditionally_visible)| { + *unconditionally_visible != UnconditionallyVisible::Yes + }) + .map(|(ty, _)| *ty) + .collect(); + + let mut def_types = def_types.into_iter().rev(); + if let Some(first) = def_types.next() { if let Some(second) = def_types.next() { Some(UnionType::from_elements( @@ -301,7 +367,63 @@ fn declarations_ty<'db>( declarations: DeclarationsIterator<'_, 'db>, undeclared_ty: Option>, ) -> DeclaredTypeResult<'db> { - let decl_types = declarations.map(|declaration| declaration_ty(db, declaration)); + let decl_types = declarations.map(|(declaration, constraints_active_at_declaration)| { + let test_expr_tys = || { + constraints_active_at_declaration.clone().map(|c| { + let ty = if let ConstraintNode::Expression(test_expr) = c.node { + let inference = infer_expression_types(db, test_expr); + let scope = test_expr.scope(db); + inference.expression_ty(test_expr.node_ref(db).scoped_expression_id(db, scope)) + } else { + // TODO: handle other constraint nodes + todo_type!() + }; + + (c, ty) + }) + }; + + if test_expr_tys().any(|(c, test_expr_ty)| { + if c.is_positive { + test_expr_ty.bool(db).is_always_false() + } else { + test_expr_ty.bool(db).is_always_true() + } + }) { + (Type::Never, UnconditionallyVisible::No) + } else { + let mut test_expr_tys_iter = test_expr_tys().peekable(); + + if test_expr_tys_iter.peek().is_some() + && test_expr_tys_iter.all(|(c, test_expr_ty)| { + if c.is_positive { + test_expr_ty.bool(db).is_always_true() + } else { + test_expr_ty.bool(db).is_always_false() + } + }) + { + (declaration_ty(db, declaration), UnconditionallyVisible::Yes) + } else { + (declaration_ty(db, declaration), UnconditionallyVisible::No) + } + } + }); + + // TODO: get rid of all the collects and clean up, obviously + let decl_types: Vec<_> = decl_types.collect(); + + // shrink the vector to only include everything from the last unconditionally visible binding + let decl_types: Vec<_> = decl_types + .iter() + .rev() + .take_while_inclusive(|(_, unconditionally_visible)| { + *unconditionally_visible != UnconditionallyVisible::Yes + }) + .map(|(ty, _)| *ty) + .collect(); + + let decl_types = decl_types.into_iter().rev(); let mut all_types = undeclared_ty.into_iter().chain(decl_types); @@ -827,26 +949,6 @@ impl<'db> Type<'db> { return false; } - // TODO: The following is a workaround that is required to unify the two different versions - // of `NoneType` and `NoDefaultType` in typeshed. This should not be required anymore once - // we understand `sys.version_info` branches. - if let ( - Type::Instance(InstanceType { class: self_class }), - Type::Instance(InstanceType { - class: target_class, - }), - ) = (self, other) - { - let self_known = self_class.known(db); - if matches!( - self_known, - Some(KnownClass::NoneType | KnownClass::NoDefaultType) - ) && self_known == target_class.known(db) - { - return true; - } - } - // type[object] ≡ type if let ( Type::SubclassOf(SubclassOfType { @@ -2522,6 +2624,14 @@ impl Truthiness { matches!(self, Truthiness::Ambiguous) } + const fn is_always_false(self) -> bool { + matches!(self, Truthiness::AlwaysFalse) + } + + const fn is_always_true(self) -> bool { + matches!(self, Truthiness::AlwaysTrue) + } + const fn negate(self) -> Self { match self { Self::AlwaysTrue => Self::AlwaysFalse, From 0075b4b9187cb056855e14cc8407ec30e003e841 Mon Sep 17 00:00:00 2001 From: David Peter Date: Thu, 5 Dec 2024 13:09:43 +0100 Subject: [PATCH 02/68] Use BitSet instead of HashSet --- .../src/semantic_index/use_def.rs | 12 ++++++----- .../semantic_index/use_def/symbol_state.rs | 20 +++++++++---------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index faeca32130b82..33657fc644cdc 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -221,8 +221,6 @@ //! snapshot, and merging a snapshot into the current state. The logic using these methods lives in //! [`SemanticIndexBuilder`](crate::semantic_index::builder::SemanticIndexBuilder), e.g. where it //! visits a `StmtIf` node. -use std::collections::HashSet; - use self::symbol_state::{ BindingIdWithConstraintsIterator, ConstraintIdIterator, DeclarationIdIterator, ScopedConstraintId, ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState, @@ -230,6 +228,8 @@ use self::symbol_state::{ use crate::semantic_index::ast_ids::ScopedUseId; use crate::semantic_index::definition::Definition; use crate::semantic_index::symbol::ScopedSymbolId; +use crate::semantic_index::use_def::bitset::BitSet; +use crate::semantic_index::use_def::symbol_state::INLINE_CONSTRAINT_BLOCKS; use crate::symbol::Boundness; use ruff_index::IndexVec; use rustc_hash::FxHashMap; @@ -562,8 +562,10 @@ pub(super) struct FlowSnapshot { symbol_states: IndexVec, } +type ActiveConstraints = BitSet; + #[derive(Clone, Debug)] -pub(super) struct ActiveConstraintsSnapshot(HashSet); +pub(super) struct ActiveConstraintsSnapshot(ActiveConstraints); #[derive(Debug, Default)] pub(super) struct UseDefMapBuilder<'db> { @@ -573,7 +575,7 @@ pub(super) struct UseDefMapBuilder<'db> { /// Append-only array of [`Constraint`]. all_constraints: IndexVec>, - active_constraints: HashSet, + active_constraints: ActiveConstraints, /// Live bindings at each so-far-recorded use. bindings_by_use: IndexVec, @@ -606,7 +608,7 @@ impl<'db> UseDefMapBuilder<'db> { for state in &mut self.symbol_states { state.record_constraint(constraint_id); } - self.active_constraints.insert(constraint_id); + self.active_constraints.insert(constraint_id.as_u32()); } pub(super) fn record_declaration( diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 7871a52d16f93..55b3bf21a8769 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -43,7 +43,7 @@ //! //! Tracking live declarations is simpler, since constraints are not involved, but otherwise very //! similar to tracking live bindings. -use std::collections::HashSet; +use crate::semantic_index::use_def::ActiveConstraints; use super::bitset::{BitSet, BitSetIterator}; use ruff_index::newtype_index; @@ -72,7 +72,7 @@ type Declarations = BitSet; type DeclarationsIterator<'a> = BitSetIterator<'a, INLINE_DECLARATION_BLOCKS>; /// Can reference this * 64 total constraints inline; more will fall back to the heap. -const INLINE_CONSTRAINT_BLOCKS: usize = 2; +pub(crate) const INLINE_CONSTRAINT_BLOCKS: usize = 2; /// Can keep inline this many live bindings per symbol at a given time; more will go to heap. const INLINE_BINDINGS_PER_SYMBOL: usize = 4; @@ -108,7 +108,7 @@ impl SymbolDeclarations { fn record_declaration( &mut self, declaration_id: ScopedDefinitionId, - active_constraints: &HashSet, + active_constraints: &ActiveConstraints, ) { self.live_declarations = Declarations::with(declaration_id.into()); self.may_be_undeclared = false; @@ -117,8 +117,8 @@ impl SymbolDeclarations { self.constraints_active_at_declaration = Constraints::with_capacity(1); self.constraints_active_at_declaration .push(BitSet::default()); - for active_constraint_id in active_constraints { - self.constraints_active_at_declaration[0].insert(active_constraint_id.as_u32()); + for active_constraint_id in active_constraints.iter() { + self.constraints_active_at_declaration[0].insert(active_constraint_id); } } @@ -181,7 +181,7 @@ impl SymbolBindings { pub(super) fn record_binding( &mut self, binding_id: ScopedDefinitionId, - active_constraints: &HashSet, + active_constraints: &ActiveConstraints, ) { // The new binding replaces all previous live bindings in this path, and has no // constraints. @@ -190,8 +190,8 @@ impl SymbolBindings { self.constraints.push(BitSet::default()); self.constraints_active_at_binding = Constraints::with_capacity(1); self.constraints_active_at_binding.push(BitSet::default()); - for active_constraint_id in active_constraints { - self.constraints_active_at_binding[0].insert(active_constraint_id.as_u32()); + for active_constraint_id in active_constraints.iter() { + self.constraints_active_at_binding[0].insert(active_constraint_id); } self.may_be_unbound = false; } @@ -241,7 +241,7 @@ impl SymbolState { pub(super) fn record_binding( &mut self, binding_id: ScopedDefinitionId, - active_constraints: &HashSet, + active_constraints: &ActiveConstraints, ) { self.bindings.record_binding(binding_id, active_constraints); } @@ -260,7 +260,7 @@ impl SymbolState { pub(super) fn record_declaration( &mut self, declaration_id: ScopedDefinitionId, - active_constraints: &HashSet, + active_constraints: &ActiveConstraints, ) { self.declarations .record_declaration(declaration_id, active_constraints); From 8f781a94140e6f26d4992c32715b5a8a9068c8e5 Mon Sep 17 00:00:00 2001 From: David Peter Date: Fri, 6 Dec 2024 11:14:26 +0100 Subject: [PATCH 03/68] Explain version bounds in tests --- .../mdtest/annotations/literal_string.md | 5 ---- .../resources/mdtest/annotations/never.md | 27 ++++++++++++------- .../resources/mdtest/narrow/issubclass.md | 5 ++-- .../resources/mdtest/pep695_type_aliases.md | 2 ++ 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/annotations/literal_string.md b/crates/red_knot_python_semantic/resources/mdtest/annotations/literal_string.md index fe0f40dc9ec23..42d616b8410c3 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/annotations/literal_string.md +++ b/crates/red_knot_python_semantic/resources/mdtest/annotations/literal_string.md @@ -1,10 +1,5 @@ # `LiteralString` -```toml -[environment] -target-version = "3.11" -``` - `LiteralString` represents a string that is either defined directly within the source code or is made up of such components. diff --git a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md index 2204b14ee62fc..1cbab75e4a98a 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md +++ b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md @@ -1,10 +1,5 @@ # NoReturn & Never -```toml -[environment] -target-version = "3.11" -``` - `NoReturn` is used to annotate the return type for functions that never return. `Never` is the bottom type, representing the empty set of Python objects. These two annotations can be used interchangeably. @@ -52,7 +47,9 @@ def f(): ## `typing.Never` -`typing.Never` is only available in Python 3.11 and later: +`typing.Never` is only available in Python 3.11 and later. + +### Python 3.11 ```toml [environment] @@ -62,8 +59,20 @@ python-version = "3.11" ```py from typing import Never -x: Never +reveal_type(Never) # revealed: typing.Never +``` + +### Python 3.10 -def f(): - reveal_type(x) # revealed: Never +```toml +[environment] +target-version = "3.10" +``` + +```py +# TODO: should raise a diagnostic +from typing import Never + +# TODO: this should be Unknown +reveal_type(Never) # revealed: Never ``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md b/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md index 282d7b2698ccf..37980e6e54759 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md +++ b/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md @@ -95,14 +95,15 @@ def _(t: type[object]): ### Handling of `None` +`types.NoneType` is only available in Python 3.10 and later: + ```toml [environment] target-version = "3.10" ``` ```py -# TODO: this error should ideally go away once we (1) understand `sys.version_info` branches, -# and (2) set the target Python version for this test to 3.10. +# TODO: this error should ideally go away once we understand `sys.version_info` branches. # error: [possibly-unbound-import] "Member `NoneType` of module `types` is possibly unbound" from types import NoneType diff --git a/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md b/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md index 084eb3db0e80a..8e6372930f0d3 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md +++ b/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md @@ -7,6 +7,8 @@ PEP 695 type aliases are only available in Python 3.12 and later: python-version = "3.12" ``` +Type aliases are only available in Python 3.12 and later: + ```toml [environment] target-version = "3.12" From f7d10b66553cc1b6d66c9ad270013e2a5bd1f95d Mon Sep 17 00:00:00 2001 From: David Peter Date: Fri, 6 Dec 2024 21:07:30 +0100 Subject: [PATCH 04/68] Add comment --- .../src/semantic_index/use_def/symbol_state.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 55b3bf21a8769..bcf82f05d8d99 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -156,6 +156,7 @@ pub(super) struct SymbolBindings { /// binding in `live_bindings`. constraints: Constraints, + /// For each live binding, which [`ScopedConstraintId`] were active *at the time of the binding*? constraints_active_at_binding: Constraints, /// Could the symbol be unbound at this point? From 9d35a3436899ec69f29ebd9d112da0095d8de880 Mon Sep 17 00:00:00 2001 From: David Peter Date: Sun, 8 Dec 2024 22:19:46 +0100 Subject: [PATCH 05/68] Handle static True/False for boundness --- .../mdtest/assignment/annotations.md | 2 - .../resources/mdtest/boolean/short_circuit.md | 5 +- .../resources/mdtest/loops/async_for.md | 2 - .../resources/mdtest/loops/for.md | 3 - .../resources/mdtest/narrow/issubclass.md | 2 - .../mdtest/statically-known-branches.md | 109 ++++++++++++++++++ .../src/semantic_index/use_def.rs | 85 ++++++++++++-- .../semantic_index/use_def/symbol_state.rs | 2 +- crates/red_knot_python_semantic/src/types.rs | 22 +++- .../src/types/infer.rs | 19 +-- 10 files changed, 214 insertions(+), 37 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/assignment/annotations.md b/crates/red_knot_python_semantic/resources/mdtest/assignment/annotations.md index c3977ed46b6c4..f696cd4ea414f 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/assignment/annotations.md +++ b/crates/red_knot_python_semantic/resources/mdtest/assignment/annotations.md @@ -33,8 +33,6 @@ b: tuple[int] = (42,) c: tuple[str, int] = ("42", 42) d: tuple[tuple[str, str], tuple[int, int]] = (("foo", "foo"), (42, 42)) e: tuple[str, ...] = () -# TODO: we should not emit this error -# error: [call-possibly-unbound-method] "Method `__class_getitem__` of type `Literal[tuple]` is possibly unbound" f: tuple[str, *tuple[int, ...], bytes] = ("42", b"42") g: tuple[str, Unpack[tuple[int, ...]], bytes] = ("42", b"42") h: tuple[list[int], list[int]] = ([], []) diff --git a/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md b/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md index 9415b515dcf16..21246f81d209c 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md +++ b/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md @@ -32,13 +32,10 @@ def _(flag: bool): ```py if True or (x := 1): - # TODO: infer that the second arm is never executed, and raise `unresolved-reference`. - # error: [possibly-unresolved-reference] + # error: [unresolved-reference] reveal_type(x) # revealed: Never if True and (x := 1): - # TODO: infer that the second arm is always executed, do not raise a diagnostic - # error: [possibly-unresolved-reference] reveal_type(x) # revealed: Literal[1] ``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/loops/async_for.md b/crates/red_knot_python_semantic/resources/mdtest/loops/async_for.md index c0735768763e2..3e634ef4320c2 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/loops/async_for.md +++ b/crates/red_knot_python_semantic/resources/mdtest/loops/async_for.md @@ -19,7 +19,6 @@ async def foo(): # TODO: should reveal `Unknown` because `__aiter__` is not defined # revealed: @Todo(async iterables/iterators) - # error: [possibly-unresolved-reference] reveal_type(x) ``` @@ -39,7 +38,6 @@ async def foo(): async for x in IntAsyncIterable(): pass - # error: [possibly-unresolved-reference] # revealed: @Todo(async iterables/iterators) reveal_type(x) ``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/loops/for.md b/crates/red_knot_python_semantic/resources/mdtest/loops/for.md index 58675475abe72..5a6077360557d 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/loops/for.md +++ b/crates/red_knot_python_semantic/resources/mdtest/loops/for.md @@ -15,7 +15,6 @@ for x in IntIterable(): pass # revealed: int -# error: [possibly-unresolved-reference] reveal_type(x) ``` @@ -88,7 +87,6 @@ for x in OldStyleIterable(): pass # revealed: int -# error: [possibly-unresolved-reference] reveal_type(x) ``` @@ -99,7 +97,6 @@ for x in (1, "a", b"foo"): pass # revealed: Literal[1] | Literal["a"] | Literal[b"foo"] -# error: [possibly-unresolved-reference] reveal_type(x) ``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md b/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md index 37980e6e54759..1d5a7d4c4e3bc 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md +++ b/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md @@ -103,8 +103,6 @@ target-version = "3.10" ``` ```py -# TODO: this error should ideally go away once we understand `sys.version_info` branches. -# error: [possibly-unbound-import] "Member `NoneType` of module `types` is possibly unbound" from types import NoneType def _(flag: bool): diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index b7cc8d386d9ee..fbfcc0a6dfef6 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -292,3 +292,112 @@ class C: reveal_type(C.x) # revealed: int ``` + +## (Un)boundness + +### Unbound, `if False` + +```py +if False: + x = 1 + +# error: [unresolved-reference] +x +``` + +### Unbound, `if True … else` + +```py +if True: + pass +else: + x = 1 + +# error: [unresolved-reference] +x +``` + +### Bound, `if True` + +```py +if True: + x = 1 + +# x is always bound, no error +x +``` + +### Bound, `if False … else` + +```py +if False: + pass +else: + x = 1 + +# x is always bound, no error +x +``` + +### Nested + +```py +if False: + if True: + x = 1 + +if True: + if False: + y = 1 + +if False: + if False: + z = 1 + +# error: [unresolved-reference] +# error: [unresolved-reference] +# error: [unresolved-reference] +(x, y, z) +``` + +### Multiple nested conditions + +```py +if True: + if False: + x = 1 + if True: + x = 2 + +# x is always bound, no error +x + +if True: + if False: + y = 1 + if True: + y = 2 + +# y is always bound, no error +y + +if False: + if False: + z = 1 + if False: + z = 2 + +# error: [unresolved-reference] +z +``` + +### Public boundness + +```py +if True: + x = 1 + +def f(): + # x is always bound, no error + x +``` diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 33657fc644cdc..5268ae145dbf3 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -225,12 +225,15 @@ use self::symbol_state::{ BindingIdWithConstraintsIterator, ConstraintIdIterator, DeclarationIdIterator, ScopedConstraintId, ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState, }; -use crate::semantic_index::ast_ids::ScopedUseId; +use crate::db; +use crate::semantic_index::ast_ids::{HasScopedExpressionId, ScopedUseId}; +use crate::semantic_index::constraint::ConstraintNode; use crate::semantic_index::definition::Definition; use crate::semantic_index::symbol::ScopedSymbolId; use crate::semantic_index::use_def::bitset::BitSet; use crate::semantic_index::use_def::symbol_state::INLINE_CONSTRAINT_BLOCKS; use crate::symbol::Boundness; +use crate::types::{infer_expression_types, KnownClass}; use ruff_index::IndexVec; use rustc_hash::FxHashMap; @@ -381,14 +384,74 @@ impl<'db> UseDefMap<'db> { self.bindings_iterator(&self.bindings_by_use[use_id]) } - pub(crate) fn use_boundness(&self, use_id: ScopedUseId) -> Boundness { - if self.bindings_by_use[use_id].may_be_unbound() { - Boundness::PossiblyUnbound + fn compute_boundness( + &self, + db: &dyn crate::db::Db, + bindings: &SymbolBindings, + ) -> Option { + let bindings_iter = self.bindings_iterator(bindings); + + let mut definitely_bound = false; + let mut definitely_unbound = true; + for binding in bindings_iter { + let test_expr_tys = || { + binding.constraints_active_at_binding.clone().map(|c| { + let ty = if let ConstraintNode::Expression(test_expr) = c.node { + let inference = infer_expression_types(db, test_expr); + let scope = test_expr.scope(db); + inference + .expression_ty(test_expr.node_ref(db).scoped_expression_id(db, scope)) + } else { + // TODO: handle other constraint nodes + KnownClass::Bool.to_instance(db) + }; + + (c, ty) + }) + }; + + let is_any_always_false = test_expr_tys().any(|(c, test_expr_ty)| { + if c.is_positive { + test_expr_ty.bool(db).is_always_false() + } else { + test_expr_ty.bool(db).is_always_true() + } + }); + if !is_any_always_false { + definitely_unbound = false; + } + + let are_all_always_true = test_expr_tys().all(|(c, test_expr_ty)| { + if c.is_positive { + test_expr_ty.bool(db).is_always_true() + } else { + test_expr_ty.bool(db).is_always_false() + } + }); + if are_all_always_true { + definitely_bound = true; + } + } + + if definitely_unbound { + None } else { - Boundness::Bound + if definitely_bound || !bindings.may_be_unbound() { + Some(Boundness::Bound) + } else { + Some(Boundness::PossiblyUnbound) + } } } + pub(crate) fn use_boundness( + &self, + db: &dyn crate::db::Db, + use_id: ScopedUseId, + ) -> Option { + self.compute_boundness(db, &self.bindings_by_use[use_id]) + } + pub(crate) fn public_bindings( &self, symbol: ScopedSymbolId, @@ -396,12 +459,12 @@ impl<'db> UseDefMap<'db> { self.bindings_iterator(self.public_symbols[symbol].bindings()) } - pub(crate) fn public_boundness(&self, symbol: ScopedSymbolId) -> Boundness { - if self.public_symbols[symbol].may_be_unbound() { - Boundness::PossiblyUnbound - } else { - Boundness::Bound - } + pub(crate) fn public_boundness( + &self, + db: &dyn crate::db::Db, + symbol: ScopedSymbolId, + ) -> Option { + self.compute_boundness(db, &self.public_symbols[symbol].bindings()) } pub(crate) fn bindings_at_declaration( diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index bcf82f05d8d99..aef4de0610ca0 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -157,7 +157,7 @@ pub(super) struct SymbolBindings { constraints: Constraints, /// For each live binding, which [`ScopedConstraintId`] were active *at the time of the binding*? - constraints_active_at_binding: Constraints, + pub(crate) constraints_active_at_binding: Constraints, /// Could the symbol be unbound at this point? may_be_unbound: bool, diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 311ac4706ef15..3d985140dcdc2 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -78,7 +78,13 @@ fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolI let undeclared_ty = if declarations.may_be_undeclared() { Some( bindings_ty(db, use_def.public_bindings(symbol)) - .map(|bindings_ty| Symbol::Type(bindings_ty, use_def.public_boundness(symbol))) + .map(|bindings_ty| { + if let Some(boundness) = use_def.public_boundness(db, symbol) { + Symbol::Type(bindings_ty, boundness) + } else { + Symbol::Unbound + } + }) .unwrap_or(Symbol::Unbound), ) } else { @@ -114,7 +120,13 @@ fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolI } } else { bindings_ty(db, use_def.public_bindings(symbol)) - .map(|bindings_ty| Symbol::Type(bindings_ty, use_def.public_boundness(symbol))) + .map(|bindings_ty| { + if let Some(boundness) = use_def.public_boundness(db, symbol) { + Symbol::Type(bindings_ty, boundness) + } else { + Symbol::Unbound + } + }) .unwrap_or(Symbol::Unbound) } } @@ -1539,7 +1551,7 @@ impl<'db> Type<'db> { /// /// This is used to determine the value that would be returned /// when `bool(x)` is called on an object `x`. - fn bool(&self, db: &'db dyn Db) -> Truthiness { + pub(crate) fn bool(&self, db: &'db dyn Db) -> Truthiness { match self { Type::Any | Type::Todo(_) | Type::Never | Type::Unknown => Truthiness::Ambiguous, Type::FunctionLiteral(_) => Truthiness::AlwaysTrue, @@ -2624,11 +2636,11 @@ impl Truthiness { matches!(self, Truthiness::Ambiguous) } - const fn is_always_false(self) -> bool { + pub(crate) const fn is_always_false(self) -> bool { matches!(self, Truthiness::AlwaysFalse) } - const fn is_always_true(self) -> bool { + pub(crate) const fn is_always_true(self) -> bool { matches!(self, Truthiness::AlwaysTrue) } diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index 1b732ce5311b8..34064a94a6a9e 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -2995,24 +2995,23 @@ impl<'db> TypeInferenceBuilder<'db> { if let Some(symbol) = self.index.symbol_table(file_scope_id).symbol_id_by_name(id) { ( bindings_ty(self.db, use_def.public_bindings(symbol)), - use_def.public_boundness(symbol), + use_def.public_boundness(self.db, symbol), ) } else { assert!( self.deferred_state.in_string_annotation(), "Expected the symbol table to create a symbol for every Name node" ); - (None, Boundness::PossiblyUnbound) + (None, Some(Boundness::PossiblyUnbound)) } } else { let use_id = name.scoped_use_id(self.db, self.scope()); ( bindings_ty(self.db, use_def.bindings_at_use(use_id)), - use_def.use_boundness(use_id), + use_def.use_boundness(self.db, use_id), ) }; - - if boundness == Boundness::PossiblyUnbound { + if boundness == Some(Boundness::PossiblyUnbound) || boundness == None { match self.lookup_name(name) { Symbol::Type(looked_up_ty, looked_up_boundness) => { if looked_up_boundness == Boundness::PossiblyUnbound { @@ -3025,14 +3024,20 @@ impl<'db> TypeInferenceBuilder<'db> { } Symbol::Unbound => { if bindings_ty.is_some() { - self.diagnostics.add_possibly_unresolved_reference(name); + if boundness == Some(Boundness::PossiblyUnbound) { + self.diagnostics.add_possibly_unresolved_reference(name); + } else { + self.diagnostics.add_unresolved_reference(name); + } } else { self.diagnostics.add_unresolved_reference(name); } bindings_ty.unwrap_or(Type::Unknown) } } - } else { + } else + /*if boundness == Some(Boundness::Bound) */ + { bindings_ty.unwrap_or(Type::Unknown) } } From 392893406f5985e9357dac50b15fd8debc2ded65 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 09:25:28 +0100 Subject: [PATCH 06/68] Handle while loops --- .../resources/mdtest/expression/if.md | 2 +- .../mdtest/statically-known-branches.md | 61 ++++++++++++++++--- .../src/semantic_index/builder.rs | 2 + .../src/semantic_index/use_def.rs | 1 - .../semantic_index/use_def/symbol_state.rs | 1 + 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/expression/if.md b/crates/red_knot_python_semantic/resources/mdtest/expression/if.md index 79faa45426855..c39991e0359e0 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/expression/if.md +++ b/crates/red_knot_python_semantic/resources/mdtest/expression/if.md @@ -7,7 +7,7 @@ def _(flag: bool): reveal_type(1 if flag else 2) # revealed: Literal[1, 2] ``` -## Statically known branches +## Statically known conditions in if expressions ```py reveal_type(1 if True else 2) # revealed: Literal[1] diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index fbfcc0a6dfef6..4069b502737d3 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -1,8 +1,10 @@ # Statically-known branches -## Always false +## If statements -### If +### Always false + +#### If ```py x = 1 @@ -13,7 +15,7 @@ if False: reveal_type(x) # revealed: Literal[1] ``` -### Else +#### Else ```py x = 1 @@ -26,9 +28,9 @@ else: reveal_type(x) # revealed: Literal[1] ``` -## Always true +### Always true -### If +#### If ```py x = 1 @@ -39,7 +41,7 @@ if True: reveal_type(x) # revealed: Literal[2] ``` -### Else +#### Else ```py x = 1 @@ -52,7 +54,7 @@ else: reveal_type(x) # revealed: Literal[2] ``` -## Combination +### Combination of always true and always false ```py x = 1 @@ -65,7 +67,7 @@ else: reveal_type(x) # revealed: Literal[2] ``` -## Nested +### Nested conditionals ```py path=nested_if_true_if_true.py x = 1 @@ -171,7 +173,9 @@ else: reveal_type(x) # revealed: Literal[3, 4] ``` -## If-expressions +## If expressions + +See also: tests in [expression/if.md](expression/if.md). ### Always true @@ -207,6 +211,45 @@ reveal_type(x) # revealed: Literal[1] reveal_type(x) # revealed: Literal[2] ``` +## While loops + +### Always false + +```py +x = 1 + +while False: + x = 2 + +reveal_type(x) # revealed: Literal[1] +``` + +### Always true + +```py +x = 1 + +while True: + x = 2 + break + +reveal_type(x) # revealed: Literal[2] +``` + +### Ambiguous + +```py +def flag() -> bool: ... + +x = 1 + +while flag(): + x = 2 + break + +reveal_type(x) # revealed: Literal[1, 2] +``` + ## Conditional declarations ```py path=if_false.py diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index 3686be4a0c532..16acfede522eb 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -845,6 +845,8 @@ where let pre_loop_constraints = self.constraints_snapshot(); let constraint = self.record_expression_constraint(test); + self.record_expression_constraint(test); + // Save aside any break states from an outer loop let saved_break_states = std::mem::take(&mut self.loop_break_states); diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 5268ae145dbf3..7e192e8a0e278 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -225,7 +225,6 @@ use self::symbol_state::{ BindingIdWithConstraintsIterator, ConstraintIdIterator, DeclarationIdIterator, ScopedConstraintId, ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState, }; -use crate::db; use crate::semantic_index::ast_ids::{HasScopedExpressionId, ScopedUseId}; use crate::semantic_index::constraint::ConstraintNode; use crate::semantic_index::definition::Definition; diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index aef4de0610ca0..0b7a080660c6e 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -485,6 +485,7 @@ impl SymbolState { } /// Could the symbol be unbound? + #[cfg(test)] pub(super) fn may_be_unbound(&self) -> bool { self.bindings.may_be_unbound() } From acf70540b84fcbb90fba6075dd6f805431740240 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 10:09:21 +0100 Subject: [PATCH 07/68] Rename to branching-condition --- .../mdtest/statically-known-branches.md | 36 +++++ .../src/semantic_index.rs | 1 + .../src/semantic_index/branching.rs | 6 + .../src/semantic_index/builder.rs | 20 ++- .../src/semantic_index/use_def.rs | 37 ++--- .../semantic_index/use_def/symbol_state.rs | 148 ++++++++---------- crates/red_knot_python_semantic/src/types.rs | 8 +- 7 files changed, 146 insertions(+), 110 deletions(-) create mode 100644 crates/red_knot_python_semantic/src/semantic_index/branching.rs diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 4069b502737d3..17e9d7c971fb1 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -173,6 +173,42 @@ else: reveal_type(x) # revealed: Literal[3, 4] ``` +### Combination with non-conditional control flow + +```py path=try_if_true.py +def may_raise() -> None: ... + +x = 1 + +try: + may_raise() + if True: + x = 2 + else: + x = 3 +except: + x = 4 + +reveal_type(x) # revealed: Literal[2, 4] +``` + +```py path=if_true_try.py +def may_raise() -> None: ... + +x = 1 + +if True: + try: + may_raise() + x = 2 + except: + x = 3 +else: + x = 4 + +reveal_type(x) # revealed: Literal[2, 3] +``` + ## If expressions See also: tests in [expression/if.md](expression/if.md). diff --git a/crates/red_knot_python_semantic/src/semantic_index.rs b/crates/red_knot_python_semantic/src/semantic_index.rs index fb681a39ee715..2d870e31f7971 100644 --- a/crates/red_knot_python_semantic/src/semantic_index.rs +++ b/crates/red_knot_python_semantic/src/semantic_index.rs @@ -20,6 +20,7 @@ use crate::semantic_index::use_def::UseDefMap; use crate::Db; pub mod ast_ids; +pub(crate) mod branching; mod builder; pub(crate) mod constraint; pub mod definition; diff --git a/crates/red_knot_python_semantic/src/semantic_index/branching.rs b/crates/red_knot_python_semantic/src/semantic_index/branching.rs new file mode 100644 index 0000000000000..e6ef93c76ac90 --- /dev/null +++ b/crates/red_knot_python_semantic/src/semantic_index/branching.rs @@ -0,0 +1,6 @@ +use super::constraint::Constraint; + +pub(crate) enum BranchingCondition<'db> { + Conditional(Constraint<'db>), + Unconditional, +} diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index 16acfede522eb..ef15b9fcd5eff 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -23,7 +23,7 @@ use crate::semantic_index::symbol::{ FileScopeId, NodeWithScopeKey, NodeWithScopeRef, Scope, ScopeId, ScopedSymbolId, SymbolTableBuilder, }; -use crate::semantic_index::use_def::{ActiveConstraintsSnapshot, FlowSnapshot, UseDefMapBuilder}; +use crate::semantic_index::use_def::{BranchingConditionsSnapshot, FlowSnapshot, UseDefMapBuilder}; use crate::semantic_index::SemanticIndex; use crate::unpack::Unpack; use crate::Db; @@ -200,20 +200,28 @@ impl<'db> SemanticIndexBuilder<'db> { self.current_use_def_map().snapshot() } - fn constraints_snapshot(&self) -> ActiveConstraintsSnapshot { + fn constraints_snapshot(&self) -> BranchingConditionsSnapshot { self.current_use_def_map().constraints_snapshot() } - fn flow_restore(&mut self, state: FlowSnapshot, active_constraints: ActiveConstraintsSnapshot) { + fn flow_restore( + &mut self, + state: FlowSnapshot, + branching_conditions: BranchingConditionsSnapshot, + ) { self.current_use_def_map_mut().restore(state); self.current_use_def_map_mut() - .restore_constraints(active_constraints); + .restore_constraints(branching_conditions); } - fn flow_merge(&mut self, state: FlowSnapshot, active_constraints: ActiveConstraintsSnapshot) { + fn flow_merge( + &mut self, + state: FlowSnapshot, + branching_conditions: BranchingConditionsSnapshot, + ) { self.current_use_def_map_mut().merge(state); self.current_use_def_map_mut() - .restore_constraints(active_constraints); + .restore_constraints(branching_conditions); } fn add_symbol(&mut self, name: Name) -> ScopedSymbolId { diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 7e192e8a0e278..7aebd7668d5a0 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -230,7 +230,7 @@ use crate::semantic_index::constraint::ConstraintNode; use crate::semantic_index::definition::Definition; use crate::semantic_index::symbol::ScopedSymbolId; use crate::semantic_index::use_def::bitset::BitSet; -use crate::semantic_index::use_def::symbol_state::INLINE_CONSTRAINT_BLOCKS; +use crate::semantic_index::use_def::symbol_state::BranchingConditions; use crate::symbol::Boundness; use crate::types::{infer_expression_types, KnownClass}; use ruff_index::IndexVec; @@ -312,8 +312,7 @@ impl<'db> UseDefMap<'db> { for binding in bindings.iter() { let definition = self.all_definitions[binding.definition]; let mut constraint_ids = binding.constraint_ids.peekable(); - let mut active_constraint_ids = - binding.constraints_active_at_binding_ids.peekable(); + let mut active_constraint_ids = binding.branching_conditions_ids.peekable(); println!(" * {definition:?}"); @@ -394,7 +393,7 @@ impl<'db> UseDefMap<'db> { let mut definitely_unbound = true; for binding in bindings_iter { let test_expr_tys = || { - binding.constraints_active_at_binding.clone().map(|c| { + binding.branching_conditions.clone().map(|c| { let ty = if let ConstraintNode::Expression(test_expr) = c.node { let inference = infer_expression_types(db, test_expr); let scope = test_expr.scope(db); @@ -553,9 +552,9 @@ impl<'map, 'db> Iterator for BindingWithConstraintsIterator<'map, 'db> { all_constraints: self.all_constraints, constraint_ids: def_id_with_constraints.constraint_ids, }, - constraints_active_at_binding: ConstraintsIterator { + branching_conditions: ConstraintsIterator { all_constraints: self.all_constraints, - constraint_ids: def_id_with_constraints.constraints_active_at_binding_ids, + constraint_ids: def_id_with_constraints.branching_conditions_ids, }, }) } @@ -566,7 +565,7 @@ impl std::iter::FusedIterator for BindingWithConstraintsIterator<'_, '_> {} pub(crate) struct BindingWithConstraints<'map, 'db> { pub(crate) binding: Definition<'db>, pub(crate) constraints: ConstraintsIterator<'map, 'db>, - pub(crate) constraints_active_at_binding: ConstraintsIterator<'map, 'db>, + pub(crate) branching_conditions: ConstraintsIterator<'map, 'db>, } #[derive(Debug, Clone)] @@ -624,10 +623,8 @@ pub(super) struct FlowSnapshot { symbol_states: IndexVec, } -type ActiveConstraints = BitSet; - #[derive(Clone, Debug)] -pub(super) struct ActiveConstraintsSnapshot(ActiveConstraints); +pub(super) struct BranchingConditionsSnapshot(BranchingConditions); #[derive(Debug, Default)] pub(super) struct UseDefMapBuilder<'db> { @@ -637,7 +634,7 @@ pub(super) struct UseDefMapBuilder<'db> { /// Append-only array of [`Constraint`]. all_constraints: IndexVec>, - active_constraints: ActiveConstraints, + branching_conditions: BranchingConditions, /// Live bindings at each so-far-recorded use. bindings_by_use: IndexVec, @@ -662,7 +659,7 @@ impl<'db> UseDefMapBuilder<'db> { binding, SymbolDefinitions::Declarations(symbol_state.declarations().clone()), ); - symbol_state.record_binding(def_id, &self.active_constraints); + symbol_state.record_binding(def_id, &self.branching_conditions); } pub(super) fn record_constraint(&mut self, constraint: Constraint<'db>) { @@ -670,7 +667,7 @@ impl<'db> UseDefMapBuilder<'db> { for state in &mut self.symbol_states { state.record_constraint(constraint_id); } - self.active_constraints.insert(constraint_id.as_u32()); + self.branching_conditions.insert(constraint_id.as_u32()); } pub(super) fn record_declaration( @@ -684,7 +681,7 @@ impl<'db> UseDefMapBuilder<'db> { declaration, SymbolDefinitions::Bindings(symbol_state.bindings().clone()), ); - symbol_state.record_declaration(def_id, &self.active_constraints); + symbol_state.record_declaration(def_id, &self.branching_conditions); } pub(super) fn record_declaration_and_binding( @@ -695,8 +692,8 @@ impl<'db> UseDefMapBuilder<'db> { // We don't need to store anything in self.definitions_by_definition. let def_id = self.all_definitions.push(definition); let symbol_state = &mut self.symbol_states[symbol]; - symbol_state.record_declaration(def_id, &self.active_constraints); - symbol_state.record_binding(def_id, &self.active_constraints); + symbol_state.record_declaration(def_id, &self.branching_conditions); + symbol_state.record_binding(def_id, &self.branching_conditions); } pub(super) fn record_use(&mut self, symbol: ScopedSymbolId, use_id: ScopedUseId) { @@ -715,8 +712,8 @@ impl<'db> UseDefMapBuilder<'db> { } } - pub(super) fn constraints_snapshot(&self) -> ActiveConstraintsSnapshot { - ActiveConstraintsSnapshot(self.active_constraints.clone()) + pub(super) fn constraints_snapshot(&self) -> BranchingConditionsSnapshot { + BranchingConditionsSnapshot(self.branching_conditions.clone()) } /// Restore the current builder symbols state to the given snapshot. @@ -737,8 +734,8 @@ impl<'db> UseDefMapBuilder<'db> { .resize(num_symbols, SymbolState::undefined()); } - pub(super) fn restore_constraints(&mut self, snapshot: ActiveConstraintsSnapshot) { - self.active_constraints = snapshot.0; + pub(super) fn restore_constraints(&mut self, snapshot: BranchingConditionsSnapshot) { + self.branching_conditions = snapshot.0; } /// Merge the given snapshot into the current state, reflecting that we might have taken either diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 0b7a080660c6e..3bc3a4bddb305 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -43,8 +43,6 @@ //! //! Tracking live declarations is simpler, since constraints are not involved, but otherwise very //! similar to tracking live bindings. -use crate::semantic_index::use_def::ActiveConstraints; - use super::bitset::{BitSet, BitSetIterator}; use ruff_index::newtype_index; use smallvec::SmallVec; @@ -72,7 +70,7 @@ type Declarations = BitSet; type DeclarationsIterator<'a> = BitSetIterator<'a, INLINE_DECLARATION_BLOCKS>; /// Can reference this * 64 total constraints inline; more will fall back to the heap. -pub(crate) const INLINE_CONSTRAINT_BLOCKS: usize = 2; +const INLINE_CONSTRAINT_BLOCKS: usize = 2; /// Can keep inline this many live bindings per symbol at a given time; more will go to heap. const INLINE_BINDINGS_PER_SYMBOL: usize = 4; @@ -83,13 +81,16 @@ type Constraints = SmallVec; type ConstraintsIterator<'a> = std::slice::Iter<'a, BitSet>; type ConstraintsIntoIterator = smallvec::IntoIter; +const INLINE_BRANCHING_CONDITIONS: usize = 2; +pub(super) type BranchingConditions = BitSet; + /// Live declarations for a single symbol at some point in control flow. #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct SymbolDeclarations { /// [`BitSet`]: which declarations (as [`ScopedDefinitionId`]) can reach the current location? live_declarations: Declarations, - constraints_active_at_declaration: Constraints, // TODO: rename to constraints_active_at_declaration + branching_conditions: Constraints, /// Could the symbol be un-declared at this point? may_be_undeclared: bool, @@ -99,7 +100,7 @@ impl SymbolDeclarations { fn undeclared() -> Self { Self { live_declarations: Declarations::default(), - constraints_active_at_declaration: Constraints::default(), + branching_conditions: Constraints::default(), may_be_undeclared: true, } } @@ -108,17 +109,16 @@ impl SymbolDeclarations { fn record_declaration( &mut self, declaration_id: ScopedDefinitionId, - active_constraints: &ActiveConstraints, + branching_conditions: &BranchingConditions, ) { self.live_declarations = Declarations::with(declaration_id.into()); self.may_be_undeclared = false; // TODO: unify code with below - self.constraints_active_at_declaration = Constraints::with_capacity(1); - self.constraints_active_at_declaration - .push(BitSet::default()); - for active_constraint_id in active_constraints.iter() { - self.constraints_active_at_declaration[0].insert(active_constraint_id); + self.branching_conditions = Constraints::with_capacity(1); + self.branching_conditions.push(BitSet::default()); + for active_constraint_id in branching_conditions.iter() { + self.branching_conditions[0].insert(active_constraint_id); } } @@ -131,7 +131,7 @@ impl SymbolDeclarations { pub(super) fn iter(&self) -> DeclarationIdIterator { DeclarationIdIterator { inner: self.live_declarations.iter(), - constraints_active_at_binding: self.constraints_active_at_declaration.iter(), + branching_conditions: self.branching_conditions.iter(), } } @@ -156,8 +156,8 @@ pub(super) struct SymbolBindings { /// binding in `live_bindings`. constraints: Constraints, - /// For each live binding, which [`ScopedConstraintId`] were active *at the time of the binding*? - pub(crate) constraints_active_at_binding: Constraints, + /// For each live binding, which [`BranchingCondition`]s were active *at the time of the binding*? + pub(crate) branching_conditions: Constraints, /// Could the symbol be unbound at this point? may_be_unbound: bool, @@ -168,7 +168,7 @@ impl SymbolBindings { Self { live_bindings: Bindings::default(), constraints: Constraints::default(), - constraints_active_at_binding: Constraints::default(), + branching_conditions: Constraints::default(), may_be_unbound: true, } } @@ -182,17 +182,17 @@ impl SymbolBindings { pub(super) fn record_binding( &mut self, binding_id: ScopedDefinitionId, - active_constraints: &ActiveConstraints, + branching_conditions: &BranchingConditions, ) { // The new binding replaces all previous live bindings in this path, and has no // constraints. self.live_bindings = Bindings::with(binding_id.into()); self.constraints = Constraints::with_capacity(1); self.constraints.push(BitSet::default()); - self.constraints_active_at_binding = Constraints::with_capacity(1); - self.constraints_active_at_binding.push(BitSet::default()); - for active_constraint_id in active_constraints.iter() { - self.constraints_active_at_binding[0].insert(active_constraint_id); + self.branching_conditions = Constraints::with_capacity(1); + self.branching_conditions.push(BitSet::default()); + for id in branching_conditions.iter() { + self.branching_conditions[0].insert(id); } self.may_be_unbound = false; } @@ -209,7 +209,7 @@ impl SymbolBindings { BindingIdWithConstraintsIterator { definitions: self.live_bindings.iter(), constraints: self.constraints.iter(), - constraints_active_at_binding: self.constraints_active_at_binding.iter(), + branching_conditions: self.branching_conditions.iter(), } } @@ -242,9 +242,10 @@ impl SymbolState { pub(super) fn record_binding( &mut self, binding_id: ScopedDefinitionId, - active_constraints: &ActiveConstraints, + branching_conditions: &BranchingConditions, ) { - self.bindings.record_binding(binding_id, active_constraints); + self.bindings + .record_binding(binding_id, branching_conditions); } /// Add given constraint to all live bindings. @@ -261,10 +262,10 @@ impl SymbolState { pub(super) fn record_declaration( &mut self, declaration_id: ScopedDefinitionId, - active_constraints: &ActiveConstraints, + branching_conditions: &BranchingConditions, ) { self.declarations - .record_declaration(declaration_id, active_constraints); + .record_declaration(declaration_id, branching_conditions); } /// Merge another [`SymbolState`] into this one. @@ -273,79 +274,71 @@ impl SymbolState { bindings: SymbolBindings { live_bindings: Bindings::default(), constraints: Constraints::default(), - constraints_active_at_binding: Constraints::default(), // TODO + branching_conditions: Constraints::default(), // TODO may_be_unbound: self.bindings.may_be_unbound || b.bindings.may_be_unbound, }, declarations: SymbolDeclarations { live_declarations: self.declarations.live_declarations.clone(), - constraints_active_at_declaration: Constraints::default(), // TODO + branching_conditions: Constraints::default(), // TODO may_be_undeclared: self.declarations.may_be_undeclared || b.declarations.may_be_undeclared, }, }; - // let mut constraints_active_at_binding = BitSet::default(); - // for active_constraint_id in active_constraints.0 { - // constraints_active_at_binding.insert(active_constraint_id.as_u32()); - // } - std::mem::swap(&mut a, self); - // self.declarations - // .live_declarations - // .union(&b.declarations.live_declarations); let mut a_decls_iter = a.declarations.live_declarations.iter(); let mut b_decls_iter = b.declarations.live_declarations.iter(); - let mut a_constraints_active_at_declaration_iter = - a.declarations.constraints_active_at_declaration.into_iter(); - let mut b_constraints_active_at_declaration_iter = - b.declarations.constraints_active_at_declaration.into_iter(); + let mut a_declaration_branching_conditions_iter = + a.declarations.branching_conditions.into_iter(); + let mut b_cdeclaration_branching_conditions_iter = + b.declarations.branching_conditions.into_iter(); let mut opt_a_decl: Option = a_decls_iter.next(); let mut opt_b_decl: Option = b_decls_iter.next(); let push = |decl, - constraints_active_at_declaration_iter: &mut ConstraintsIntoIterator, + declaration_branching_conditions_iter: &mut ConstraintsIntoIterator, merged: &mut Self| { merged.declarations.live_declarations.insert(decl); - let constraints_active_at_binding = constraints_active_at_declaration_iter + let branching_conditions = declaration_branching_conditions_iter .next() - .expect("declarations and constraints_active_at_binding length mismatch"); + .expect("declarations and branching_conditions length mismatch"); merged .declarations - .constraints_active_at_declaration - .push(constraints_active_at_binding); + .branching_conditions + .push(branching_conditions); }; loop { match (opt_a_decl, opt_b_decl) { (Some(a_decl), Some(b_decl)) => match a_decl.cmp(&b_decl) { std::cmp::Ordering::Less => { - push(a_decl, &mut a_constraints_active_at_declaration_iter, self); + push(a_decl, &mut a_declaration_branching_conditions_iter, self); opt_a_decl = a_decls_iter.next(); } std::cmp::Ordering::Greater => { - push(b_decl, &mut b_constraints_active_at_declaration_iter, self); + push(b_decl, &mut b_cdeclaration_branching_conditions_iter, self); opt_b_decl = b_decls_iter.next(); } std::cmp::Ordering::Equal => { - push(a_decl, &mut b_constraints_active_at_declaration_iter, self); + push(a_decl, &mut b_cdeclaration_branching_conditions_iter, self); self.declarations - .constraints_active_at_declaration + .branching_conditions .last_mut() .unwrap() - .intersect(&a_constraints_active_at_declaration_iter.next().unwrap()); + .intersect(&a_declaration_branching_conditions_iter.next().unwrap()); opt_a_decl = a_decls_iter.next(); opt_b_decl = b_decls_iter.next(); } }, (Some(a_decl), None) => { - push(a_decl, &mut a_constraints_active_at_declaration_iter, self); + push(a_decl, &mut a_declaration_branching_conditions_iter, self); opt_a_decl = a_decls_iter.next(); } (None, Some(b_decl)) => { - push(b_decl, &mut b_constraints_active_at_declaration_iter, self); + push(b_decl, &mut b_cdeclaration_branching_conditions_iter, self); opt_b_decl = b_decls_iter.next(); } (None, None) => break, @@ -356,10 +349,8 @@ impl SymbolState { let mut b_defs_iter = b.bindings.live_bindings.iter(); let mut a_constraints_iter = a.bindings.constraints.into_iter(); let mut b_constraints_iter = b.bindings.constraints.into_iter(); - let mut a_constraints_active_at_binding_iter = - a.bindings.constraints_active_at_binding.into_iter(); - let mut b_constraints_active_at_binding_iter = - b.bindings.constraints_active_at_binding.into_iter(); + let mut a_binding_branching_conditions_iter = a.bindings.branching_conditions.into_iter(); + let mut b_binding_branching_conditions_iter = b.bindings.branching_conditions.into_iter(); let mut opt_a_def: Option = a_defs_iter.next(); let mut opt_b_def: Option = b_defs_iter.next(); @@ -373,7 +364,7 @@ impl SymbolState { // Helper to push `def`, with constraints in `constraints_iter`, onto `self`. let push = |def, constraints_iter: &mut ConstraintsIntoIterator, - constraints_active_at_binding_iter: &mut ConstraintsIntoIterator, + branching_conditions_iter: &mut ConstraintsIntoIterator, merged: &mut Self| { merged.bindings.live_bindings.insert(def); // SAFETY: we only ever create SymbolState with either no definitions and no constraint @@ -384,14 +375,14 @@ impl SymbolState { let constraints = constraints_iter .next() .expect("definitions and constraints length mismatch"); - let constraints_active_at_binding = constraints_active_at_binding_iter + let branching_conditions = branching_conditions_iter .next() - .expect("definitions and constraints_active_at_binding length mismatch"); + .expect("definitions and branching_conditions length mismatch"); merged.bindings.constraints.push(constraints); merged .bindings - .constraints_active_at_binding - .push(constraints_active_at_binding); + .branching_conditions + .push(branching_conditions); }; loop { @@ -402,7 +393,7 @@ impl SymbolState { push( a_def, &mut a_constraints_iter, - &mut a_constraints_active_at_binding_iter, + &mut a_binding_branching_conditions_iter, self, ); opt_a_def = a_defs_iter.next(); @@ -412,7 +403,7 @@ impl SymbolState { push( b_def, &mut b_constraints_iter, - &mut b_constraints_active_at_binding_iter, + &mut b_binding_branching_conditions_iter, self, ); opt_b_def = b_defs_iter.next(); @@ -422,7 +413,7 @@ impl SymbolState { push( a_def, &mut b_constraints_iter, - &mut b_constraints_active_at_binding_iter, + &mut b_binding_branching_conditions_iter, self, ); // SAFETY: we only ever create SymbolState with either no definitions and @@ -433,10 +424,7 @@ impl SymbolState { let a_constraints = a_constraints_iter .next() .expect("definitions and constraints length mismatch"); - // let _a_constraints_active_at_binding = - // a_constraints_active_at_binding_iter.next().expect( - // "definitions and constraints_active_at_binding length mismatch", - // ); // TODO: perform check that we see the same constraints in both paths + // TODO: perform check that we see the same branching_conditions in both paths? // If the same definition is visible through both paths, any constraint // that applies on only one path is irrelevant to the resulting type from @@ -456,7 +444,7 @@ impl SymbolState { push( a_def, &mut a_constraints_iter, - &mut a_constraints_active_at_binding_iter, + &mut a_binding_branching_conditions_iter, self, ); opt_a_def = a_defs_iter.next(); @@ -466,7 +454,7 @@ impl SymbolState { push( b_def, &mut b_constraints_iter, - &mut b_constraints_active_at_binding_iter, + &mut b_binding_branching_conditions_iter, self, ); opt_b_def = b_defs_iter.next(); @@ -505,14 +493,14 @@ impl Default for SymbolState { pub(super) struct BindingIdWithConstraints<'a> { pub(super) definition: ScopedDefinitionId, pub(super) constraint_ids: ConstraintIdIterator<'a>, - pub(super) constraints_active_at_binding_ids: ConstraintIdIterator<'a>, + pub(super) branching_conditions_ids: ConstraintIdIterator<'a>, } #[derive(Debug, Clone)] pub(super) struct BindingIdWithConstraintsIterator<'a> { definitions: BindingsIterator<'a>, constraints: ConstraintsIterator<'a>, - constraints_active_at_binding: ConstraintsIterator<'a>, + branching_conditions: ConstraintsIterator<'a>, } impl<'a> Iterator for BindingIdWithConstraintsIterator<'a> { @@ -522,17 +510,17 @@ impl<'a> Iterator for BindingIdWithConstraintsIterator<'a> { match ( self.definitions.next(), self.constraints.next(), - self.constraints_active_at_binding.next(), + self.branching_conditions.next(), ) { (None, None, None) => None, - (Some(def), Some(constraints), Some(constraints_active_at_binding)) => { + (Some(def), Some(constraints), Some(branching_conditions)) => { Some(BindingIdWithConstraints { definition: ScopedDefinitionId::from_u32(def), constraint_ids: ConstraintIdIterator { wrapped: constraints.iter(), }, - constraints_active_at_binding_ids: ConstraintIdIterator { - wrapped: constraints_active_at_binding.iter(), + branching_conditions_ids: ConstraintIdIterator { + wrapped: branching_conditions.iter(), }, }) } @@ -562,7 +550,7 @@ impl std::iter::FusedIterator for ConstraintIdIterator<'_> {} #[derive(Debug)] pub(super) struct DeclarationIdIterator<'a> { inner: DeclarationsIterator<'a>, - constraints_active_at_binding: ConstraintsIterator<'a>, + branching_conditions: ConstraintsIterator<'a>, } impl<'a> Iterator for DeclarationIdIterator<'a> { @@ -570,16 +558,16 @@ impl<'a> Iterator for DeclarationIdIterator<'a> { fn next(&mut self) -> Option { // self.inner.next().map(ScopedDefinitionId::from_u32) - match (self.inner.next(), self.constraints_active_at_binding.next()) { + match (self.inner.next(), self.branching_conditions.next()) { (None, None) => None, - (Some(declaration), Some(constraints_active_at_binding)) => Some(( + (Some(declaration), Some(branching_conditions)) => Some(( ScopedDefinitionId::from_u32(declaration), ConstraintIdIterator { - wrapped: constraints_active_at_binding.iter(), + wrapped: branching_conditions.iter(), }, )), // SAFETY: see above. - _ => unreachable!("declarations and constraints_active_at_binding length mismatch"), + _ => unreachable!("declarations and branching_conditions length mismatch"), } } } diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 3d985140dcdc2..95755e90f2d2b 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -266,10 +266,10 @@ fn bindings_ty<'db>( |BindingWithConstraints { binding, constraints, - constraints_active_at_binding, + branching_conditions, }| { let test_expr_tys = || { - constraints_active_at_binding.clone().map(|c| { + branching_conditions.clone().map(|c| { let ty = if let ConstraintNode::Expression(test_expr) = c.node { let inference = infer_expression_types(db, test_expr); let scope = test_expr.scope(db); @@ -379,9 +379,9 @@ fn declarations_ty<'db>( declarations: DeclarationsIterator<'_, 'db>, undeclared_ty: Option>, ) -> DeclaredTypeResult<'db> { - let decl_types = declarations.map(|(declaration, constraints_active_at_declaration)| { + let decl_types = declarations.map(|(declaration, branching_conditions)| { let test_expr_tys = || { - constraints_active_at_declaration.clone().map(|c| { + branching_conditions.clone().map(|c| { let ty = if let ConstraintNode::Expression(test_expr) = c.node { let inference = infer_expression_types(db, test_expr); let scope = test_expr.scope(db); From d6b8acff55ea65f150e9b8a6cf99515b6bd63af0 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 11:03:11 +0100 Subject: [PATCH 08/68] Handle unconditional branching --- .../src/semantic_index.rs | 28 -- .../src/semantic_index/branching.rs | 1 + .../src/semantic_index/builder.rs | 43 +-- .../src/semantic_index/use_def.rs | 273 ++++++++---------- .../semantic_index/use_def/symbol_state.rs | 39 ++- crates/red_knot_python_semantic/src/types.rs | 54 ++-- 6 files changed, 188 insertions(+), 250 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index.rs b/crates/red_knot_python_semantic/src/semantic_index.rs index 2d870e31f7971..800feaa1d3860 100644 --- a/crates/red_knot_python_semantic/src/semantic_index.rs +++ b/crates/red_knot_python_semantic/src/semantic_index.rs @@ -1252,32 +1252,4 @@ match 1: assert!(matches!(binding.kind(&db), DefinitionKind::For(_))); } - - #[test] - #[ignore] - fn if_statement() { - let TestCase { db, file } = test_case( - " -x = False - -if True: - x: bool -", - ); - - let index = semantic_index(&db, file); - // let global_table = index.symbol_table(FileScopeId::global()); - - let use_def = index.use_def_map(FileScopeId::global()); - - // use_def - - use_def.print(&db); - - panic!(); - // let binding = use_def - // .first_public_binding(global_table.symbol_id_by_name(name).expect("symbol exists")) - // .expect("Expected with item definition for {name}"); - // assert!(matches!(binding.kind(&db), DefinitionKind::WithItem(_))); - } } diff --git a/crates/red_knot_python_semantic/src/semantic_index/branching.rs b/crates/red_knot_python_semantic/src/semantic_index/branching.rs index e6ef93c76ac90..58e70a49b0138 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/branching.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/branching.rs @@ -1,5 +1,6 @@ use super::constraint::Constraint; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum BranchingCondition<'db> { Conditional(Constraint<'db>), Unconditional, diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index ef15b9fcd5eff..5476aa25e6ee3 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -200,8 +200,8 @@ impl<'db> SemanticIndexBuilder<'db> { self.current_use_def_map().snapshot() } - fn constraints_snapshot(&self) -> BranchingConditionsSnapshot { - self.current_use_def_map().constraints_snapshot() + fn branching_conditions_snapshot(&self) -> BranchingConditionsSnapshot { + self.current_use_def_map().branching_conditions_snapshot() } fn flow_restore( @@ -211,7 +211,7 @@ impl<'db> SemanticIndexBuilder<'db> { ) { self.current_use_def_map_mut().restore(state); self.current_use_def_map_mut() - .restore_constraints(branching_conditions); + .restore_branching_conditions(branching_conditions); } fn flow_merge( @@ -221,7 +221,7 @@ impl<'db> SemanticIndexBuilder<'db> { ) { self.current_use_def_map_mut().merge(state); self.current_use_def_map_mut() - .restore_constraints(branching_conditions); + .restore_branching_conditions(branching_conditions); } fn add_symbol(&mut self, name: Name) -> ScopedSymbolId { @@ -301,6 +301,11 @@ impl<'db> SemanticIndexBuilder<'db> { self.current_use_def_map_mut().record_constraint(constraint); } + fn record_unconditional_branching(&mut self) { + self.current_use_def_map_mut() + .record_unconditional_branching(); + } + fn build_constraint(&mut self, constraint_node: &Expr) -> Constraint<'db> { let expression = self.add_standalone_expression(constraint_node); Constraint { @@ -801,7 +806,7 @@ where ast::Stmt::If(node) => { self.visit_expr(&node.test); let pre_if = self.flow_snapshot(); - let pre_if_constraints = self.constraints_snapshot(); + let pre_if_constraints = self.branching_conditions_snapshot(); let constraint = self.record_expression_constraint(&node.test); let mut constraints = vec![constraint]; self.visit_body(&node.body); @@ -850,11 +855,9 @@ where self.visit_expr(test); let pre_loop = self.flow_snapshot(); - let pre_loop_constraints = self.constraints_snapshot(); + let pre_loop_constraints = self.branching_conditions_snapshot(); let constraint = self.record_expression_constraint(test); - self.record_expression_constraint(test); - // Save aside any break states from an outer loop let saved_break_states = std::mem::take(&mut self.loop_break_states); @@ -922,7 +925,7 @@ where self.visit_expr(iter); let pre_loop = self.flow_snapshot(); - let pre_loop_constraints = self.constraints_snapshot(); + let pre_loop_constraints = self.branching_conditions_snapshot(); let saved_break_states = std::mem::take(&mut self.loop_break_states); debug_assert_eq!(&self.current_assignments, &[]); @@ -961,7 +964,7 @@ where self.visit_expr(subject); let after_subject = self.flow_snapshot(); - let after_subject_cs = self.constraints_snapshot(); + let after_subject_cs = self.branching_conditions_snapshot(); let Some((first, remaining)) = cases.split_first() else { return; }; @@ -1000,7 +1003,9 @@ where // We will merge this state with all of the intermediate // states during the `try` block before visiting those suites. let pre_try_block_state = self.flow_snapshot(); - let pre_try_block_constraints = self.constraints_snapshot(); + let pre_try_block_conditions = self.branching_conditions_snapshot(); + + self.record_unconditional_branching(); self.try_node_context_stack_manager.push_context(); @@ -1021,19 +1026,21 @@ where // as there necessarily must have been 0 `except` blocks executed // if we hit the `else` block. let post_try_block_state = self.flow_snapshot(); - let post_try_block_constraints = self.constraints_snapshot(); + let post_try_block_constraints = self.branching_conditions_snapshot(); // Prepare for visiting the `except` block(s) - self.flow_restore(pre_try_block_state, pre_try_block_constraints.clone()); + self.flow_restore(pre_try_block_state, pre_try_block_conditions.clone()); for state in try_block_snapshots { - self.flow_merge(state, pre_try_block_constraints.clone()); + self.flow_merge(state, pre_try_block_conditions.clone()); // TODO? } let pre_except_state = self.flow_snapshot(); - let pre_except_constraints = self.constraints_snapshot(); + let pre_except_constraints = self.branching_conditions_snapshot(); let num_handlers = handlers.len(); + self.record_unconditional_branching(); + for (i, except_handler) in handlers.iter().enumerate() { let ast::ExceptHandler::ExceptHandler(except_handler) = except_handler; let ast::ExceptHandlerExceptHandler { @@ -1085,7 +1092,7 @@ where self.visit_body(orelse); for post_except_state in post_except_states { - self.flow_merge(post_except_state, pre_try_block_constraints.clone()); + self.flow_merge(post_except_state, pre_try_block_conditions.clone()); } // TODO: there's lots of complexity here that isn't yet handled by our model. @@ -1242,7 +1249,7 @@ where }) => { self.visit_expr(test); let pre_if = self.flow_snapshot(); - let pre_if_constraints = self.constraints_snapshot(); + let pre_if_constraints = self.branching_conditions_snapshot(); let constraint = self.record_expression_constraint(test); self.visit_expr(body); let post_body = self.flow_snapshot(); @@ -1311,7 +1318,7 @@ where // AST inspection, so we can't simplify here, need to record test expression for // later checking) let mut snapshots = vec![]; - let pre_op_constraints = self.constraints_snapshot(); + let pre_op_constraints = self.branching_conditions_snapshot(); for (index, value) in values.iter().enumerate() { self.visit_expr(value); // In the last value we don't need to take a snapshot nor add a constraint diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 7aebd7668d5a0..5a92b2917670c 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -226,13 +226,17 @@ use self::symbol_state::{ ScopedConstraintId, ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState, }; use crate::semantic_index::ast_ids::{HasScopedExpressionId, ScopedUseId}; +use crate::semantic_index::branching::BranchingCondition; use crate::semantic_index::constraint::ConstraintNode; use crate::semantic_index::definition::Definition; use crate::semantic_index::symbol::ScopedSymbolId; use crate::semantic_index::use_def::bitset::BitSet; -use crate::semantic_index::use_def::symbol_state::BranchingConditions; +use crate::semantic_index::use_def::symbol_state::{ + BranchingConditionIdIterator, BranchingConditions, ScopedBranchingConditionId, +}; use crate::symbol::Boundness; -use crate::types::{infer_expression_types, KnownClass}; +use crate::types::{infer_expression_types, KnownClass, Truthiness}; +use crate::Db; use ruff_index::IndexVec; use rustc_hash::FxHashMap; @@ -250,6 +254,9 @@ pub(crate) struct UseDefMap<'db> { /// Array of [`Constraint`] in this scope. all_constraints: IndexVec>, + /// Array of [`BranchingCondition`] in this scope. + all_branching_conditions: IndexVec>, + /// [`SymbolBindings`] reaching a [`ScopedUseId`]. bindings_by_use: IndexVec, @@ -272,109 +279,6 @@ pub(crate) struct UseDefMap<'db> { } impl<'db> UseDefMap<'db> { - #[cfg(test)] - #[allow(clippy::print_stdout)] - pub(crate) fn print(&self, db: &dyn crate::db::Db) { - use crate::semantic_index::constraint::ConstraintNode; - - println!("all_definitions:"); - println!("================"); - - for (id, d) in self.all_definitions.iter_enumerated() { - println!( - "{:?}: {:?} {:?} {:?}", - id, - d.category(db), - d.scope(db), - d.symbol(db), - ); - println!(" {:?}", d.kind(db)); - println!(); - } - - println!("all_constraints:"); - println!("================"); - - for (id, c) in self.all_constraints.iter_enumerated() { - println!("{:?}: {:?}", id, c.node); - if let ConstraintNode::Expression(e) = c.node { - println!(" {:?}", e.node_ref(db)); - } - } - - println!(); - - println!("bindings_by_use:"); - println!("================"); - - for (id, bindings) in self.bindings_by_use.iter_enumerated() { - println!("{id:?}:"); - for binding in bindings.iter() { - let definition = self.all_definitions[binding.definition]; - let mut constraint_ids = binding.constraint_ids.peekable(); - let mut active_constraint_ids = binding.branching_conditions_ids.peekable(); - - println!(" * {definition:?}"); - - if constraint_ids.peek().is_some() { - println!(" Constraints:"); - for constraint_id in constraint_ids { - println!(" {:?}", self.all_constraints[constraint_id]); - } - } else { - println!(" No constraints"); - } - - println!(); - - if active_constraint_ids.peek().is_some() { - println!(" Active constraints at binding:"); - for constraint_id in active_constraint_ids { - println!(" {:?}", self.all_constraints[constraint_id]); - } - } else { - println!(" No active constraints at binding"); - } - } - } - - println!(); - - println!("public_symbols:"); - println!("================"); - - for (id, symbol) in self.public_symbols.iter_enumerated() { - println!("{id:?}:"); - println!(" * Bindings:"); - for binding in symbol.bindings().iter() { - let definition = self.all_definitions[binding.definition]; - let mut constraint_ids = binding.constraint_ids.peekable(); - - println!(" {definition:?}"); - - if constraint_ids.peek().is_some() { - println!(" Constraints:"); - for constraint_id in constraint_ids { - println!(" {:?}", self.all_constraints[constraint_id]); - } - } else { - println!(" No constraints"); - } - } - - println!(" * Declarations:"); - for (declaration, _) in symbol.declarations().iter() { - let definition = self.all_definitions[declaration]; - println!(" {definition:?}"); - } - - println!(); - } - - println!(); - println!(); - } - pub(crate) fn bindings_at_use( &self, use_id: ScopedUseId, @@ -392,41 +296,13 @@ impl<'db> UseDefMap<'db> { let mut definitely_bound = false; let mut definitely_unbound = true; for binding in bindings_iter { - let test_expr_tys = || { - binding.branching_conditions.clone().map(|c| { - let ty = if let ConstraintNode::Expression(test_expr) = c.node { - let inference = infer_expression_types(db, test_expr); - let scope = test_expr.scope(db); - inference - .expression_ty(test_expr.node_ref(db).scoped_expression_id(db, scope)) - } else { - // TODO: handle other constraint nodes - KnownClass::Bool.to_instance(db) - }; - - (c, ty) - }) - }; + let truthiness = binding.branching_conditions.branch_condition_truthiness(db); - let is_any_always_false = test_expr_tys().any(|(c, test_expr_ty)| { - if c.is_positive { - test_expr_ty.bool(db).is_always_false() - } else { - test_expr_ty.bool(db).is_always_true() - } - }); - if !is_any_always_false { + if !truthiness.any_always_false { definitely_unbound = false; } - let are_all_always_true = test_expr_tys().all(|(c, test_expr_ty)| { - if c.is_positive { - test_expr_ty.bool(db).is_always_true() - } else { - test_expr_ty.bool(db).is_always_false() - } - }); - if are_all_always_true { + if truthiness.all_always_true { definitely_bound = true; } } @@ -509,6 +385,7 @@ impl<'db> UseDefMap<'db> { BindingWithConstraintsIterator { all_definitions: &self.all_definitions, all_constraints: &self.all_constraints, + all_branching_conditions: &self.all_branching_conditions, inner: bindings.iter(), } } @@ -537,6 +414,7 @@ enum SymbolDefinitions { pub(crate) struct BindingWithConstraintsIterator<'map, 'db> { all_definitions: &'map IndexVec>, all_constraints: &'map IndexVec>, + all_branching_conditions: &'map IndexVec>, inner: BindingIdWithConstraintsIterator<'map>, } @@ -544,19 +422,17 @@ impl<'map, 'db> Iterator for BindingWithConstraintsIterator<'map, 'db> { type Item = BindingWithConstraints<'map, 'db>; fn next(&mut self) -> Option { - self.inner - .next() - .map(|def_id_with_constraints| BindingWithConstraints { - binding: self.all_definitions[def_id_with_constraints.definition], - constraints: ConstraintsIterator { - all_constraints: self.all_constraints, - constraint_ids: def_id_with_constraints.constraint_ids, - }, - branching_conditions: ConstraintsIterator { - all_constraints: self.all_constraints, - constraint_ids: def_id_with_constraints.branching_conditions_ids, - }, - }) + self.inner.next().map(|binding| BindingWithConstraints { + binding: self.all_definitions[binding.definition], + constraints: ConstraintsIterator { + all_constraints: self.all_constraints, + constraint_ids: binding.constraint_ids, + }, + branching_conditions: BranchingConditionsIterator { + all_branching_conditions: self.all_branching_conditions, + branching_condition_ids: binding.branching_conditions_ids, + }, + }) } } @@ -565,7 +441,7 @@ impl std::iter::FusedIterator for BindingWithConstraintsIterator<'_, '_> {} pub(crate) struct BindingWithConstraints<'map, 'db> { pub(crate) binding: Definition<'db>, pub(crate) constraints: ConstraintsIterator<'map, 'db>, - pub(crate) branching_conditions: ConstraintsIterator<'map, 'db>, + pub(crate) branching_conditions: BranchingConditionsIterator<'map, 'db>, } #[derive(Debug, Clone)] @@ -586,6 +462,67 @@ impl<'db> Iterator for ConstraintsIterator<'_, 'db> { impl std::iter::FusedIterator for ConstraintsIterator<'_, '_> {} +#[derive(Debug, Clone)] +pub(crate) struct BranchingConditionsIterator<'map, 'db> { + all_branching_conditions: &'map IndexVec>, + branching_condition_ids: BranchingConditionIdIterator<'map>, +} + +impl<'db> Iterator for BranchingConditionsIterator<'_, 'db> { + type Item = BranchingCondition<'db>; + + fn next(&mut self) -> Option { + self.branching_condition_ids + .next() + .map(|branching_condition_id| self.all_branching_conditions[branching_condition_id]) + } +} + +pub struct BranchConditionTruthiness { + pub any_always_false: bool, + pub all_always_true: bool, + pub at_least_one_condition: bool, +} + +impl<'db> BranchingConditionsIterator<'_, 'db> { + pub(crate) fn branch_condition_truthiness(self, db: &'db dyn Db) -> BranchConditionTruthiness { + let mut result = BranchConditionTruthiness { + any_always_false: false, + all_always_true: true, + at_least_one_condition: false, + }; + + for condition in self { + let truthiness = match condition { + BranchingCondition::Conditional(Constraint { + node: ConstraintNode::Expression(test_expr), + is_positive, + }) => { + let inference = infer_expression_types(db, test_expr); + let scope = test_expr.scope(db); + let ty = inference + .expression_ty(test_expr.node_ref(db).scoped_expression_id(db, scope)); + + ty.bool(db).negate_if(!is_positive) + } + BranchingCondition::Conditional(Constraint { + node: ConstraintNode::Pattern(..), + .. + }) => Truthiness::Ambiguous, + BranchingCondition::Unconditional => Truthiness::Ambiguous, + }; + + result.any_always_false |= truthiness.is_always_false(); + result.all_always_true &= truthiness.is_always_true(); + result.at_least_one_condition = true; + } + + result + } +} + +impl std::iter::FusedIterator for BranchingConditionsIterator<'_, '_> {} + pub(crate) struct DeclarationsIterator<'map, 'db> { all_definitions: &'map IndexVec>, all_constraints: &'map IndexVec>, @@ -634,7 +571,11 @@ pub(super) struct UseDefMapBuilder<'db> { /// Append-only array of [`Constraint`]. all_constraints: IndexVec>, - branching_conditions: BranchingConditions, + /// Append-only array of [`BranchingCondition`]. + all_branching_conditions: IndexVec>, + + /// Active branching conditions. + active_branching_conditions: BranchingConditions, /// Live bindings at each so-far-recorded use. bindings_by_use: IndexVec, @@ -659,7 +600,7 @@ impl<'db> UseDefMapBuilder<'db> { binding, SymbolDefinitions::Declarations(symbol_state.declarations().clone()), ); - symbol_state.record_binding(def_id, &self.branching_conditions); + symbol_state.record_binding(def_id, &self.active_branching_conditions); } pub(super) fn record_constraint(&mut self, constraint: Constraint<'db>) { @@ -667,7 +608,18 @@ impl<'db> UseDefMapBuilder<'db> { for state in &mut self.symbol_states { state.record_constraint(constraint_id); } - self.branching_conditions.insert(constraint_id.as_u32()); + + self.record_branching_condition(BranchingCondition::Conditional(constraint)); + } + + pub(super) fn record_unconditional_branching(&mut self) { + self.record_branching_condition(BranchingCondition::Unconditional); + } + + pub(super) fn record_branching_condition(&mut self, condition: BranchingCondition<'db>) { + let condition_id = self.all_branching_conditions.push(condition); + self.active_branching_conditions + .insert(condition_id.as_u32()); } pub(super) fn record_declaration( @@ -681,7 +633,7 @@ impl<'db> UseDefMapBuilder<'db> { declaration, SymbolDefinitions::Bindings(symbol_state.bindings().clone()), ); - symbol_state.record_declaration(def_id, &self.branching_conditions); + symbol_state.record_declaration(def_id, &self.active_branching_conditions); } pub(super) fn record_declaration_and_binding( @@ -692,8 +644,8 @@ impl<'db> UseDefMapBuilder<'db> { // We don't need to store anything in self.definitions_by_definition. let def_id = self.all_definitions.push(definition); let symbol_state = &mut self.symbol_states[symbol]; - symbol_state.record_declaration(def_id, &self.branching_conditions); - symbol_state.record_binding(def_id, &self.branching_conditions); + symbol_state.record_declaration(def_id, &self.active_branching_conditions); + symbol_state.record_binding(def_id, &self.active_branching_conditions); } pub(super) fn record_use(&mut self, symbol: ScopedSymbolId, use_id: ScopedUseId) { @@ -712,8 +664,8 @@ impl<'db> UseDefMapBuilder<'db> { } } - pub(super) fn constraints_snapshot(&self) -> BranchingConditionsSnapshot { - BranchingConditionsSnapshot(self.branching_conditions.clone()) + pub(super) fn branching_conditions_snapshot(&self) -> BranchingConditionsSnapshot { + BranchingConditionsSnapshot(self.active_branching_conditions.clone()) } /// Restore the current builder symbols state to the given snapshot. @@ -734,8 +686,8 @@ impl<'db> UseDefMapBuilder<'db> { .resize(num_symbols, SymbolState::undefined()); } - pub(super) fn restore_constraints(&mut self, snapshot: BranchingConditionsSnapshot) { - self.branching_conditions = snapshot.0; + pub(super) fn restore_branching_conditions(&mut self, snapshot: BranchingConditionsSnapshot) { + self.active_branching_conditions = snapshot.0; } /// Merge the given snapshot into the current state, reflecting that we might have taken either @@ -769,6 +721,7 @@ impl<'db> UseDefMapBuilder<'db> { UseDefMap { all_definitions: self.all_definitions, all_constraints: self.all_constraints, + all_branching_conditions: self.all_branching_conditions, bindings_by_use: self.bindings_by_use, public_symbols: self.symbol_states, definitions_by_definition: self.definitions_by_definition, diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 3bc3a4bddb305..02157ff9d1837 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -55,6 +55,10 @@ pub(super) struct ScopedDefinitionId; #[newtype_index] pub(super) struct ScopedConstraintId; +/// A newtype-index for a branching condition in a particular scope. +#[newtype_index] +pub(super) struct ScopedBranchingConditionId; + /// Can reference this * 64 total definitions inline; more will fall back to the heap. const INLINE_BINDING_BLOCKS: usize = 3; @@ -83,6 +87,9 @@ type ConstraintsIntoIterator = smallvec::IntoIter; const INLINE_BRANCHING_CONDITIONS: usize = 2; pub(super) type BranchingConditions = BitSet; +type BranchingConditionsPerBinding = SmallVec<[BranchingConditions; INLINE_BINDINGS_PER_SYMBOL]>; + +type BranchingConditionsIterator<'a> = std::slice::Iter<'a, BitSet>; /// Live declarations for a single symbol at some point in control flow. #[derive(Clone, Debug, PartialEq, Eq)] @@ -156,8 +163,8 @@ pub(super) struct SymbolBindings { /// binding in `live_bindings`. constraints: Constraints, - /// For each live binding, which [`BranchingCondition`]s were active *at the time of the binding*? - pub(crate) branching_conditions: Constraints, + /// For each live binding, which [`BranchingConditions`] were active *at the time of the binding*? + pub(crate) branching_conditions: BranchingConditionsPerBinding, /// Could the symbol be unbound at this point? may_be_unbound: bool, @@ -168,7 +175,7 @@ impl SymbolBindings { Self { live_bindings: Bindings::default(), constraints: Constraints::default(), - branching_conditions: Constraints::default(), + branching_conditions: BranchingConditionsPerBinding::default(), may_be_unbound: true, } } @@ -189,7 +196,8 @@ impl SymbolBindings { self.live_bindings = Bindings::with(binding_id.into()); self.constraints = Constraints::with_capacity(1); self.constraints.push(BitSet::default()); - self.branching_conditions = Constraints::with_capacity(1); + + self.branching_conditions = BranchingConditionsPerBinding::with_capacity(1); self.branching_conditions.push(BitSet::default()); for id in branching_conditions.iter() { self.branching_conditions[0].insert(id); @@ -493,14 +501,14 @@ impl Default for SymbolState { pub(super) struct BindingIdWithConstraints<'a> { pub(super) definition: ScopedDefinitionId, pub(super) constraint_ids: ConstraintIdIterator<'a>, - pub(super) branching_conditions_ids: ConstraintIdIterator<'a>, + pub(super) branching_conditions_ids: BranchingConditionIdIterator<'a>, } #[derive(Debug, Clone)] pub(super) struct BindingIdWithConstraintsIterator<'a> { definitions: BindingsIterator<'a>, constraints: ConstraintsIterator<'a>, - branching_conditions: ConstraintsIterator<'a>, + branching_conditions: BranchingConditionsIterator<'a>, } impl<'a> Iterator for BindingIdWithConstraintsIterator<'a> { @@ -519,7 +527,7 @@ impl<'a> Iterator for BindingIdWithConstraintsIterator<'a> { constraint_ids: ConstraintIdIterator { wrapped: constraints.iter(), }, - branching_conditions_ids: ConstraintIdIterator { + branching_conditions_ids: BranchingConditionIdIterator { wrapped: branching_conditions.iter(), }, }) @@ -547,6 +555,23 @@ impl Iterator for ConstraintIdIterator<'_> { impl std::iter::FusedIterator for ConstraintIdIterator<'_> {} +#[derive(Debug, Clone)] +pub(super) struct BranchingConditionIdIterator<'a> { + wrapped: BitSetIterator<'a, INLINE_CONSTRAINT_BLOCKS>, +} + +impl Iterator for BranchingConditionIdIterator<'_> { + type Item = ScopedBranchingConditionId; + + fn next(&mut self) -> Option { + self.wrapped + .next() + .map(ScopedBranchingConditionId::from_u32) + } +} + +impl std::iter::FusedIterator for BranchingConditionIdIterator<'_> {} + #[derive(Debug)] pub(super) struct DeclarationIdIterator<'a> { inner: DeclarationsIterator<'a>, diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 95755e90f2d2b..b933d865c62e8 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -268,46 +268,18 @@ fn bindings_ty<'db>( constraints, branching_conditions, }| { - let test_expr_tys = || { - branching_conditions.clone().map(|c| { - let ty = if let ConstraintNode::Expression(test_expr) = c.node { - let inference = infer_expression_types(db, test_expr); - let scope = test_expr.scope(db); - inference - .expression_ty(test_expr.node_ref(db).scoped_expression_id(db, scope)) - } else { - // TODO: handle other constraint nodes - todo_type!() - }; - - (c, ty) - }) - }; + let result = branching_conditions.branch_condition_truthiness(db); - if test_expr_tys().any(|(c, test_expr_ty)| { - if c.is_positive { - test_expr_ty.bool(db).is_always_false() - } else { - test_expr_ty.bool(db).is_always_true() - } - }) { + if result.any_always_false { // TODO: do we need to call binding_ty(…) even if we don't need the result? (Type::Never, UnconditionallyVisible::No) } else { - let mut test_expr_tys_iter = test_expr_tys().peekable(); - - let unconditionally_visible = if test_expr_tys_iter.peek().is_some() - && test_expr_tys_iter.all(|(c, test_expr_ty)| { - if c.is_positive { - test_expr_ty.bool(db).is_always_true() - } else { - test_expr_ty.bool(db).is_always_false() - } - }) { - UnconditionallyVisible::Yes - } else { - UnconditionallyVisible::No - }; + let unconditionally_visible = + if result.at_least_one_condition && result.all_always_true { + UnconditionallyVisible::Yes + } else { + UnconditionallyVisible::No + }; let mut constraint_tys = constraints .filter_map(|constraint| narrowing_constraint(db, constraint, binding)) @@ -2644,7 +2616,7 @@ impl Truthiness { matches!(self, Truthiness::AlwaysTrue) } - const fn negate(self) -> Self { + pub(crate) const fn negate(self) -> Self { match self { Self::AlwaysTrue => Self::AlwaysFalse, Self::AlwaysFalse => Self::AlwaysTrue, @@ -2652,6 +2624,14 @@ impl Truthiness { } } + pub(crate) const fn negate_if(self, condition: bool) -> Self { + if condition { + self.negate() + } else { + self + } + } + fn into_type(self, db: &dyn Db) -> Type { match self { Self::AlwaysTrue => Type::BooleanLiteral(true), From 531a38ad378a6fc4b5be882dc9f2e057b86e58e8 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 11:11:30 +0100 Subject: [PATCH 09/68] Same handling for definitions --- .../src/semantic_index/use_def.rs | 19 +++++----- .../semantic_index/use_def/symbol_state.rs | 6 ++-- crates/red_knot_python_semantic/src/types.rs | 36 ++----------------- 3 files changed, 15 insertions(+), 46 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 5a92b2917670c..f435878bea784 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -230,12 +230,11 @@ use crate::semantic_index::branching::BranchingCondition; use crate::semantic_index::constraint::ConstraintNode; use crate::semantic_index::definition::Definition; use crate::semantic_index::symbol::ScopedSymbolId; -use crate::semantic_index::use_def::bitset::BitSet; use crate::semantic_index::use_def::symbol_state::{ BranchingConditionIdIterator, BranchingConditions, ScopedBranchingConditionId, }; use crate::symbol::Boundness; -use crate::types::{infer_expression_types, KnownClass, Truthiness}; +use crate::types::{infer_expression_types, Truthiness}; use crate::Db; use ruff_index::IndexVec; use rustc_hash::FxHashMap; @@ -396,7 +395,7 @@ impl<'db> UseDefMap<'db> { ) -> DeclarationsIterator<'a, 'db> { DeclarationsIterator { all_definitions: &self.all_definitions, - all_constraints: &self.all_constraints, + all_branching_conditions: &self.all_branching_conditions, inner: declarations.iter(), may_be_undeclared: declarations.may_be_undeclared(), } @@ -478,7 +477,7 @@ impl<'db> Iterator for BranchingConditionsIterator<'_, 'db> { } } -pub struct BranchConditionTruthiness { +pub(crate) struct BranchConditionTruthiness { pub any_always_false: bool, pub all_always_true: bool, pub at_least_one_condition: bool, @@ -525,7 +524,7 @@ impl std::iter::FusedIterator for BranchingConditionsIterator<'_, '_> {} pub(crate) struct DeclarationsIterator<'map, 'db> { all_definitions: &'map IndexVec>, - all_constraints: &'map IndexVec>, + all_branching_conditions: &'map IndexVec>, inner: DeclarationIdIterator<'map>, may_be_undeclared: bool, } @@ -537,15 +536,15 @@ impl DeclarationsIterator<'_, '_> { } impl<'map, 'db> Iterator for DeclarationsIterator<'map, 'db> { - type Item = (Definition<'db>, ConstraintsIterator<'map, 'db>); + type Item = (Definition<'db>, BranchingConditionsIterator<'map, 'db>); fn next(&mut self) -> Option { - self.inner.next().map(|(def_id, constraints)| { + self.inner.next().map(|(def_id, branching_condition_ids)| { ( self.all_definitions[def_id], - ConstraintsIterator { - all_constraints: self.all_constraints, - constraint_ids: constraints, + BranchingConditionsIterator { + all_branching_conditions: self.all_branching_conditions, + branching_condition_ids, }, ) }) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 02157ff9d1837..51f4b0cd0cf6f 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -575,11 +575,11 @@ impl std::iter::FusedIterator for BranchingConditionIdIterator<'_> {} #[derive(Debug)] pub(super) struct DeclarationIdIterator<'a> { inner: DeclarationsIterator<'a>, - branching_conditions: ConstraintsIterator<'a>, + branching_conditions: BranchingConditionsIterator<'a>, } impl<'a> Iterator for DeclarationIdIterator<'a> { - type Item = (ScopedDefinitionId, ConstraintIdIterator<'a>); + type Item = (ScopedDefinitionId, BranchingConditionIdIterator<'a>); fn next(&mut self) -> Option { // self.inner.next().map(ScopedDefinitionId::from_u32) @@ -587,7 +587,7 @@ impl<'a> Iterator for DeclarationIdIterator<'a> { (None, None) => None, (Some(declaration), Some(branching_conditions)) => Some(( ScopedDefinitionId::from_u32(declaration), - ConstraintIdIterator { + BranchingConditionIdIterator { wrapped: branching_conditions.iter(), }, )), diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index b933d865c62e8..09d3b1b3a91cb 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -16,7 +16,6 @@ pub(crate) use self::infer::{ pub(crate) use self::signatures::Signature; use crate::module_resolver::file_to_module; use crate::semantic_index::ast_ids::HasScopedExpressionId; -use crate::semantic_index::constraint::ConstraintNode; use crate::semantic_index::definition::Definition; use crate::semantic_index::symbol::{self as symbol, ScopeId, ScopedSymbolId}; use crate::semantic_index::{ @@ -352,41 +351,12 @@ fn declarations_ty<'db>( undeclared_ty: Option>, ) -> DeclaredTypeResult<'db> { let decl_types = declarations.map(|(declaration, branching_conditions)| { - let test_expr_tys = || { - branching_conditions.clone().map(|c| { - let ty = if let ConstraintNode::Expression(test_expr) = c.node { - let inference = infer_expression_types(db, test_expr); - let scope = test_expr.scope(db); - inference.expression_ty(test_expr.node_ref(db).scoped_expression_id(db, scope)) - } else { - // TODO: handle other constraint nodes - todo_type!() - }; - - (c, ty) - }) - }; + let result = branching_conditions.branch_condition_truthiness(db); - if test_expr_tys().any(|(c, test_expr_ty)| { - if c.is_positive { - test_expr_ty.bool(db).is_always_false() - } else { - test_expr_ty.bool(db).is_always_true() - } - }) { + if result.any_always_false { (Type::Never, UnconditionallyVisible::No) } else { - let mut test_expr_tys_iter = test_expr_tys().peekable(); - - if test_expr_tys_iter.peek().is_some() - && test_expr_tys_iter.all(|(c, test_expr_ty)| { - if c.is_positive { - test_expr_ty.bool(db).is_always_true() - } else { - test_expr_ty.bool(db).is_always_false() - } - }) - { + if result.at_least_one_condition && result.all_always_true { (declaration_ty(db, declaration), UnconditionallyVisible::Yes) } else { (declaration_ty(db, declaration), UnconditionallyVisible::No) From afd56f073859e0c02c36e0a0bf8e515b6fb6cbbd Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 11:50:14 +0100 Subject: [PATCH 10/68] Remove comment --- .../red_knot_python_semantic/src/semantic_index/builder.rs | 3 --- .../red_knot_python_semantic/src/semantic_index/use_def.rs | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index 5476aa25e6ee3..df21cd8c74d46 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -1314,9 +1314,6 @@ where range: _, op, }) => { - // TODO detect statically known truthy or falsy values (via type inference, not naive - // AST inspection, so we can't simplify here, need to record test expression for - // later checking) let mut snapshots = vec![]; let pre_op_constraints = self.branching_conditions_snapshot(); for (index, value) in values.iter().enumerate() { diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index f435878bea784..018d238ee752a 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -295,13 +295,13 @@ impl<'db> UseDefMap<'db> { let mut definitely_bound = false; let mut definitely_unbound = true; for binding in bindings_iter { - let truthiness = binding.branching_conditions.branch_condition_truthiness(db); + let result = binding.branching_conditions.branch_condition_truthiness(db); - if !truthiness.any_always_false { + if !result.any_always_false { definitely_unbound = false; } - if truthiness.all_always_true { + if result.all_always_true { definitely_bound = true; } } From 6db9e8a392dbddc91d50c26231850bb9bd9a8430 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 12:01:54 +0100 Subject: [PATCH 11/68] Add tests for 'while'-narrowing --- .../resources/mdtest/narrow/while.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/red_knot_python_semantic/resources/mdtest/narrow/while.md b/crates/red_knot_python_semantic/resources/mdtest/narrow/while.md index ac91a576c9de1..17f630d023ddf 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/narrow/while.md +++ b/crates/red_knot_python_semantic/resources/mdtest/narrow/while.md @@ -1,3 +1,4 @@ +<<<<<<< HEAD # Narrowing in `while` loops We only make sure that narrowing works for `while` loops in general, we do not exhaustively test all @@ -56,3 +57,24 @@ while x != 1: x = next_item() ``` +||||||| parent of 8247164bf (Add tests for 'while'-narrowing) +======= +# Narrowing in `while` loops + +We only make sure that narrowing works for `while` loops in general, we do not exhaustively test all +narrowing forms here, as they are covered in other tests. + +## Basic example + +```py +def next_item() -> int | None: ... + +x = next_item() + +while x is not None: + reveal_type(x) # revealed: int + x = next_item() +else: + reveal_type(x) # revealed: None +``` +>>>>>>> 8247164bf (Add tests for 'while'-narrowing) From af922c9eabde4d43f48348887664a24de0d31522 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 12:44:25 +0100 Subject: [PATCH 12/68] More Boolean expression tests --- .../mdtest/statically-known-branches.md | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 17e9d7c971fb1..c71e9298b319c 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -231,7 +231,7 @@ reveal_type(x) # revealed: Literal[2] ## Boolean expressions -### Always true +### Always true, `or` ```py (x := 1) == 1 or (x := 2) @@ -239,7 +239,15 @@ reveal_type(x) # revealed: Literal[2] reveal_type(x) # revealed: Literal[1] ``` -### Always false +### Always true, `and` + +```py +(x := 1) == 1 and (x := 2) + +reveal_type(x) # revealed: Literal[2] +``` + +### Always false, `or` ```py (x := 1) == 0 or (x := 2) @@ -247,6 +255,14 @@ reveal_type(x) # revealed: Literal[1] reveal_type(x) # revealed: Literal[2] ``` +### Always false, `and` + +```py +(x := 1) == 0 and (x := 2) + +reveal_type(x) # revealed: Literal[1] +``` + ## While loops ### Always false From c29fbfc668d10f899168c3f1092dd0612fc2aea6 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 13:04:41 +0100 Subject: [PATCH 13/68] More tests --- .../mdtest/statically-known-branches.md | 163 +++++++++++++++--- 1 file changed, 142 insertions(+), 21 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index c71e9298b319c..a6798939eee7a 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -54,6 +54,21 @@ else: reveal_type(x) # revealed: Literal[2] ``` +### Ambiguous + +Just for comparison, we still infer the combined type if the condition is not statically known: + +```py +def flag() -> bool: ... + +x = 1 + +if flag(): + x = 2 + +reveal_type(x) # revealed: Literal[1, 2] +``` + ### Combination of always true and always false ```py @@ -290,6 +305,8 @@ reveal_type(x) # revealed: Literal[2] ### Ambiguous +Make sure that we still infer the combined type if the condition is not statically known: + ```py def flag() -> bool: ... @@ -297,13 +314,35 @@ x = 1 while flag(): x = 2 - break reveal_type(x) # revealed: Literal[1, 2] ``` +### `while` ... `else` + +```py path=while_false.py +while False: + x = 1 +else: + x = 2 + +reveal_type(x) # revealed: Literal[2] +``` + +```py path=while_true.py +while True: + x = 1 + break +else: + x = 2 + +reveal_type(x) # revealed: Literal[1] +``` + ## Conditional declarations +### Always false + ```py path=if_false.py x: str @@ -326,6 +365,8 @@ def f() -> None: reveal_type(x) # revealed: str ``` +### Always true + ```py path=if_true.py x: str @@ -348,6 +389,8 @@ def f() -> None: reveal_type(x) # revealed: int ``` +### Ambiguous + ```py path=if_bool.py def flag() -> bool: ... @@ -360,7 +403,7 @@ def f() -> None: reveal_type(x) # revealed: str | int ``` -## Conditionally defined functions +## Conditional function definitions ```py def f() -> int: ... @@ -376,7 +419,21 @@ reveal_type(f()) # revealed: str reveal_type(g()) # revealed: int ``` -## Conditionally defined class attributes +## Conditional class definitions + +```py +if True: + class C: + x: int = 1 + +else: + class C: + x: str = "a" + +reveal_type(C.x) # revealed: int +``` + +## Conditional class attributes ```py class C: @@ -434,54 +491,78 @@ else: x ``` -### Nested +### Ambiguous, possibly unbound + +For comparison, we still definitions inside non-statically known branches as possibly unbound: ```py +def flag() -> bool: ... + +if flag(): + x = 1 + +# error: [possibly-unresolved-reference] +x +``` + +### Nested conditionals + +```py +def flag() -> bool: ... + if False: if True: - x = 1 + unbound1 = 1 if True: if False: - y = 1 + unbound2 = 1 if False: if False: - z = 1 + unbound3 = 1 +if False: + if flag(): + unbound4 = 1 + +if flag(): + if False: + unbound5 = 1 + +# error: [unresolved-reference] +# error: [unresolved-reference] # error: [unresolved-reference] # error: [unresolved-reference] # error: [unresolved-reference] -(x, y, z) +(unbound1, unbound2, unbound3, unbound4, unbound5) ``` -### Multiple nested conditions +### Chained conditionals ```py +if False: + x = 1 if True: - if False: - x = 1 - if True: - x = 2 + x = 2 # x is always bound, no error x +if False: + y = 1 if True: - if False: - y = 1 - if True: - y = 2 + y = 2 # y is always bound, no error y if False: - if False: - z = 1 - if False: - z = 2 + z = 1 +if False: + z = 2 +# z is never bound: # error: [unresolved-reference] z ``` @@ -496,3 +577,43 @@ def f(): # x is always bound, no error x ``` + +### Imports of conditionally defined symbols + +#### Always false, unbound + +```py path=module.py +if False: + symbol = 1 +``` + +```py +# error: [unresolved-import] +from module import symbol +``` + +#### Always true, bound + +```py path=module.py +if True: + symbol = 1 +``` + +```py +# no error +from module import symbol +``` + +#### Ambiguous, possibly unbound + +```py path=module.py +def flag() -> bool: ... + +if flag(): + symbol = 1 +``` + +```py +# error: [possibly-unbound-import] +from module import symbol +``` From cab9c2f24c1c21e020f7c288c828b356fb404c5a Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 13:16:06 +0100 Subject: [PATCH 14/68] Clarify TODO comments --- .../resources/mdtest/annotations/never.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md index 1cbab75e4a98a..3e4b8063783fb 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md +++ b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md @@ -70,9 +70,9 @@ target-version = "3.10" ``` ```py -# TODO: should raise a diagnostic +# TODO: should raise a diagnostic, see https://github.com/astral-sh/ruff/issues/14297 from typing import Never -# TODO: this should be Unknown +# TODO: this should be Unknown, not Never reveal_type(Never) # revealed: Never ``` From fabecf75ca26c0bf2b135a750e69a1463fbe9732 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 13:55:56 +0100 Subject: [PATCH 15/68] Add typing.TYPE_CHECKING --- .../resources/mdtest/annotations/never.md | 2 +- .../resources/mdtest/literal/ellipsis.md | 4 ++-- .../resources/mdtest/narrow/issubclass.md | 2 +- .../resources/mdtest/pep695_type_aliases.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md index 3e4b8063783fb..07ba7a508ce11 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md +++ b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md @@ -66,7 +66,7 @@ reveal_type(Never) # revealed: typing.Never ```toml [environment] -target-version = "3.10" +python-version = "3.10" ``` ```py diff --git a/crates/red_knot_python_semantic/resources/mdtest/literal/ellipsis.md b/crates/red_knot_python_semantic/resources/mdtest/literal/ellipsis.md index ab7f18f6ee18c..241b498372d49 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/literal/ellipsis.md +++ b/crates/red_knot_python_semantic/resources/mdtest/literal/ellipsis.md @@ -4,7 +4,7 @@ ```toml [environment] -target-version = "3.9" +python-version = "3.9" ``` ```py @@ -15,7 +15,7 @@ reveal_type(...) # revealed: ellipsis ```toml [environment] -target-version = "3.10" +python-version = "3.10" ``` ```py diff --git a/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md b/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md index 1d5a7d4c4e3bc..b1099a1f7ae83 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md +++ b/crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md @@ -99,7 +99,7 @@ def _(t: type[object]): ```toml [environment] -target-version = "3.10" +python-version = "3.10" ``` ```py diff --git a/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md b/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md index 8e6372930f0d3..dd94e6b2d55c4 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md +++ b/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md @@ -11,7 +11,7 @@ Type aliases are only available in Python 3.12 and later: ```toml [environment] -target-version = "3.12" +python-version = "3.12" ``` ## Basic From 50b6c1a533d2c435b1c0896652a4125d77a591b0 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 14:05:43 +0100 Subject: [PATCH 16/68] Infer Unknown instead of Never --- .../resources/mdtest/boolean/short_circuit.md | 2 +- crates/red_knot_python_semantic/src/types.rs | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md b/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md index 21246f81d209c..6ad75f185bb35 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md +++ b/crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md @@ -33,7 +33,7 @@ def _(flag: bool): ```py if True or (x := 1): # error: [unresolved-reference] - reveal_type(x) # revealed: Never + reveal_type(x) # revealed: Unknown if True and (x := 1): reveal_type(x) # revealed: Literal[1] diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 09d3b1b3a91cb..3befa12b872b5 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -271,7 +271,7 @@ fn bindings_ty<'db>( if result.any_always_false { // TODO: do we need to call binding_ty(…) even if we don't need the result? - (Type::Never, UnconditionallyVisible::No) + (None, UnconditionallyVisible::No) } else { let unconditionally_visible = if result.at_least_one_condition && result.all_always_true { @@ -292,9 +292,9 @@ fn bindings_ty<'db>( IntersectionBuilder::add_positive, ) .build(); - (intersection_ty, unconditionally_visible) + (Some(intersection_ty), unconditionally_visible) } else { - (binding_ty, unconditionally_visible) + (Some(binding_ty), unconditionally_visible) } } }, @@ -303,6 +303,10 @@ fn bindings_ty<'db>( // TODO: get rid of all the collects and clean up, obviously let def_types: Vec<_> = def_types.collect(); + if !def_types.is_empty() && def_types.iter().all(|(ty, _)| *ty == None) { + return Some(Type::Unknown); + } + // shrink the vector to only include everything from the last unconditionally visible binding let def_types: Vec<_> = def_types .iter() @@ -310,7 +314,7 @@ fn bindings_ty<'db>( .take_while_inclusive(|(_, unconditionally_visible)| { *unconditionally_visible != UnconditionallyVisible::Yes }) - .map(|(ty, _)| *ty) + .map(|(ty, _)| ty.unwrap_or(Type::Never)) .collect(); let mut def_types = def_types.into_iter().rev(); From 3868d75cd0e1089d59fb5d9148f90c6d4e7ff4c6 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 14:06:28 +0100 Subject: [PATCH 17/68] Remove superfluous test --- .../resources/mdtest/annotations/never.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md index 07ba7a508ce11..49da0fae2a7ad 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md +++ b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md @@ -72,7 +72,4 @@ python-version = "3.10" ```py # TODO: should raise a diagnostic, see https://github.com/astral-sh/ruff/issues/14297 from typing import Never - -# TODO: this should be Unknown, not Never -reveal_type(Never) # revealed: Never ``` From 659ef545e587edb81c9889b428a4668a6f67c7d7 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 14:25:06 +0100 Subject: [PATCH 18/68] Proper for-loop handling --- .../resources/mdtest/expression/if.md | 2 +- .../resources/mdtest/loops/async_for.md | 2 + .../resources/mdtest/loops/for.md | 3 + .../mdtest/statically-known-branches.md | 64 +++++++++++++++++++ .../src/semantic_index/builder.rs | 34 +++++----- 5 files changed, 88 insertions(+), 17 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/expression/if.md b/crates/red_knot_python_semantic/resources/mdtest/expression/if.md index c39991e0359e0..6461522cefa46 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/expression/if.md +++ b/crates/red_knot_python_semantic/resources/mdtest/expression/if.md @@ -7,7 +7,7 @@ def _(flag: bool): reveal_type(1 if flag else 2) # revealed: Literal[1, 2] ``` -## Statically known conditions in if expressions +## Statically known conditions in if-expressions ```py reveal_type(1 if True else 2) # revealed: Literal[1] diff --git a/crates/red_knot_python_semantic/resources/mdtest/loops/async_for.md b/crates/red_knot_python_semantic/resources/mdtest/loops/async_for.md index 3e634ef4320c2..c0735768763e2 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/loops/async_for.md +++ b/crates/red_knot_python_semantic/resources/mdtest/loops/async_for.md @@ -19,6 +19,7 @@ async def foo(): # TODO: should reveal `Unknown` because `__aiter__` is not defined # revealed: @Todo(async iterables/iterators) + # error: [possibly-unresolved-reference] reveal_type(x) ``` @@ -38,6 +39,7 @@ async def foo(): async for x in IntAsyncIterable(): pass + # error: [possibly-unresolved-reference] # revealed: @Todo(async iterables/iterators) reveal_type(x) ``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/loops/for.md b/crates/red_knot_python_semantic/resources/mdtest/loops/for.md index 5a6077360557d..58675475abe72 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/loops/for.md +++ b/crates/red_knot_python_semantic/resources/mdtest/loops/for.md @@ -15,6 +15,7 @@ for x in IntIterable(): pass # revealed: int +# error: [possibly-unresolved-reference] reveal_type(x) ``` @@ -87,6 +88,7 @@ for x in OldStyleIterable(): pass # revealed: int +# error: [possibly-unresolved-reference] reveal_type(x) ``` @@ -97,6 +99,7 @@ for x in (1, "a", b"foo"): pass # revealed: Literal[1] | Literal["a"] | Literal[b"foo"] +# error: [possibly-unresolved-reference] reveal_type(x) ``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index a6798939eee7a..80ba34e8262d3 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -190,6 +190,8 @@ reveal_type(x) # revealed: Literal[3, 4] ### Combination with non-conditional control flow +#### `try` ... `except` + ```py path=try_if_true.py def may_raise() -> None: ... @@ -224,6 +226,68 @@ else: reveal_type(x) # revealed: Literal[2, 3] ``` +#### `for` loops + +```py path=for_if_true.py +def iterable() -> list[()]: ... + +x = 1 + +for _ in iterable(): + if True: + x = 2 + else: + x = 3 + +reveal_type(x) # revealed: Literal[1, 2] +``` + +```py path=for_else_if_true.py +def iterable() -> list[()]: ... + +x = 1 + +for _ in iterable(): + x = 2 +else: + if True: + x = 3 + else: + x = 4 + +reveal_type(x) # revealed: Literal[3] +``` + +```py path=if_true_for.py +def iterable() -> list[()]: ... + +x = 1 + +if True: + for _ in iterable(): + x = 2 +else: + x = 3 + +reveal_type(x) # revealed: Literal[1, 2] +``` + +```py path=if_true_for_else.py +def iterable() -> list[()]: ... + +x = 1 + +if True: + for _ in iterable(): + x = 2 + else: + x = 3 +else: + x = 4 + +reveal_type(x) # revealed: Literal[3] +``` + ## If expressions See also: tests in [expression/if.md](expression/if.md). diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index df21cd8c74d46..b1c346b5e145d 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -806,7 +806,7 @@ where ast::Stmt::If(node) => { self.visit_expr(&node.test); let pre_if = self.flow_snapshot(); - let pre_if_constraints = self.branching_conditions_snapshot(); + let pre_if_condutions = self.branching_conditions_snapshot(); let constraint = self.record_expression_constraint(&node.test); let mut constraints = vec![constraint]; self.visit_body(&node.body); @@ -832,7 +832,7 @@ where post_clauses.push(self.flow_snapshot()); // we can only take an elif/else branch if none of the previous ones were // taken, so the block entry state is always `pre_if` - self.flow_restore(pre_if.clone(), pre_if_constraints.clone()); + self.flow_restore(pre_if.clone(), pre_if_condutions.clone()); for constraint in &constraints { self.record_negated_constraint(*constraint); } @@ -843,7 +843,7 @@ where self.visit_body(clause_body); } for post_clause_state in post_clauses { - self.flow_merge(post_clause_state, pre_if_constraints.clone()); + self.flow_merge(post_clause_state, pre_if_condutions.clone()); } } ast::Stmt::While(ast::StmtWhile { @@ -855,7 +855,7 @@ where self.visit_expr(test); let pre_loop = self.flow_snapshot(); - let pre_loop_constraints = self.branching_conditions_snapshot(); + let pre_loop_conditions = self.branching_conditions_snapshot(); let constraint = self.record_expression_constraint(test); // Save aside any break states from an outer loop @@ -875,14 +875,14 @@ where // We may execute the `else` clause without ever executing the body, so merge in // the pre-loop state before visiting `else`. - self.flow_merge(pre_loop, pre_loop_constraints.clone()); + self.flow_merge(pre_loop, pre_loop_conditions.clone()); self.record_negated_constraint(constraint); self.visit_body(orelse); // Breaking out of a while loop bypasses the `else` clause, so merge in the break // states after visiting `else`. for break_state in break_states { - self.flow_merge(break_state, pre_loop_constraints.clone()); // TODO? + self.flow_merge(break_state, pre_loop_conditions.clone()); // TODO? } } ast::Stmt::With(ast::StmtWith { @@ -925,9 +925,11 @@ where self.visit_expr(iter); let pre_loop = self.flow_snapshot(); - let pre_loop_constraints = self.branching_conditions_snapshot(); + let pre_loop_conditions = self.branching_conditions_snapshot(); let saved_break_states = std::mem::take(&mut self.loop_break_states); + self.record_unconditional_branching(); + debug_assert_eq!(&self.current_assignments, &[]); self.push_assignment(for_stmt.into()); self.visit_expr(target); @@ -946,13 +948,13 @@ where // We may execute the `else` clause without ever executing the body, so merge in // the pre-loop state before visiting `else`. - self.flow_merge(pre_loop, pre_loop_constraints.clone()); + self.flow_merge(pre_loop, pre_loop_conditions.clone()); self.visit_body(orelse); // Breaking out of a `for` loop bypasses the `else` clause, so merge in the break // states after visiting `else`. for break_state in break_states { - self.flow_merge(break_state, pre_loop_constraints.clone()); + self.flow_merge(break_state, pre_loop_conditions.clone()); } } ast::Stmt::Match(ast::StmtMatch { @@ -1036,7 +1038,7 @@ where } let pre_except_state = self.flow_snapshot(); - let pre_except_constraints = self.branching_conditions_snapshot(); + let pre_except_conditions = self.branching_conditions_snapshot(); let num_handlers = handlers.len(); self.record_unconditional_branching(); @@ -1079,7 +1081,7 @@ where if i < (num_handlers - 1) { self.flow_restore( pre_except_state.clone(), - pre_except_constraints.clone(), + pre_except_conditions.clone(), ); } } @@ -1249,15 +1251,15 @@ where }) => { self.visit_expr(test); let pre_if = self.flow_snapshot(); - let pre_if_constraints = self.branching_conditions_snapshot(); + let pre_if_conditions = self.branching_conditions_snapshot(); let constraint = self.record_expression_constraint(test); self.visit_expr(body); let post_body = self.flow_snapshot(); - self.flow_restore(pre_if, pre_if_constraints.clone()); + self.flow_restore(pre_if, pre_if_conditions.clone()); self.record_negated_constraint(constraint); self.visit_expr(orelse); - self.flow_merge(post_body, pre_if_constraints); + self.flow_merge(post_body, pre_if_conditions); } ast::Expr::ListComp( list_comprehension @ ast::ExprListComp { @@ -1315,7 +1317,7 @@ where op, }) => { let mut snapshots = vec![]; - let pre_op_constraints = self.branching_conditions_snapshot(); + let pre_op_conditions = self.branching_conditions_snapshot(); for (index, value) in values.iter().enumerate() { self.visit_expr(value); // In the last value we don't need to take a snapshot nor add a constraint @@ -1330,7 +1332,7 @@ where } } for snapshot in snapshots { - self.flow_merge(snapshot, pre_op_constraints.clone()); + self.flow_merge(snapshot, pre_op_conditions.clone()); } } _ => { From 88646aac2895d011d793be42b3395e296ee71f5f Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 15:39:31 +0100 Subject: [PATCH 19/68] Renaming --- .../src/semantic_index/branching.rs | 4 ++-- .../src/semantic_index/builder.rs | 6 +++--- .../src/semantic_index/use_def.rs | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/branching.rs b/crates/red_knot_python_semantic/src/semantic_index/branching.rs index 58e70a49b0138..d2e322be28dcb 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/branching.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/branching.rs @@ -2,6 +2,6 @@ use super::constraint::Constraint; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum BranchingCondition<'db> { - Conditional(Constraint<'db>), - Unconditional, + ConditionalOn(Constraint<'db>), + Ambiguous, } diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index b1c346b5e145d..1ed43dc0f4202 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -303,7 +303,7 @@ impl<'db> SemanticIndexBuilder<'db> { fn record_unconditional_branching(&mut self) { self.current_use_def_map_mut() - .record_unconditional_branching(); + .record_unconstrained_branch_point(); } fn build_constraint(&mut self, constraint_node: &Expr) -> Constraint<'db> { @@ -1028,7 +1028,7 @@ where // as there necessarily must have been 0 `except` blocks executed // if we hit the `else` block. let post_try_block_state = self.flow_snapshot(); - let post_try_block_constraints = self.branching_conditions_snapshot(); + let post_try_block_conditions = self.branching_conditions_snapshot(); // Prepare for visiting the `except` block(s) self.flow_restore(pre_try_block_state, pre_try_block_conditions.clone()); @@ -1088,7 +1088,7 @@ where // If we get to the `else` block, we know that 0 of the `except` blocks can have been executed, // and the entire `try` block must have been executed: - self.flow_restore(post_try_block_state, post_try_block_constraints); + self.flow_restore(post_try_block_state, post_try_block_conditions); } self.visit_body(orelse); diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 018d238ee752a..7ab12f5a8fcff 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -493,7 +493,7 @@ impl<'db> BranchingConditionsIterator<'_, 'db> { for condition in self { let truthiness = match condition { - BranchingCondition::Conditional(Constraint { + BranchingCondition::ConditionalOn(Constraint { node: ConstraintNode::Expression(test_expr), is_positive, }) => { @@ -504,11 +504,11 @@ impl<'db> BranchingConditionsIterator<'_, 'db> { ty.bool(db).negate_if(!is_positive) } - BranchingCondition::Conditional(Constraint { + BranchingCondition::ConditionalOn(Constraint { node: ConstraintNode::Pattern(..), .. }) => Truthiness::Ambiguous, - BranchingCondition::Unconditional => Truthiness::Ambiguous, + BranchingCondition::Ambiguous => Truthiness::Ambiguous, }; result.any_always_false |= truthiness.is_always_false(); @@ -608,11 +608,11 @@ impl<'db> UseDefMapBuilder<'db> { state.record_constraint(constraint_id); } - self.record_branching_condition(BranchingCondition::Conditional(constraint)); + self.record_branching_condition(BranchingCondition::ConditionalOn(constraint)); } - pub(super) fn record_unconditional_branching(&mut self) { - self.record_branching_condition(BranchingCondition::Unconditional); + pub(super) fn record_unconstrained_branch_point(&mut self) { + self.record_branching_condition(BranchingCondition::Ambiguous); } pub(super) fn record_branching_condition(&mut self, condition: BranchingCondition<'db>) { From c56a50c64c6ee7263af94def943277ff8b2e06d6 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 15:50:20 +0100 Subject: [PATCH 20/68] Clippy suggestions --- crates/red_knot_python_semantic/src/semantic_index/use_def.rs | 2 +- crates/red_knot_python_semantic/src/types.rs | 2 +- crates/red_knot_python_semantic/src/types/infer.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 7ab12f5a8fcff..47fd21f09e076 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -337,7 +337,7 @@ impl<'db> UseDefMap<'db> { db: &dyn crate::db::Db, symbol: ScopedSymbolId, ) -> Option { - self.compute_boundness(db, &self.public_symbols[symbol].bindings()) + self.compute_boundness(db, self.public_symbols[symbol].bindings()) } pub(crate) fn bindings_at_declaration( diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 3befa12b872b5..f0a0cdb2c3374 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -303,7 +303,7 @@ fn bindings_ty<'db>( // TODO: get rid of all the collects and clean up, obviously let def_types: Vec<_> = def_types.collect(); - if !def_types.is_empty() && def_types.iter().all(|(ty, _)| *ty == None) { + if !def_types.is_empty() && def_types.iter().all(|(ty, _)| ty.is_none()) { return Some(Type::Unknown); } diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index 34064a94a6a9e..3b82fdd3f4eb3 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -3011,7 +3011,7 @@ impl<'db> TypeInferenceBuilder<'db> { use_def.use_boundness(self.db, use_id), ) }; - if boundness == Some(Boundness::PossiblyUnbound) || boundness == None { + if boundness == Some(Boundness::PossiblyUnbound) || boundness.is_none() { match self.lookup_name(name) { Symbol::Type(looked_up_ty, looked_up_boundness) => { if looked_up_boundness == Boundness::PossiblyUnbound { From c9956bcde3bd2b1c1dba3cf31b2983458763cff2 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 20:08:11 +0100 Subject: [PATCH 21/68] Fix typo --- .../red_knot_python_semantic/src/semantic_index/builder.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index 1ed43dc0f4202..2bb761624c9c1 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -806,7 +806,7 @@ where ast::Stmt::If(node) => { self.visit_expr(&node.test); let pre_if = self.flow_snapshot(); - let pre_if_condutions = self.branching_conditions_snapshot(); + let pre_if_conditions = self.branching_conditions_snapshot(); let constraint = self.record_expression_constraint(&node.test); let mut constraints = vec![constraint]; self.visit_body(&node.body); @@ -832,7 +832,7 @@ where post_clauses.push(self.flow_snapshot()); // we can only take an elif/else branch if none of the previous ones were // taken, so the block entry state is always `pre_if` - self.flow_restore(pre_if.clone(), pre_if_condutions.clone()); + self.flow_restore(pre_if.clone(), pre_if_conditions.clone()); for constraint in &constraints { self.record_negated_constraint(*constraint); } @@ -843,7 +843,7 @@ where self.visit_body(clause_body); } for post_clause_state in post_clauses { - self.flow_merge(post_clause_state, pre_if_condutions.clone()); + self.flow_merge(post_clause_state, pre_if_conditions.clone()); } } ast::Stmt::While(ast::StmtWhile { @@ -1034,7 +1034,6 @@ where self.flow_restore(pre_try_block_state, pre_try_block_conditions.clone()); for state in try_block_snapshots { self.flow_merge(state, pre_try_block_conditions.clone()); - // TODO? } let pre_except_state = self.flow_snapshot(); From 46209e558fc8c147d8e5fb6ef6d032c556f04b45 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 20:08:35 +0100 Subject: [PATCH 22/68] Missing word --- .../resources/mdtest/statically-known-branches.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 80ba34e8262d3..00a7d433be41d 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -557,7 +557,8 @@ x ### Ambiguous, possibly unbound -For comparison, we still definitions inside non-statically known branches as possibly unbound: +For comparison, we still detect definitions inside non-statically known branches as possibly +unbound: ```py def flag() -> bool: ... From 0762bf1353aa84735ebc1e6dd0353bfb06e181ea Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 9 Dec 2024 20:31:28 +0100 Subject: [PATCH 23/68] Some cleanup and performance work --- .../src/semantic_index/use_def/bitset.rs | 43 ++++++ .../semantic_index/use_def/symbol_state.rs | 24 ++-- crates/red_knot_python_semantic/src/types.rs | 123 ++++++++---------- 3 files changed, 109 insertions(+), 81 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs index 84ac7305d8b0f..b79c374bf8697 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs @@ -120,6 +120,15 @@ impl BitSet { current_block: blocks[0], } } + + pub(super) fn iter_rev(&self) -> ReverseBitSetIterator<'_, B> { + let blocks = self.blocks(); + ReverseBitSetIterator { + blocks, + current_block_index: self.blocks().len() - 1, + current_block: blocks[self.blocks().len() - 1], + } + } } /// Iterator over values in a [`BitSet`]. @@ -159,10 +168,44 @@ impl Iterator for BitSetIterator<'_, B> { impl std::iter::FusedIterator for BitSetIterator<'_, B> {} +/// Iterator over values in a [`BitSet`]. +#[derive(Debug, Clone)] +pub(super) struct ReverseBitSetIterator<'a, const B: usize> { + /// The blocks we are iterating over. + blocks: &'a [u64], + + /// The index of the block we are currently iterating through. + current_block_index: usize, + + /// The block we are currently iterating through (and zeroing as we go.) + current_block: u64, +} + +impl Iterator for ReverseBitSetIterator<'_, B> { + type Item = u32; + + fn next(&mut self) -> Option { + while self.current_block == 0 { + if self.current_block_index == 0 { + return None; + } + self.current_block_index -= 1; + self.current_block = self.blocks[self.current_block_index]; + } + + let highest_bit_set = 63 - self.current_block.leading_zeros() as u64; + // TODO: efficiency, safety comment, etc. + self.current_block &= !(1u64 << highest_bit_set); + #[allow(clippy::cast_possible_truncation)] + Some(highest_bit_set as u32 + (64 * self.current_block_index) as u32) + } +} + #[cfg(test)] mod tests { use super::BitSet; + #[track_caller] fn assert_bitset(bitset: &BitSet, contents: &[u32]) { assert_eq!(bitset.iter().collect::>(), contents); } diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 51f4b0cd0cf6f..6d8038100ce5c 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -43,6 +43,8 @@ //! //! Tracking live declarations is simpler, since constraints are not involved, but otherwise very //! similar to tracking live bindings. +use crate::semantic_index::use_def::bitset::ReverseBitSetIterator; + use super::bitset::{BitSet, BitSetIterator}; use ruff_index::newtype_index; use smallvec::SmallVec; @@ -64,14 +66,14 @@ const INLINE_BINDING_BLOCKS: usize = 3; /// A [`BitSet`] of [`ScopedDefinitionId`], representing live bindings of a symbol in a scope. type Bindings = BitSet; -type BindingsIterator<'a> = BitSetIterator<'a, INLINE_BINDING_BLOCKS>; +type BindingsIterator<'a> = ReverseBitSetIterator<'a, INLINE_BINDING_BLOCKS>; /// Can reference this * 64 total declarations inline; more will fall back to the heap. const INLINE_DECLARATION_BLOCKS: usize = 3; /// A [`BitSet`] of [`ScopedDefinitionId`], representing live declarations of a symbol in a scope. type Declarations = BitSet; -type DeclarationsIterator<'a> = BitSetIterator<'a, INLINE_DECLARATION_BLOCKS>; +type DeclarationsIterator<'a> = ReverseBitSetIterator<'a, INLINE_DECLARATION_BLOCKS>; /// Can reference this * 64 total constraints inline; more will fall back to the heap. const INLINE_CONSTRAINT_BLOCKS: usize = 2; @@ -137,8 +139,8 @@ impl SymbolDeclarations { /// Return an iterator over live declarations for this symbol. pub(super) fn iter(&self) -> DeclarationIdIterator { DeclarationIdIterator { - inner: self.live_declarations.iter(), - branching_conditions: self.branching_conditions.iter(), + inner: self.live_declarations.iter_rev(), + branching_conditions: self.branching_conditions.iter().rev(), } } @@ -215,9 +217,9 @@ impl SymbolBindings { /// Iterate over currently live bindings for this symbol. pub(super) fn iter(&self) -> BindingIdWithConstraintsIterator { BindingIdWithConstraintsIterator { - definitions: self.live_bindings.iter(), - constraints: self.constraints.iter(), - branching_conditions: self.branching_conditions.iter(), + definitions: self.live_bindings.iter_rev(), + constraints: self.constraints.iter().rev(), + branching_conditions: self.branching_conditions.iter().rev(), } } @@ -507,8 +509,8 @@ pub(super) struct BindingIdWithConstraints<'a> { #[derive(Debug, Clone)] pub(super) struct BindingIdWithConstraintsIterator<'a> { definitions: BindingsIterator<'a>, - constraints: ConstraintsIterator<'a>, - branching_conditions: BranchingConditionsIterator<'a>, + constraints: std::iter::Rev>, + branching_conditions: std::iter::Rev>, } impl<'a> Iterator for BindingIdWithConstraintsIterator<'a> { @@ -538,8 +540,6 @@ impl<'a> Iterator for BindingIdWithConstraintsIterator<'a> { } } -impl std::iter::FusedIterator for BindingIdWithConstraintsIterator<'_> {} - #[derive(Debug, Clone)] pub(super) struct ConstraintIdIterator<'a> { wrapped: BitSetIterator<'a, INLINE_CONSTRAINT_BLOCKS>, @@ -575,7 +575,7 @@ impl std::iter::FusedIterator for BranchingConditionIdIterator<'_> {} #[derive(Debug)] pub(super) struct DeclarationIdIterator<'a> { inner: DeclarationsIterator<'a>, - branching_conditions: BranchingConditionsIterator<'a>, + branching_conditions: std::iter::Rev>, } impl<'a> Iterator for DeclarationIdIterator<'a> { diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index f0a0cdb2c3374..8375c38d10cbf 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -261,63 +261,56 @@ fn bindings_ty<'db>( db: &'db dyn Db, bindings_with_constraints: BindingWithConstraintsIterator<'_, 'db>, ) -> Option> { - let def_types = bindings_with_constraints.map( - |BindingWithConstraints { - binding, - constraints, - branching_conditions, - }| { - let result = branching_conditions.branch_condition_truthiness(db); - - if result.any_always_false { - // TODO: do we need to call binding_ty(…) even if we don't need the result? - (None, UnconditionallyVisible::No) - } else { - let unconditionally_visible = - if result.at_least_one_condition && result.all_always_true { - UnconditionallyVisible::Yes - } else { - UnconditionallyVisible::No - }; - - let mut constraint_tys = constraints - .filter_map(|constraint| narrowing_constraint(db, constraint, binding)) - .peekable(); - - let binding_ty = binding_ty(db, binding); - if constraint_tys.peek().is_some() { - let intersection_ty = constraint_tys - .fold( - IntersectionBuilder::new(db).add_positive(binding_ty), - IntersectionBuilder::add_positive, - ) - .build(); - (Some(intersection_ty), unconditionally_visible) + let def_types = bindings_with_constraints + .map( + |BindingWithConstraints { + binding, + constraints, + branching_conditions, + }| { + let result = branching_conditions.branch_condition_truthiness(db); + + if result.any_always_false { + // TODO: do we need to call binding_ty(…) even if we don't need the result? + (None, UnconditionallyVisible::No) } else { - (Some(binding_ty), unconditionally_visible) + let unconditionally_visible = + if result.at_least_one_condition && result.all_always_true { + UnconditionallyVisible::Yes + } else { + UnconditionallyVisible::No + }; + + let mut constraint_tys = constraints + .filter_map(|constraint| narrowing_constraint(db, constraint, binding)) + .peekable(); + + let binding_ty = binding_ty(db, binding); + if constraint_tys.peek().is_some() { + let intersection_ty = constraint_tys + .fold( + IntersectionBuilder::new(db).add_positive(binding_ty), + IntersectionBuilder::add_positive, + ) + .build(); + (Some(intersection_ty), unconditionally_visible) + } else { + (Some(binding_ty), unconditionally_visible) + } } - } - }, - ); + }, + ) + .take_while_inclusive(|(_, uv)| *uv != UnconditionallyVisible::Yes) + .map(|(ty, _)| ty); // TODO: get rid of all the collects and clean up, obviously let def_types: Vec<_> = def_types.collect(); - if !def_types.is_empty() && def_types.iter().all(|(ty, _)| ty.is_none()) { + if !def_types.is_empty() && def_types.iter().all(|ty| ty.is_none()) { return Some(Type::Unknown); } - // shrink the vector to only include everything from the last unconditionally visible binding - let def_types: Vec<_> = def_types - .iter() - .rev() - .take_while_inclusive(|(_, unconditionally_visible)| { - *unconditionally_visible != UnconditionallyVisible::Yes - }) - .map(|(ty, _)| ty.unwrap_or(Type::Never)) - .collect(); - - let mut def_types = def_types.into_iter().rev(); + let mut def_types = def_types.iter().map(|ty| ty.unwrap_or(Type::Never)).rev(); if let Some(first) = def_types.next() { if let Some(second) = def_types.next() { @@ -354,33 +347,25 @@ fn declarations_ty<'db>( declarations: DeclarationsIterator<'_, 'db>, undeclared_ty: Option>, ) -> DeclaredTypeResult<'db> { - let decl_types = declarations.map(|(declaration, branching_conditions)| { - let result = branching_conditions.branch_condition_truthiness(db); + let decl_types = declarations + .map(|(declaration, branching_conditions)| { + let result = branching_conditions.branch_condition_truthiness(db); - if result.any_always_false { - (Type::Never, UnconditionallyVisible::No) - } else { - if result.at_least_one_condition && result.all_always_true { - (declaration_ty(db, declaration), UnconditionallyVisible::Yes) + if result.any_always_false { + (Type::Never, UnconditionallyVisible::No) } else { - (declaration_ty(db, declaration), UnconditionallyVisible::No) + if result.at_least_one_condition && result.all_always_true { + (declaration_ty(db, declaration), UnconditionallyVisible::Yes) + } else { + (declaration_ty(db, declaration), UnconditionallyVisible::No) + } } - } - }); + }) + .take_while_inclusive(|(_, uv)| *uv != UnconditionallyVisible::Yes) + .map(|(ty, _)| ty); - // TODO: get rid of all the collects and clean up, obviously let decl_types: Vec<_> = decl_types.collect(); - // shrink the vector to only include everything from the last unconditionally visible binding - let decl_types: Vec<_> = decl_types - .iter() - .rev() - .take_while_inclusive(|(_, unconditionally_visible)| { - *unconditionally_visible != UnconditionallyVisible::Yes - }) - .map(|(ty, _)| *ty) - .collect(); - let decl_types = decl_types.into_iter().rev(); let mut all_types = undeclared_ty.into_iter().chain(decl_types); From 0336d4b57602844aa33e4ca31fde6d6fb7d2f90b Mon Sep 17 00:00:00 2001 From: David Peter Date: Tue, 10 Dec 2024 09:35:12 +0100 Subject: [PATCH 24/68] Simplify named-expression tests --- .../resources/mdtest/statically-known-branches.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 00a7d433be41d..03693062fca83 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -313,7 +313,7 @@ reveal_type(x) # revealed: Literal[2] ### Always true, `or` ```py -(x := 1) == 1 or (x := 2) +(x := 1) or (x := 2) reveal_type(x) # revealed: Literal[1] ``` @@ -321,7 +321,7 @@ reveal_type(x) # revealed: Literal[1] ### Always true, `and` ```py -(x := 1) == 1 and (x := 2) +(x := 1) and (x := 2) reveal_type(x) # revealed: Literal[2] ``` @@ -329,7 +329,7 @@ reveal_type(x) # revealed: Literal[2] ### Always false, `or` ```py -(x := 1) == 0 or (x := 2) +(x := 0) or (x := 2) reveal_type(x) # revealed: Literal[2] ``` @@ -337,9 +337,9 @@ reveal_type(x) # revealed: Literal[2] ### Always false, `and` ```py -(x := 1) == 0 and (x := 2) +(x := 0) and (x := 2) -reveal_type(x) # revealed: Literal[1] +reveal_type(x) # revealed: Literal[0] ``` ## While loops From 5db680ec136ac1dc16cbeffadc4f0758b74d1ad3 Mon Sep 17 00:00:00 2001 From: David Peter Date: Tue, 10 Dec 2024 11:46:53 +0100 Subject: [PATCH 25/68] Fix behavior of staticallyk-known undeclared symbols --- .../resources/mdtest/annotations/never.md | 2 +- .../mdtest/statically-known-branches.md | 26 ++++++ .../src/semantic_index/use_def.rs | 62 +++++++++----- .../semantic_index/use_def/symbol_state.rs | 8 +- crates/red_knot_python_semantic/src/types.rs | 81 ++++++++----------- .../src/types/infer.rs | 2 +- 6 files changed, 109 insertions(+), 72 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md index 49da0fae2a7ad..1a5c5dd000564 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md +++ b/crates/red_knot_python_semantic/resources/mdtest/annotations/never.md @@ -70,6 +70,6 @@ python-version = "3.10" ``` ```py -# TODO: should raise a diagnostic, see https://github.com/astral-sh/ruff/issues/14297 +# error: [unresolved-import] from typing import Never ``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 03693062fca83..d00981d0e7617 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -682,3 +682,29 @@ if flag(): # error: [possibly-unbound-import] from module import symbol ``` + +#### Always false, undeclared + +```py path=module.py +if False: + symbol: int +``` + +```py +# error: [unresolved-import] +from module import symbol + +reveal_type(symbol) # revealed: Unknown +``` + +#### Always true, declared + +```py path=module.py +if True: + symbol: int +``` + +```py +# no error +from module import symbol +``` diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 47fd21f09e076..4734a432e3684 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -285,17 +285,20 @@ impl<'db> UseDefMap<'db> { self.bindings_iterator(&self.bindings_by_use[use_id]) } - fn compute_boundness( + fn compute_boundness<'map, C>( &self, db: &dyn crate::db::Db, - bindings: &SymbolBindings, - ) -> Option { - let bindings_iter = self.bindings_iterator(bindings); - + conditions: C, + may_be_undefined: bool, + ) -> Option + where + 'db: 'map, + C: Iterator>, + { let mut definitely_bound = false; let mut definitely_unbound = true; - for binding in bindings_iter { - let result = binding.branching_conditions.branch_condition_truthiness(db); + for condition in conditions { + let result = condition.truthiness(db); if !result.any_always_false { definitely_unbound = false; @@ -309,7 +312,7 @@ impl<'db> UseDefMap<'db> { if definitely_unbound { None } else { - if definitely_bound || !bindings.may_be_unbound() { + if definitely_bound || !may_be_undefined { Some(Boundness::Bound) } else { Some(Boundness::PossiblyUnbound) @@ -322,7 +325,11 @@ impl<'db> UseDefMap<'db> { db: &dyn crate::db::Db, use_id: ScopedUseId, ) -> Option { - self.compute_boundness(db, &self.bindings_by_use[use_id]) + let bindings = &self.bindings_by_use[use_id]; + let conditions = self + .bindings_iterator(bindings) + .map(|binding| binding.branching_conditions); + self.compute_boundness(db, conditions, bindings.may_be_unbound()) } pub(crate) fn public_bindings( @@ -337,7 +344,11 @@ impl<'db> UseDefMap<'db> { db: &dyn crate::db::Db, symbol: ScopedSymbolId, ) -> Option { - self.compute_boundness(db, self.public_symbols[symbol].bindings()) + let bindings = self.public_symbols[symbol].bindings(); + let conditions = self + .bindings_iterator(bindings) + .map(|binding| binding.branching_conditions); + self.compute_boundness(db, conditions, bindings.may_be_unbound()) } pub(crate) fn bindings_at_declaration( @@ -373,8 +384,26 @@ impl<'db> UseDefMap<'db> { self.declarations_iterator(declarations) } - pub(crate) fn has_public_declarations(&self, symbol: ScopedSymbolId) -> bool { - !self.public_symbols[symbol].declarations().is_empty() + pub(crate) fn declaredness<'map>( + &self, + db: &dyn crate::db::Db, + declarations: DeclarationsIterator<'map, 'db>, + ) -> Option { + let may_be_undeclared = declarations.may_be_undeclared; + let conditions = declarations.map(|(_, conditions)| conditions); + self.compute_boundness(db, conditions, may_be_undeclared) + } + + pub(crate) fn may_be_undeclared<'map>( + &self, + db: &dyn crate::db::Db, + declarations: DeclarationsIterator<'map, 'db>, + ) -> bool { + match self.declaredness(db, declarations) { + Some(Boundness::Bound) => false, + Some(Boundness::PossiblyUnbound) => true, + None => true, + } } fn bindings_iterator<'a>( @@ -484,7 +513,7 @@ pub(crate) struct BranchConditionTruthiness { } impl<'db> BranchingConditionsIterator<'_, 'db> { - pub(crate) fn branch_condition_truthiness(self, db: &'db dyn Db) -> BranchConditionTruthiness { + pub(crate) fn truthiness(self, db: &'db dyn Db) -> BranchConditionTruthiness { let mut result = BranchConditionTruthiness { any_always_false: false, all_always_true: true, @@ -522,6 +551,7 @@ impl<'db> BranchingConditionsIterator<'_, 'db> { impl std::iter::FusedIterator for BranchingConditionsIterator<'_, '_> {} +#[derive(Clone)] pub(crate) struct DeclarationsIterator<'map, 'db> { all_definitions: &'map IndexVec>, all_branching_conditions: &'map IndexVec>, @@ -529,12 +559,6 @@ pub(crate) struct DeclarationsIterator<'map, 'db> { may_be_undeclared: bool, } -impl DeclarationsIterator<'_, '_> { - pub(crate) fn may_be_undeclared(&self) -> bool { - self.may_be_undeclared - } -} - impl<'map, 'db> Iterator for DeclarationsIterator<'map, 'db> { type Item = (Definition<'db>, BranchingConditionsIterator<'map, 'db>); diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 6d8038100ce5c..f45a36d05c515 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -99,7 +99,7 @@ pub(super) struct SymbolDeclarations { /// [`BitSet`]: which declarations (as [`ScopedDefinitionId`]) can reach the current location? live_declarations: Declarations, - branching_conditions: Constraints, + branching_conditions: BranchingConditionsPerBinding, /// Could the symbol be un-declared at this point? may_be_undeclared: bool, @@ -109,7 +109,7 @@ impl SymbolDeclarations { fn undeclared() -> Self { Self { live_declarations: Declarations::default(), - branching_conditions: Constraints::default(), + branching_conditions: BranchingConditionsPerBinding::default(), may_be_undeclared: true, } } @@ -124,7 +124,7 @@ impl SymbolDeclarations { self.may_be_undeclared = false; // TODO: unify code with below - self.branching_conditions = Constraints::with_capacity(1); + self.branching_conditions = BranchingConditionsPerBinding::with_capacity(1); self.branching_conditions.push(BitSet::default()); for active_constraint_id in branching_conditions.iter() { self.branching_conditions[0].insert(active_constraint_id); @@ -572,7 +572,7 @@ impl Iterator for BranchingConditionIdIterator<'_> { impl std::iter::FusedIterator for BranchingConditionIdIterator<'_> {} -#[derive(Debug)] +#[derive(Debug, Clone)] pub(super) struct DeclarationIdIterator<'a> { inner: DeclarationsIterator<'a>, branching_conditions: std::iter::Rev>, diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 8375c38d10cbf..a396520b52850 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -17,7 +17,7 @@ pub(crate) use self::signatures::Signature; use crate::module_resolver::file_to_module; use crate::semantic_index::ast_ids::HasScopedExpressionId; use crate::semantic_index::definition::Definition; -use crate::semantic_index::symbol::{self as symbol, ScopeId, ScopedSymbolId}; +use crate::semantic_index::symbol::{self as symbol, FileScopeId, ScopeId, ScopedSymbolId}; use crate::semantic_index::{ global_scope, semantic_index, symbol_table, use_def_map, BindingWithConstraints, BindingWithConstraintsIterator, DeclarationsIterator, @@ -71,10 +71,22 @@ fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolI // If the symbol is declared, the public type is based on declarations; otherwise, it's based // on inference from bindings. - if use_def.has_public_declarations(symbol) { - let declarations = use_def.public_declarations(symbol); - // If the symbol is undeclared in some paths, include the inferred type in the public type. - let undeclared_ty = if declarations.may_be_undeclared() { + let declaredness = use_def.declaredness(db, use_def.public_declarations(symbol)); + + let undeclared_ty = match declaredness { + None => { + return bindings_ty(db, use_def.public_bindings(symbol)) + .map(|bindings_ty| { + if let Some(boundness) = use_def.public_boundness(db, symbol) { + Symbol::Type(bindings_ty, boundness) + } else { + Symbol::Unbound + } + }) + .unwrap_or(Symbol::Unbound); + } + Some(Boundness::PossiblyUnbound) => { + // If the symbol is undeclared in some paths, include the inferred type in the public type. Some( bindings_ty(db, use_def.public_bindings(symbol)) .map(|bindings_ty| { @@ -86,47 +98,22 @@ fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolI }) .unwrap_or(Symbol::Unbound), ) - } else { - None - }; - // Intentionally ignore conflicting declared types; that's not our problem, it's the - // problem of the module we are importing from. - - // TODO: Our handling of boundness currently only depends on bindings, and ignores - // declarations. This is inconsistent, since we only look at bindings if the symbol - // may be undeclared. Consider the following example: - // ```py - // x: int - // - // if flag: - // y: int - // else - // y = 3 - // ``` - // If we import from this module, we will currently report `x` as a definitely-bound - // symbol (even though it has no bindings at all!) but report `y` as possibly-unbound - // (even though every path has either a binding or a declaration for it.) - - match undeclared_ty { - Some(Symbol::Type(ty, boundness)) => Symbol::Type( - declarations_ty(db, declarations, Some(ty)).unwrap_or_else(|(ty, _)| ty), - boundness, - ), - None | Some(Symbol::Unbound) => Symbol::Type( - declarations_ty(db, declarations, None).unwrap_or_else(|(ty, _)| ty), - Boundness::Bound, - ), } - } else { - bindings_ty(db, use_def.public_bindings(symbol)) - .map(|bindings_ty| { - if let Some(boundness) = use_def.public_boundness(db, symbol) { - Symbol::Type(bindings_ty, boundness) - } else { - Symbol::Unbound - } - }) - .unwrap_or(Symbol::Unbound) + Some(Boundness::Bound) => None, + }; + + // Intentionally ignore conflicting declared types; that's not our problem, it's the + // problem of the module we are importing from. + let declarations = use_def.public_declarations(symbol); + match undeclared_ty { + Some(Symbol::Type(ty, boundness)) => Symbol::Type( + declarations_ty(db, declarations, Some(ty)).unwrap_or_else(|(ty, _)| ty), + boundness, + ), + None | Some(Symbol::Unbound) => Symbol::Type( + declarations_ty(db, declarations, None).unwrap_or_else(|(ty, _)| ty), + Boundness::Bound, + ), } } @@ -268,7 +255,7 @@ fn bindings_ty<'db>( constraints, branching_conditions, }| { - let result = branching_conditions.branch_condition_truthiness(db); + let result = branching_conditions.truthiness(db); if result.any_always_false { // TODO: do we need to call binding_ty(…) even if we don't need the result? @@ -349,7 +336,7 @@ fn declarations_ty<'db>( ) -> DeclaredTypeResult<'db> { let decl_types = declarations .map(|(declaration, branching_conditions)| { - let result = branching_conditions.branch_condition_truthiness(db); + let result = branching_conditions.truthiness(db); if result.any_always_false { (Type::Never, UnconditionallyVisible::No) diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index 3b82fdd3f4eb3..5142e9d5113b3 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -783,7 +783,7 @@ impl<'db> TypeInferenceBuilder<'db> { debug_assert!(binding.is_binding(self.db)); let use_def = self.index.use_def_map(binding.file_scope(self.db)); let declarations = use_def.declarations_at_binding(binding); - let undeclared_ty = if declarations.may_be_undeclared() { + let undeclared_ty = if use_def.may_be_undeclared(self.db, declarations.clone()) { Some(Type::Unknown) } else { None From d590a49846f8a2ebbce5d9bcebf1fa49d8c0cb99 Mon Sep 17 00:00:00 2001 From: David Peter Date: Tue, 10 Dec 2024 12:06:08 +0100 Subject: [PATCH 26/68] Clippy suggestions --- .../src/semantic_index/use_def.rs | 111 ++++++++---------- .../src/semantic_index/use_def/bitset.rs | 13 +- .../semantic_index/use_def/symbol_state.rs | 4 - crates/red_knot_python_semantic/src/types.rs | 6 +- .../src/types/infer.rs | 2 +- 5 files changed, 57 insertions(+), 79 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 4734a432e3684..cc5b4e7ada674 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -244,6 +244,40 @@ use super::constraint::Constraint; mod bitset; mod symbol_state; +fn compute_boundness<'db, 'map, C>( + db: &dyn crate::db::Db, + conditions: C, + may_be_undefined: bool, +) -> Option +where + 'db: 'map, + C: Iterator>, +{ + let mut definitely_bound = false; + let mut definitely_unbound = true; + for condition in conditions { + let result = condition.truthiness(db); + + if !result.any_always_false { + definitely_unbound = false; + } + + if result.all_always_true { + definitely_bound = true; + } + } + + if definitely_unbound { + None + } else { + if definitely_bound || !may_be_undefined { + Some(Boundness::Bound) + } else { + Some(Boundness::PossiblyUnbound) + } + } +} + /// Applicable definitions and constraints for every use of a name. #[derive(Debug, PartialEq, Eq)] pub(crate) struct UseDefMap<'db> { @@ -285,41 +319,6 @@ impl<'db> UseDefMap<'db> { self.bindings_iterator(&self.bindings_by_use[use_id]) } - fn compute_boundness<'map, C>( - &self, - db: &dyn crate::db::Db, - conditions: C, - may_be_undefined: bool, - ) -> Option - where - 'db: 'map, - C: Iterator>, - { - let mut definitely_bound = false; - let mut definitely_unbound = true; - for condition in conditions { - let result = condition.truthiness(db); - - if !result.any_always_false { - definitely_unbound = false; - } - - if result.all_always_true { - definitely_bound = true; - } - } - - if definitely_unbound { - None - } else { - if definitely_bound || !may_be_undefined { - Some(Boundness::Bound) - } else { - Some(Boundness::PossiblyUnbound) - } - } - } - pub(crate) fn use_boundness( &self, db: &dyn crate::db::Db, @@ -329,7 +328,7 @@ impl<'db> UseDefMap<'db> { let conditions = self .bindings_iterator(bindings) .map(|binding| binding.branching_conditions); - self.compute_boundness(db, conditions, bindings.may_be_unbound()) + compute_boundness(db, conditions, bindings.may_be_unbound()) } pub(crate) fn public_bindings( @@ -348,7 +347,7 @@ impl<'db> UseDefMap<'db> { let conditions = self .bindings_iterator(bindings) .map(|binding| binding.branching_conditions); - self.compute_boundness(db, conditions, bindings.may_be_unbound()) + compute_boundness(db, conditions, bindings.may_be_unbound()) } pub(crate) fn bindings_at_declaration( @@ -384,28 +383,6 @@ impl<'db> UseDefMap<'db> { self.declarations_iterator(declarations) } - pub(crate) fn declaredness<'map>( - &self, - db: &dyn crate::db::Db, - declarations: DeclarationsIterator<'map, 'db>, - ) -> Option { - let may_be_undeclared = declarations.may_be_undeclared; - let conditions = declarations.map(|(_, conditions)| conditions); - self.compute_boundness(db, conditions, may_be_undeclared) - } - - pub(crate) fn may_be_undeclared<'map>( - &self, - db: &dyn crate::db::Db, - declarations: DeclarationsIterator<'map, 'db>, - ) -> bool { - match self.declaredness(db, declarations) { - Some(Boundness::Bound) => false, - Some(Boundness::PossiblyUnbound) => true, - None => true, - } - } - fn bindings_iterator<'a>( &'a self, bindings: &'a SymbolBindings, @@ -559,6 +536,22 @@ pub(crate) struct DeclarationsIterator<'map, 'db> { may_be_undeclared: bool, } +impl DeclarationsIterator<'_, '_> { + pub(crate) fn declaredness(self, db: &dyn crate::db::Db) -> Option { + let may_be_undeclared = self.may_be_undeclared; + let conditions = self.map(|(_, conditions)| conditions); + compute_boundness(db, conditions, may_be_undeclared) + } + + pub(crate) fn may_be_undeclared(self, db: &dyn crate::db::Db) -> bool { + match self.declaredness(db) { + Some(Boundness::Bound) => false, + Some(Boundness::PossiblyUnbound) => true, + None => true, + } + } +} + impl<'map, 'db> Iterator for DeclarationsIterator<'map, 'db> { type Item = (Definition<'db>, BranchingConditionsIterator<'map, 'db>); diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs index b79c374bf8697..09f74f8f72f94 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs @@ -32,10 +32,6 @@ impl BitSet { bitset } - pub(super) fn is_empty(&self) -> bool { - self.blocks().iter().all(|&b| b == 0) - } - /// Convert from Inline to Heap, if needed, and resize the Heap vector, if needed. fn resize(&mut self, value: u32) { let num_blocks_needed = (value / 64) + 1; @@ -193,7 +189,7 @@ impl Iterator for ReverseBitSetIterator<'_, B> { self.current_block = self.blocks[self.current_block_index]; } - let highest_bit_set = 63 - self.current_block.leading_zeros() as u64; + let highest_bit_set = 63 - u64::from(self.current_block.leading_zeros()); // TODO: efficiency, safety comment, etc. self.current_block &= !(1u64 << highest_bit_set); #[allow(clippy::cast_possible_truncation)] @@ -343,11 +339,4 @@ mod tests { assert!(matches!(b, BitSet::Inline(_))); assert_bitset(&b, &[45, 120]); } - - #[test] - fn empty() { - let b = BitSet::<1>::default(); - - assert!(b.is_empty()); - } } diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index f45a36d05c515..fcca269fc13a4 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -144,10 +144,6 @@ impl SymbolDeclarations { } } - pub(super) fn is_empty(&self) -> bool { - self.live_declarations.is_empty() - } - pub(super) fn may_be_undeclared(&self) -> bool { self.may_be_undeclared } diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index a396520b52850..24c497e98d5ff 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -17,7 +17,7 @@ pub(crate) use self::signatures::Signature; use crate::module_resolver::file_to_module; use crate::semantic_index::ast_ids::HasScopedExpressionId; use crate::semantic_index::definition::Definition; -use crate::semantic_index::symbol::{self as symbol, FileScopeId, ScopeId, ScopedSymbolId}; +use crate::semantic_index::symbol::{self as symbol, ScopeId, ScopedSymbolId}; use crate::semantic_index::{ global_scope, semantic_index, symbol_table, use_def_map, BindingWithConstraints, BindingWithConstraintsIterator, DeclarationsIterator, @@ -71,7 +71,7 @@ fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolI // If the symbol is declared, the public type is based on declarations; otherwise, it's based // on inference from bindings. - let declaredness = use_def.declaredness(db, use_def.public_declarations(symbol)); + let declaredness = use_def.public_declarations(symbol).declaredness(db); let undeclared_ty = match declaredness { None => { @@ -293,7 +293,7 @@ fn bindings_ty<'db>( // TODO: get rid of all the collects and clean up, obviously let def_types: Vec<_> = def_types.collect(); - if !def_types.is_empty() && def_types.iter().all(|ty| ty.is_none()) { + if !def_types.is_empty() && def_types.iter().all(Option::is_none) { return Some(Type::Unknown); } diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index 5142e9d5113b3..8dcac9de6823c 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -783,7 +783,7 @@ impl<'db> TypeInferenceBuilder<'db> { debug_assert!(binding.is_binding(self.db)); let use_def = self.index.use_def_map(binding.file_scope(self.db)); let declarations = use_def.declarations_at_binding(binding); - let undeclared_ty = if use_def.may_be_undeclared(self.db, declarations.clone()) { + let undeclared_ty = if declarations.clone().may_be_undeclared(self.db) { Some(Type::Unknown) } else { None From 0fda5cda25f6ef6d04064868347b6924649b22c6 Mon Sep 17 00:00:00 2001 From: David Peter Date: Tue, 10 Dec 2024 21:29:07 +0100 Subject: [PATCH 27/68] =?UTF-8?q?Use=20sub-headings=20instead=20of=20path?= =?UTF-8?q?=3D=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mdtest/statically-known-branches.md | 78 ++++++++++++++----- 1 file changed, 58 insertions(+), 20 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index d00981d0e7617..a0e4380f04c27 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -84,7 +84,9 @@ reveal_type(x) # revealed: Literal[2] ### Nested conditionals -```py path=nested_if_true_if_true.py +#### `if True` inside `if True` + +```py x = 1 if True: @@ -98,7 +100,9 @@ else: reveal_type(x) # revealed: Literal[2] ``` -```py path=nested_if_true_if_false.py +#### `if False` inside `if True` + +```py x = 1 if True: @@ -112,7 +116,9 @@ else: reveal_type(x) # revealed: Literal[3] ``` -```py path=nested_if_true_if_bool.py +#### `if ` inside `if True` + +```py def flag() -> bool: ... x = 1 @@ -128,7 +134,9 @@ else: reveal_type(x) # revealed: Literal[2, 3] ``` -```py path=nested_if_bool_if_true.py +#### `if True` inside `if ` + +```py def flag() -> bool: ... x = 1 @@ -144,7 +152,9 @@ else: reveal_type(x) # revealed: Literal[2, 4] ``` -```py path=nested_else_if_true.py +#### `if True` inside `if False` ... `else` + +```py x = 1 if False: @@ -158,7 +168,9 @@ else: reveal_type(x) # revealed: Literal[3] ``` -```py path=nested_else_if_false.py +#### `if False` inside `if False` ... `else` + +```py x = 1 if False: @@ -172,7 +184,9 @@ else: reveal_type(x) # revealed: Literal[4] ``` -```py path=nested_else_if_bool.py +#### `if ` inside `if False` ... `else` + +```py def flag() -> bool: ... x = 1 @@ -192,7 +206,9 @@ reveal_type(x) # revealed: Literal[3, 4] #### `try` ... `except` -```py path=try_if_true.py +##### `if True` inside `try` + +```py def may_raise() -> None: ... x = 1 @@ -209,7 +225,9 @@ except: reveal_type(x) # revealed: Literal[2, 4] ``` -```py path=if_true_try.py +##### `try` inside `if True` + +```py def may_raise() -> None: ... x = 1 @@ -228,7 +246,9 @@ reveal_type(x) # revealed: Literal[2, 3] #### `for` loops -```py path=for_if_true.py +##### `if True` inside `for` + +```py def iterable() -> list[()]: ... x = 1 @@ -242,7 +262,9 @@ for _ in iterable(): reveal_type(x) # revealed: Literal[1, 2] ``` -```py path=for_else_if_true.py +##### `if True` inside `for` ... `else` + +```py def iterable() -> list[()]: ... x = 1 @@ -258,7 +280,9 @@ else: reveal_type(x) # revealed: Literal[3] ``` -```py path=if_true_for.py +##### `for` inside `if True` + +```py def iterable() -> list[()]: ... x = 1 @@ -272,7 +296,9 @@ else: reveal_type(x) # revealed: Literal[1, 2] ``` -```py path=if_true_for_else.py +##### `for` ... `else` inside `if True` + +```py def iterable() -> list[()]: ... x = 1 @@ -384,7 +410,9 @@ reveal_type(x) # revealed: Literal[1, 2] ### `while` ... `else` -```py path=while_false.py +#### `while False` + +```py while False: x = 1 else: @@ -393,7 +421,9 @@ else: reveal_type(x) # revealed: Literal[2] ``` -```py path=while_true.py +#### `while True` + +```py while True: x = 1 break @@ -407,7 +437,9 @@ reveal_type(x) # revealed: Literal[1] ### Always false -```py path=if_false.py +#### `if False` + +```py x: str if False: @@ -417,7 +449,9 @@ def f() -> None: reveal_type(x) # revealed: str ``` -```py path=if_true_else.py +#### `if True … else` + +```py x: str if True: @@ -431,7 +465,9 @@ def f() -> None: ### Always true -```py path=if_true.py +#### `if True` + +```py x: str if True: @@ -441,7 +477,9 @@ def f() -> None: reveal_type(x) # revealed: int ``` -```py path=if_false_else.py +#### `if False … else` + +```py x: str if False: @@ -455,7 +493,7 @@ def f() -> None: ### Ambiguous -```py path=if_bool.py +```py def flag() -> bool: ... x: str From 544da5362612e611b7f97e5939fff758b6a78753 Mon Sep 17 00:00:00 2001 From: David Peter Date: Tue, 10 Dec 2024 22:05:53 +0100 Subject: [PATCH 28/68] Add test with break inside 'if True' --- .../mdtest/statically-known-branches.md | 22 +++++++++++++++++++ .../src/semantic_index/builder.rs | 3 ++- .../src/semantic_index/use_def.rs | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index a0e4380f04c27..bd6c474ed99a1 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -314,6 +314,28 @@ else: reveal_type(x) # revealed: Literal[3] ``` +##### `for` loop with `break` inside `if True` + +```py +def iterable() -> list[()]: ... +def flag() -> bool: ... + +x = 1 + +if True: + x = 2 + for _ in iterable(): + x = 3 + if flag(): + break + else: + x = 4 +else: + x = 5 + +reveal_type(x) # revealed: Literal[3, 4] +``` + ## If expressions See also: tests in [expression/if.md](expression/if.md). diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index 2bb761624c9c1..a05b8cc236d3d 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -303,7 +303,7 @@ impl<'db> SemanticIndexBuilder<'db> { fn record_unconditional_branching(&mut self) { self.current_use_def_map_mut() - .record_unconstrained_branch_point(); + .record_unconditional_branching(); } fn build_constraint(&mut self, constraint_node: &Expr) -> Constraint<'db> { @@ -949,6 +949,7 @@ where // We may execute the `else` clause without ever executing the body, so merge in // the pre-loop state before visiting `else`. self.flow_merge(pre_loop, pre_loop_conditions.clone()); + self.record_unconditional_branching(); self.visit_body(orelse); // Breaking out of a `for` loop bypasses the `else` clause, so merge in the break diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index cc5b4e7ada674..0bc50e8dadadb 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -628,7 +628,7 @@ impl<'db> UseDefMapBuilder<'db> { self.record_branching_condition(BranchingCondition::ConditionalOn(constraint)); } - pub(super) fn record_unconstrained_branch_point(&mut self) { + pub(super) fn record_unconditional_branching(&mut self) { self.record_branching_condition(BranchingCondition::Ambiguous); } From 00e97c39fb04cb2256063d30321abb79feca01a0 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 08:44:07 +0100 Subject: [PATCH 29/68] Minor fixes in for-loop tests --- .../resources/mdtest/statically-known-branches.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index bd6c474ed99a1..4ebe998cea1a8 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -249,7 +249,7 @@ reveal_type(x) # revealed: Literal[2, 3] ##### `if True` inside `for` ```py -def iterable() -> list[()]: ... +def iterable() -> list[object]: ... x = 1 @@ -265,7 +265,7 @@ reveal_type(x) # revealed: Literal[1, 2] ##### `if True` inside `for` ... `else` ```py -def iterable() -> list[()]: ... +def iterable() -> list[object]: ... x = 1 @@ -283,7 +283,7 @@ reveal_type(x) # revealed: Literal[3] ##### `for` inside `if True` ```py -def iterable() -> list[()]: ... +def iterable() -> list[object]: ... x = 1 @@ -299,7 +299,7 @@ reveal_type(x) # revealed: Literal[1, 2] ##### `for` ... `else` inside `if True` ```py -def iterable() -> list[()]: ... +def iterable() -> list[object]: ... x = 1 @@ -317,8 +317,7 @@ reveal_type(x) # revealed: Literal[3] ##### `for` loop with `break` inside `if True` ```py -def iterable() -> list[()]: ... -def flag() -> bool: ... +def iterable() -> list[object]: ... x = 1 @@ -326,8 +325,7 @@ if True: x = 2 for _ in iterable(): x = 3 - if flag(): - break + break else: x = 4 else: From 3f30833e6788b5cd89a56e9b8c86e2e374b72dc9 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 10:45:02 +0100 Subject: [PATCH 30/68] Straighten definitions of bitsets --- .../semantic_index/use_def/symbol_state.rs | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index fcca269fc13a4..685b581d5f038 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -81,17 +81,26 @@ const INLINE_CONSTRAINT_BLOCKS: usize = 2; /// Can keep inline this many live bindings per symbol at a given time; more will go to heap. const INLINE_BINDINGS_PER_SYMBOL: usize = 4; +/// Which constraints apply to a given binding? +type Constraints = BitSet; + +type InlineConstraintArray = [Constraints; INLINE_BINDINGS_PER_SYMBOL]; + /// One [`BitSet`] of applicable [`ScopedConstraintId`] per live binding. -type InlineConstraintArray = [BitSet; INLINE_BINDINGS_PER_SYMBOL]; -type Constraints = SmallVec; -type ConstraintsIterator<'a> = std::slice::Iter<'a, BitSet>; -type ConstraintsIntoIterator = smallvec::IntoIter; +type ConstraintsPerBinding = SmallVec; -const INLINE_BRANCHING_CONDITIONS: usize = 2; -pub(super) type BranchingConditions = BitSet; -type BranchingConditionsPerBinding = SmallVec<[BranchingConditions; INLINE_BINDINGS_PER_SYMBOL]>; +/// Iterate over all constraints for a single binding. +type ConstraintsIterator<'a> = std::slice::Iter<'a, Constraints>; +type ConstraintsIntoIterator = smallvec::IntoIter; -type BranchingConditionsIterator<'a> = std::slice::Iter<'a, BitSet>; +/// Similar to what we have for constraints, but for active branching conditions. +const INLINE_BRANCHING_BLOCKS: usize = 2; +const INLINE_BRANCHING_CONDITIONS: usize = 4; +pub(super) type BranchingConditions = BitSet; +type InlineBranchingConditionsArray = [BranchingConditions; INLINE_BRANCHING_CONDITIONS]; +type BranchingConditionsPerBinding = SmallVec; +type BranchingConditionsIterator<'a> = std::slice::Iter<'a, BranchingConditions>; +type BranchingConditionsIntoIterator = smallvec::IntoIter; /// Live declarations for a single symbol at some point in control flow. #[derive(Clone, Debug, PartialEq, Eq)] @@ -159,10 +168,10 @@ pub(super) struct SymbolBindings { /// /// This is a [`smallvec::SmallVec`] which should always have one [`BitSet`] of constraints per /// binding in `live_bindings`. - constraints: Constraints, + constraints: ConstraintsPerBinding, /// For each live binding, which [`BranchingConditions`] were active *at the time of the binding*? - pub(crate) branching_conditions: BranchingConditionsPerBinding, + branching_conditions: BranchingConditionsPerBinding, /// Could the symbol be unbound at this point? may_be_unbound: bool, @@ -172,7 +181,7 @@ impl SymbolBindings { fn unbound() -> Self { Self { live_bindings: Bindings::default(), - constraints: Constraints::default(), + constraints: ConstraintsPerBinding::default(), branching_conditions: BranchingConditionsPerBinding::default(), may_be_unbound: true, } @@ -192,7 +201,7 @@ impl SymbolBindings { // The new binding replaces all previous live bindings in this path, and has no // constraints. self.live_bindings = Bindings::with(binding_id.into()); - self.constraints = Constraints::with_capacity(1); + self.constraints = ConstraintsPerBinding::with_capacity(1); self.constraints.push(BitSet::default()); self.branching_conditions = BranchingConditionsPerBinding::with_capacity(1); @@ -279,13 +288,13 @@ impl SymbolState { let mut a = Self { bindings: SymbolBindings { live_bindings: Bindings::default(), - constraints: Constraints::default(), - branching_conditions: Constraints::default(), // TODO + constraints: ConstraintsPerBinding::default(), + branching_conditions: BranchingConditionsPerBinding::default(), may_be_unbound: self.bindings.may_be_unbound || b.bindings.may_be_unbound, }, declarations: SymbolDeclarations { live_declarations: self.declarations.live_declarations.clone(), - branching_conditions: Constraints::default(), // TODO + branching_conditions: BranchingConditionsPerBinding::default(), may_be_undeclared: self.declarations.may_be_undeclared || b.declarations.may_be_undeclared, }, @@ -304,7 +313,7 @@ impl SymbolState { let mut opt_b_decl: Option = b_decls_iter.next(); let push = |decl, - declaration_branching_conditions_iter: &mut ConstraintsIntoIterator, + declaration_branching_conditions_iter: &mut BranchingConditionsIntoIterator, merged: &mut Self| { merged.declarations.live_declarations.insert(decl); let branching_conditions = declaration_branching_conditions_iter @@ -370,7 +379,7 @@ impl SymbolState { // Helper to push `def`, with constraints in `constraints_iter`, onto `self`. let push = |def, constraints_iter: &mut ConstraintsIntoIterator, - branching_conditions_iter: &mut ConstraintsIntoIterator, + branching_conditions_iter: &mut BranchingConditionsIntoIterator, merged: &mut Self| { merged.bindings.live_bindings.insert(def); // SAFETY: we only ever create SymbolState with either no definitions and no constraint @@ -553,7 +562,7 @@ impl std::iter::FusedIterator for ConstraintIdIterator<'_> {} #[derive(Debug, Clone)] pub(super) struct BranchingConditionIdIterator<'a> { - wrapped: BitSetIterator<'a, INLINE_CONSTRAINT_BLOCKS>, + wrapped: BitSetIterator<'a, INLINE_BRANCHING_BLOCKS>, } impl Iterator for BranchingConditionIdIterator<'_> { @@ -578,7 +587,6 @@ impl<'a> Iterator for DeclarationIdIterator<'a> { type Item = (ScopedDefinitionId, BranchingConditionIdIterator<'a>); fn next(&mut self) -> Option { - // self.inner.next().map(ScopedDefinitionId::from_u32) match (self.inner.next(), self.branching_conditions.next()) { (None, None) => None, (Some(declaration), Some(branching_conditions)) => Some(( From 4e10222a75dd4511b40de3bf3dab435ea60f2b6e Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 11:06:02 +0100 Subject: [PATCH 31/68] Documentation --- .../src/semantic_index.rs | 2 +- .../src/semantic_index/branching.rs | 7 ----- .../src/semantic_index/branching_condition.rs | 29 +++++++++++++++++++ .../src/semantic_index/use_def.rs | 3 +- .../semantic_index/use_def/symbol_state.rs | 5 ++-- 5 files changed, 35 insertions(+), 11 deletions(-) delete mode 100644 crates/red_knot_python_semantic/src/semantic_index/branching.rs create mode 100644 crates/red_knot_python_semantic/src/semantic_index/branching_condition.rs diff --git a/crates/red_knot_python_semantic/src/semantic_index.rs b/crates/red_knot_python_semantic/src/semantic_index.rs index 800feaa1d3860..bd9a94fda74e9 100644 --- a/crates/red_knot_python_semantic/src/semantic_index.rs +++ b/crates/red_knot_python_semantic/src/semantic_index.rs @@ -20,7 +20,7 @@ use crate::semantic_index::use_def::UseDefMap; use crate::Db; pub mod ast_ids; -pub(crate) mod branching; +pub(crate) mod branching_condition; mod builder; pub(crate) mod constraint; pub mod definition; diff --git a/crates/red_knot_python_semantic/src/semantic_index/branching.rs b/crates/red_knot_python_semantic/src/semantic_index/branching.rs deleted file mode 100644 index d2e322be28dcb..0000000000000 --- a/crates/red_knot_python_semantic/src/semantic_index/branching.rs +++ /dev/null @@ -1,7 +0,0 @@ -use super::constraint::Constraint; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum BranchingCondition<'db> { - ConditionalOn(Constraint<'db>), - Ambiguous, -} diff --git a/crates/red_knot_python_semantic/src/semantic_index/branching_condition.rs b/crates/red_knot_python_semantic/src/semantic_index/branching_condition.rs new file mode 100644 index 0000000000000..d43b3edf31781 --- /dev/null +++ b/crates/red_knot_python_semantic/src/semantic_index/branching_condition.rs @@ -0,0 +1,29 @@ +use super::constraint::Constraint; + +/// Used to represent active branching conditions that apply to a particular definition. +/// A definition can either be conditional on a specific constraint from a `if`, `elif`, +/// `while` statement, an `if`-expression, or a Boolean expression. Or it can be marked +/// as 'ambiguous' if it occurred in a control-flow path that is not conditional on any +/// specific expression that can be statically analyzed (`for` loop, `try` ... `except`). +/// +/// +/// For example: +/// ```py +/// a = 1 # no active branching conditions +/// +/// if test1: +/// b = 1 # ConditionalOn(test1) +/// +/// if test2: +/// c = 1 # ConditionalOn(test1), ConditionalOn(test2) +/// +/// for _ in range(10): +/// d = 1 # ConditionalOn(test1), Ambiguous +/// else: +/// d = 1 # ConditionalOn(~test1) +/// ``` +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum BranchingCondition<'db> { + ConditionalOn(Constraint<'db>), + Ambiguous, +} diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 0bc50e8dadadb..02c016321aef5 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -226,7 +226,7 @@ use self::symbol_state::{ ScopedConstraintId, ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState, }; use crate::semantic_index::ast_ids::{HasScopedExpressionId, ScopedUseId}; -use crate::semantic_index::branching::BranchingCondition; +use crate::semantic_index::branching_condition::BranchingCondition; use crate::semantic_index::constraint::ConstraintNode; use crate::semantic_index::definition::Definition; use crate::semantic_index::symbol::ScopedSymbolId; @@ -576,6 +576,7 @@ pub(super) struct FlowSnapshot { symbol_states: IndexVec, } +/// A snapshot of the active branching conditions at a particular point in control flow. #[derive(Clone, Debug)] pub(super) struct BranchingConditionsSnapshot(BranchingConditions); diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 685b581d5f038..c6bece86710ad 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -57,7 +57,7 @@ pub(super) struct ScopedDefinitionId; #[newtype_index] pub(super) struct ScopedConstraintId; -/// A newtype-index for a branching condition in a particular scope. +/// A newtype-index for a [`crate::semantic_index::branching::BranchingCondition`] in a particular scope. #[newtype_index] pub(super) struct ScopedBranchingConditionId; @@ -86,7 +86,7 @@ type Constraints = BitSet; type InlineConstraintArray = [Constraints; INLINE_BINDINGS_PER_SYMBOL]; -/// One [`BitSet`] of applicable [`ScopedConstraintId`] per live binding. +/// One [`BitSet`] of applicable [`ScopedConstraintId`]s per live binding. type ConstraintsPerBinding = SmallVec; /// Iterate over all constraints for a single binding. @@ -98,6 +98,7 @@ const INLINE_BRANCHING_BLOCKS: usize = 2; const INLINE_BRANCHING_CONDITIONS: usize = 4; pub(super) type BranchingConditions = BitSet; type InlineBranchingConditionsArray = [BranchingConditions; INLINE_BRANCHING_CONDITIONS]; +/// One [`BitSet`] of active [`ScopedBranchingConditionId`]s per live binding. type BranchingConditionsPerBinding = SmallVec; type BranchingConditionsIterator<'a> = std::slice::Iter<'a, BranchingConditions>; type BranchingConditionsIntoIterator = smallvec::IntoIter; From aeae261f1796021442c7fbdedb60eefdaf254022 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 11:39:18 +0100 Subject: [PATCH 32/68] Fix docs --- .../src/semantic_index/use_def/symbol_state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index c6bece86710ad..ea89967c38895 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -57,7 +57,7 @@ pub(super) struct ScopedDefinitionId; #[newtype_index] pub(super) struct ScopedConstraintId; -/// A newtype-index for a [`crate::semantic_index::branching::BranchingCondition`] in a particular scope. +/// A newtype-index for a [`crate::semantic_index::branching_condition::BranchingCondition`] in a particular scope. #[newtype_index] pub(super) struct ScopedBranchingConditionId; From 73c705d0a7b10f52cefa368199e769674f5e5fcb Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 11:39:42 +0100 Subject: [PATCH 33/68] Remove TODO --- crates/red_knot_python_semantic/src/semantic_index/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index a05b8cc236d3d..82179c35c12b5 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -882,7 +882,7 @@ where // Breaking out of a while loop bypasses the `else` clause, so merge in the break // states after visiting `else`. for break_state in break_states { - self.flow_merge(break_state, pre_loop_conditions.clone()); // TODO? + self.flow_merge(break_state, pre_loop_conditions.clone()); } } ast::Stmt::With(ast::StmtWith { From e773c743aa12dbf36e80f57d65ec1f1982a696a8 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 12:07:01 +0100 Subject: [PATCH 34/68] Handle control flow in try..else..finally --- .../mdtest/statically-known-branches.md | 52 +++++++++++++++++-- .../src/semantic_index/builder.rs | 17 +++--- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 4ebe998cea1a8..35bef26fe5b70 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -236,12 +236,58 @@ if True: try: may_raise() x = 2 - except: + except KeyError: x = 3 + except ValueError: + x = 4 else: - x = 4 + x = 5 -reveal_type(x) # revealed: Literal[2, 3] +reveal_type(x) # revealed: Literal[2, 3, 4] +``` + +##### `try` with `else` inside `if True` + +```py +def may_raise() -> None: ... + +x = 1 + +if True: + try: + may_raise() + x = 2 + except KeyError: + x = 3 + else: + x = 4 +else: + x = 5 + +reveal_type(x) # revealed: Literal[3, 4] +``` + +##### `try` with `finally` inside `if True` + +```py +def may_raise() -> None: ... + +x = 1 + +if True: + try: + may_raise() + x = 2 + except KeyError: + x = 3 + else: + x = 4 + finally: + x = 5 +else: + x = 6 + +reveal_type(x) # revealed: Literal[5] ``` #### `for` loops diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index 82179c35c12b5..1e6fa603432e5 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -1029,7 +1029,6 @@ where // as there necessarily must have been 0 `except` blocks executed // if we hit the `else` block. let post_try_block_state = self.flow_snapshot(); - let post_try_block_conditions = self.branching_conditions_snapshot(); // Prepare for visiting the `except` block(s) self.flow_restore(pre_try_block_state, pre_try_block_conditions.clone()); @@ -1038,11 +1037,8 @@ where } let pre_except_state = self.flow_snapshot(); - let pre_except_conditions = self.branching_conditions_snapshot(); let num_handlers = handlers.len(); - self.record_unconditional_branching(); - for (i, except_handler) in handlers.iter().enumerate() { let ast::ExceptHandler::ExceptHandler(except_handler) = except_handler; let ast::ExceptHandlerExceptHandler { @@ -1052,6 +1048,8 @@ where range: _, } = except_handler; + self.record_unconditional_branching(); + if let Some(handled_exceptions) = handled_exceptions { self.visit_expr(handled_exceptions); } @@ -1081,16 +1079,18 @@ where if i < (num_handlers - 1) { self.flow_restore( pre_except_state.clone(), - pre_except_conditions.clone(), + pre_try_block_conditions.clone(), ); } } // If we get to the `else` block, we know that 0 of the `except` blocks can have been executed, // and the entire `try` block must have been executed: - self.flow_restore(post_try_block_state, post_try_block_conditions); + self.flow_restore(post_try_block_state, pre_try_block_conditions.clone()); } + self.record_unconditional_branching(); + self.visit_body(orelse); for post_except_state in post_except_states { @@ -1107,7 +1107,12 @@ where // For more details, see: // - https://astral-sh.notion.site/Exception-handler-control-flow-11348797e1ca80bb8ce1e9aedbbe439d // - https://github.com/astral-sh/ruff/pull/13633#discussion_r1788626702 + self.record_unconditional_branching(); + self.visit_body(finalbody); + + self.current_use_def_map_mut() + .restore_branching_conditions(pre_try_block_conditions); } _ => { walk_stmt(self, stmt); From 8639b6b713867c5150c586e7b9d0f5bf390add87 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 14:12:32 +0100 Subject: [PATCH 35/68] Support analysis of 'match' statements --- .../mdtest/statically-known-branches.md | 87 +++++++++++++++++++ .../src/semantic_index/builder.rs | 27 +++--- .../src/semantic_index/constraint.rs | 15 +++- .../src/semantic_index/use_def.rs | 30 ++++++- crates/red_knot_python_semantic/src/types.rs | 9 +- .../src/types/infer.rs | 23 +++-- .../src/types/narrow.rs | 41 +++------ 7 files changed, 178 insertions(+), 54 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 35bef26fe5b70..082ace0b5808c 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -499,6 +499,93 @@ else: reveal_type(x) # revealed: Literal[1] ``` +## `match` statements + +### Single-valued types, always true + +```py +x = 1 + +match "a": + case "a": + x = 2 + case "b": + x = 3 + +reveal_type(x) # revealed: Literal[2] +``` + +### Single-valued types, always false + +```py +x = 1 + +match "something else": + case "a": + x = 1 + case "b": + x = 2 + +reveal_type(x) # revealed: Literal[1] +``` + +### Single-valued types, with wildcard pattern + +This is a case that we can not handle at the moment. Our reasoning about match patterns is too +local. We can infer that the `x = 2` binding is unconditionally visible. But when we traverse all +bindings backwards, we first see the `x = 3` binding which is also visible. At the moment, we do not +mark it as *unconditionally* visible to avoid blocking off previous bindings (we would infer +`Literal[3]` otherwise). + +```py +x = 1 + +match "a": + case "a": + x = 2 + case _: + x = 3 + +# TODO: ideally, this should be Literal[2] +reveal_type(x) # revealed: Literal[2, 3] +``` + +### Non-single-valued types + +```py +def _(s: str): + match s: + case "a": + x = 1 + case _: + x = 2 + + reveal_type(x) # revealed: Literal[1, 2] +``` + +### `sys.version_info` + +```toml +[environment] +python-version = "3.13" +``` + +```py +import sys + +minor = "too old" + +match sys.version_info.minor: + case 12: + minor = 12 + case 13: + minor = 13 + case _: + pass + +reveal_type(minor) # revealed: Literal[13] +``` + ## Conditional declarations ### Always false diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index 1e6fa603432e5..302fa6fb56fcb 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -6,14 +6,15 @@ use rustc_hash::FxHashMap; use ruff_db::files::File; use ruff_db::parsed::ParsedModule; use ruff_index::IndexVec; -use ruff_python_ast as ast; use ruff_python_ast::name::Name; use ruff_python_ast::visitor::{walk_expr, walk_pattern, walk_stmt, Visitor}; +use ruff_python_ast::{self as ast, Pattern}; use ruff_python_ast::{BoolOp, Expr}; use crate::ast_node_ref::AstNodeRef; use crate::semantic_index::ast_ids::node_key::ExpressionNodeKey; use crate::semantic_index::ast_ids::AstIdsBuilder; +use crate::semantic_index::constraint::PatternConstraintKind; use crate::semantic_index::definition::{ AssignmentDefinitionNodeRef, ComprehensionDefinitionNodeRef, Definition, DefinitionNodeKey, DefinitionNodeRef, ForStmtDefinitionNodeRef, ImportFromDefinitionNodeRef, @@ -341,22 +342,24 @@ impl<'db> SemanticIndexBuilder<'db> { fn add_pattern_constraint( &mut self, - subject: &ast::Expr, + subject: Expression<'db>, pattern: &ast::Pattern, ) -> PatternConstraint<'db> { - #[allow(unsafe_code)] - let (subject, pattern) = unsafe { - ( - AstNodeRef::new(self.module.clone(), subject), - AstNodeRef::new(self.module.clone(), pattern), - ) + let kind = match pattern { + Pattern::MatchValue(pattern) => { + let value = self.add_standalone_expression(&pattern.value); + PatternConstraintKind::Value(value) + } + Pattern::MatchSingleton(singleton) => PatternConstraintKind::Singleton(singleton.value), + _ => PatternConstraintKind::Unsupported, }; + let pattern_constraint = PatternConstraint::new( self.db, self.file, self.current_scope(), subject, - pattern, + kind, countme::Count::default(), ); self.current_use_def_map_mut() @@ -963,7 +966,7 @@ where cases, range: _, }) => { - self.add_standalone_expression(subject); + let subject_expr = self.add_standalone_expression(subject); self.visit_expr(subject); let after_subject = self.flow_snapshot(); @@ -971,14 +974,14 @@ where let Some((first, remaining)) = cases.split_first() else { return; }; - self.add_pattern_constraint(subject, &first.pattern); + self.add_pattern_constraint(subject_expr, &first.pattern); self.visit_match_case(first); let mut post_case_snapshots = vec![]; for case in remaining { post_case_snapshots.push(self.flow_snapshot()); self.flow_restore(after_subject.clone(), after_subject_cs.clone()); - self.add_pattern_constraint(subject, &case.pattern); + self.add_pattern_constraint(subject_expr, &case.pattern); self.visit_match_case(case); } for post_clause_state in post_case_snapshots { diff --git a/crates/red_knot_python_semantic/src/semantic_index/constraint.rs b/crates/red_knot_python_semantic/src/semantic_index/constraint.rs index 44b542f0e90ac..347f0ebaac4f7 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/constraint.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/constraint.rs @@ -1,7 +1,6 @@ use ruff_db::files::File; -use ruff_python_ast as ast; +use ruff_python_ast::Singleton; -use crate::ast_node_ref::AstNodeRef; use crate::db::Db; use crate::semantic_index::expression::Expression; use crate::semantic_index::symbol::{FileScopeId, ScopeId}; @@ -18,6 +17,14 @@ pub(crate) enum ConstraintNode<'db> { Pattern(PatternConstraint<'db>), } +/// Pattern kinds for which we do support type narrowing and/or static truthiness analysis. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum PatternConstraintKind<'db> { + Singleton(Singleton), + Value(Expression<'db>), + Unsupported, +} + #[salsa::tracked] pub(crate) struct PatternConstraint<'db> { #[id] @@ -28,11 +35,11 @@ pub(crate) struct PatternConstraint<'db> { #[no_eq] #[return_ref] - pub(crate) subject: AstNodeRef, + pub(crate) subject: Expression<'db>, #[no_eq] #[return_ref] - pub(crate) pattern: AstNodeRef, + pub(crate) kind: PatternConstraintKind<'db>, #[no_eq] count: countme::Count>, diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 02c016321aef5..4c0ac6cba798d 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -511,9 +511,35 @@ impl<'db> BranchingConditionsIterator<'_, 'db> { ty.bool(db).negate_if(!is_positive) } BranchingCondition::ConditionalOn(Constraint { - node: ConstraintNode::Pattern(..), + node: ConstraintNode::Pattern(inner), .. - }) => Truthiness::Ambiguous, + }) => match inner.kind(db) { + super::constraint::PatternConstraintKind::Value(value) => { + let subject_expression = inner.subject(db); + let inference = infer_expression_types(db, *subject_expression); + let scope = subject_expression.scope(db); + let subject_ty = inference.expression_ty( + subject_expression + .node_ref(db) + .scoped_expression_id(db, scope), + ); + + let inference = infer_expression_types(db, *value); + let scope = value.scope(db); + let value_ty = inference + .expression_ty(value.node_ref(db).scoped_expression_id(db, scope)); + + if subject_ty.is_single_valued(db) { + Truthiness::from_bool(subject_ty.is_equivalent_to(db, value_ty)) + } else { + Truthiness::Ambiguous + } + } + super::constraint::PatternConstraintKind::Singleton(_) + | super::constraint::PatternConstraintKind::Unsupported => { + Truthiness::Ambiguous + } + }, BranchingCondition::Ambiguous => Truthiness::Ambiguous, }; diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 24c497e98d5ff..9c9a99588daea 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -258,7 +258,6 @@ fn bindings_ty<'db>( let result = branching_conditions.truthiness(db); if result.any_always_false { - // TODO: do we need to call binding_ty(…) even if we don't need the result? (None, UnconditionallyVisible::No) } else { let unconditionally_visible = @@ -2550,6 +2549,14 @@ pub enum Truthiness { } impl Truthiness { + pub(crate) const fn from_bool(value: bool) -> Self { + if value { + Self::AlwaysTrue + } else { + Self::AlwaysFalse + } + } + const fn is_ambiguous(self) -> bool { matches!(self, Truthiness::Ambiguous) } diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index 8dcac9de6823c..b03ae5ee41480 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -1718,13 +1718,24 @@ impl<'db> TypeInferenceBuilder<'db> { fn infer_match_pattern(&mut self, pattern: &ast::Pattern) { // TODO(dhruvmanila): Add a Salsa query for inferring pattern types and matching against // the subject expression: https://github.com/astral-sh/ruff/pull/13147#discussion_r1739424510 + match pattern { + ast::Pattern::MatchValue(match_value) => { + self.infer_standalone_expression(&match_value.value); + } + _ => { + self.infer_match_pattern_impl(pattern); + } + } + } + + fn infer_match_pattern_impl(&mut self, pattern: &ast::Pattern) { match pattern { ast::Pattern::MatchValue(match_value) => { self.infer_expression(&match_value.value); } ast::Pattern::MatchSequence(match_sequence) => { for pattern in &match_sequence.patterns { - self.infer_match_pattern(pattern); + self.infer_match_pattern_impl(pattern); } } ast::Pattern::MatchMapping(match_mapping) => { @@ -1738,7 +1749,7 @@ impl<'db> TypeInferenceBuilder<'db> { self.infer_expression(key); } for pattern in patterns { - self.infer_match_pattern(pattern); + self.infer_match_pattern_impl(pattern); } } ast::Pattern::MatchClass(match_class) => { @@ -1748,21 +1759,21 @@ impl<'db> TypeInferenceBuilder<'db> { arguments, } = match_class; for pattern in &arguments.patterns { - self.infer_match_pattern(pattern); + self.infer_match_pattern_impl(pattern); } for keyword in &arguments.keywords { - self.infer_match_pattern(&keyword.pattern); + self.infer_match_pattern_impl(&keyword.pattern); } self.infer_expression(cls); } ast::Pattern::MatchAs(match_as) => { if let Some(pattern) = &match_as.pattern { - self.infer_match_pattern(pattern); + self.infer_match_pattern_impl(pattern); } } ast::Pattern::MatchOr(match_or) => { for pattern in &match_or.patterns { - self.infer_match_pattern(pattern); + self.infer_match_pattern_impl(pattern); } } ast::Pattern::MatchStar(_) | ast::Pattern::MatchSingleton(_) => {} diff --git a/crates/red_knot_python_semantic/src/types/narrow.rs b/crates/red_knot_python_semantic/src/types/narrow.rs index 69513ccfba4c1..bf74bdfd38364 100644 --- a/crates/red_knot_python_semantic/src/types/narrow.rs +++ b/crates/red_knot_python_semantic/src/types/narrow.rs @@ -1,5 +1,7 @@ use crate::semantic_index::ast_ids::HasScopedExpressionId; -use crate::semantic_index::constraint::{Constraint, ConstraintNode, PatternConstraint}; +use crate::semantic_index::constraint::{ + Constraint, ConstraintNode, PatternConstraint, PatternConstraintKind, +}; use crate::semantic_index::definition::Definition; use crate::semantic_index::expression::Expression; use crate::semantic_index::symbol::{ScopeId, ScopedSymbolId, SymbolTable}; @@ -215,31 +217,12 @@ impl<'db> NarrowingConstraintsBuilder<'db> { ) -> Option> { let subject = pattern.subject(self.db); - match pattern.pattern(self.db).node() { - ast::Pattern::MatchValue(_) => { - None // TODO - } - ast::Pattern::MatchSingleton(singleton_pattern) => { - self.evaluate_match_pattern_singleton(subject, singleton_pattern) - } - ast::Pattern::MatchSequence(_) => { - None // TODO - } - ast::Pattern::MatchMapping(_) => { - None // TODO - } - ast::Pattern::MatchClass(_) => { - None // TODO - } - ast::Pattern::MatchStar(_) => { - None // TODO - } - ast::Pattern::MatchAs(_) => { - None // TODO - } - ast::Pattern::MatchOr(_) => { - None // TODO + match pattern.kind(self.db) { + PatternConstraintKind::Singleton(singleton) => { + self.evaluate_match_pattern_singleton(*subject, *singleton) } + // TODO: support more pattern kinds + PatternConstraintKind::Value(_) | PatternConstraintKind::Unsupported => None, } } @@ -457,14 +440,14 @@ impl<'db> NarrowingConstraintsBuilder<'db> { fn evaluate_match_pattern_singleton( &mut self, - subject: &ast::Expr, - pattern: &ast::PatternMatchSingleton, + subject: Expression<'db>, + singleton: ast::Singleton, ) -> Option> { - if let Some(ast::ExprName { id, .. }) = subject.as_name_expr() { + if let Some(ast::ExprName { id, .. }) = subject.node_ref(self.db).as_name_expr() { // SAFETY: we should always have a symbol for every Name node. let symbol = self.symbols().symbol_id_by_name(id).unwrap(); - let ty = match pattern.value { + let ty = match singleton { ast::Singleton::None => Type::none(self.db), ast::Singleton::True => Type::BooleanLiteral(true), ast::Singleton::False => Type::BooleanLiteral(false), From 888faad7837d0009fa0d878b8ec62891f1115082 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 14:26:34 +0100 Subject: [PATCH 36/68] Move static truthiness to new module --- .../src/semantic_index.rs | 3 +- .../src/semantic_index/use_def.rs | 81 +------------- crates/red_knot_python_semantic/src/types.rs | 6 +- .../src/types/static_truthiness.rs | 103 ++++++++++++++++++ 4 files changed, 114 insertions(+), 79 deletions(-) create mode 100644 crates/red_knot_python_semantic/src/types/static_truthiness.rs diff --git a/crates/red_knot_python_semantic/src/semantic_index.rs b/crates/red_knot_python_semantic/src/semantic_index.rs index bd9a94fda74e9..637b1b32a14b2 100644 --- a/crates/red_knot_python_semantic/src/semantic_index.rs +++ b/crates/red_knot_python_semantic/src/semantic_index.rs @@ -29,7 +29,8 @@ pub mod symbol; mod use_def; pub(crate) use self::use_def::{ - BindingWithConstraints, BindingWithConstraintsIterator, DeclarationsIterator, + BindingWithConstraints, BindingWithConstraintsIterator, BranchingConditionsIterator, + DeclarationsIterator, }; type SymbolMap = hashbrown::HashMap; diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 4c0ac6cba798d..ca496947934a3 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -225,17 +225,15 @@ use self::symbol_state::{ BindingIdWithConstraintsIterator, ConstraintIdIterator, DeclarationIdIterator, ScopedConstraintId, ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState, }; -use crate::semantic_index::ast_ids::{HasScopedExpressionId, ScopedUseId}; +use crate::semantic_index::ast_ids::ScopedUseId; use crate::semantic_index::branching_condition::BranchingCondition; -use crate::semantic_index::constraint::ConstraintNode; use crate::semantic_index::definition::Definition; use crate::semantic_index::symbol::ScopedSymbolId; use crate::semantic_index::use_def::symbol_state::{ BranchingConditionIdIterator, BranchingConditions, ScopedBranchingConditionId, }; use crate::symbol::Boundness; -use crate::types::{infer_expression_types, Truthiness}; -use crate::Db; +use crate::types::StaticTruthiness; use ruff_index::IndexVec; use rustc_hash::FxHashMap; @@ -246,7 +244,7 @@ mod symbol_state; fn compute_boundness<'db, 'map, C>( db: &dyn crate::db::Db, - conditions: C, + conditions_per_binding: C, may_be_undefined: bool, ) -> Option where @@ -255,8 +253,8 @@ where { let mut definitely_bound = false; let mut definitely_unbound = true; - for condition in conditions { - let result = condition.truthiness(db); + for conditions in conditions_per_binding { + let result = StaticTruthiness::analyze(db, conditions); if !result.any_always_false { definitely_unbound = false; @@ -483,75 +481,6 @@ impl<'db> Iterator for BranchingConditionsIterator<'_, 'db> { } } -pub(crate) struct BranchConditionTruthiness { - pub any_always_false: bool, - pub all_always_true: bool, - pub at_least_one_condition: bool, -} - -impl<'db> BranchingConditionsIterator<'_, 'db> { - pub(crate) fn truthiness(self, db: &'db dyn Db) -> BranchConditionTruthiness { - let mut result = BranchConditionTruthiness { - any_always_false: false, - all_always_true: true, - at_least_one_condition: false, - }; - - for condition in self { - let truthiness = match condition { - BranchingCondition::ConditionalOn(Constraint { - node: ConstraintNode::Expression(test_expr), - is_positive, - }) => { - let inference = infer_expression_types(db, test_expr); - let scope = test_expr.scope(db); - let ty = inference - .expression_ty(test_expr.node_ref(db).scoped_expression_id(db, scope)); - - ty.bool(db).negate_if(!is_positive) - } - BranchingCondition::ConditionalOn(Constraint { - node: ConstraintNode::Pattern(inner), - .. - }) => match inner.kind(db) { - super::constraint::PatternConstraintKind::Value(value) => { - let subject_expression = inner.subject(db); - let inference = infer_expression_types(db, *subject_expression); - let scope = subject_expression.scope(db); - let subject_ty = inference.expression_ty( - subject_expression - .node_ref(db) - .scoped_expression_id(db, scope), - ); - - let inference = infer_expression_types(db, *value); - let scope = value.scope(db); - let value_ty = inference - .expression_ty(value.node_ref(db).scoped_expression_id(db, scope)); - - if subject_ty.is_single_valued(db) { - Truthiness::from_bool(subject_ty.is_equivalent_to(db, value_ty)) - } else { - Truthiness::Ambiguous - } - } - super::constraint::PatternConstraintKind::Singleton(_) - | super::constraint::PatternConstraintKind::Unsupported => { - Truthiness::Ambiguous - } - }, - BranchingCondition::Ambiguous => Truthiness::Ambiguous, - }; - - result.any_always_false |= truthiness.is_always_false(); - result.all_always_true &= truthiness.is_always_true(); - result.at_least_one_condition = true; - } - - result - } -} - impl std::iter::FusedIterator for BranchingConditionsIterator<'_, '_> {} #[derive(Clone)] diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 9c9a99588daea..41250b5da3770 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -31,6 +31,7 @@ use crate::types::diagnostic::TypeCheckDiagnosticsBuilder; use crate::types::mro::{ClassBase, Mro, MroError, MroIterator}; use crate::types::narrow::narrowing_constraint; use crate::{Db, FxOrderSet, Module, Program, PythonVersion}; +pub(crate) use static_truthiness::StaticTruthiness; mod builder; mod call; @@ -40,6 +41,7 @@ mod infer; mod mro; mod narrow; mod signatures; +mod static_truthiness; mod string_annotation; mod unpacker; @@ -255,7 +257,7 @@ fn bindings_ty<'db>( constraints, branching_conditions, }| { - let result = branching_conditions.truthiness(db); + let result = StaticTruthiness::analyze(db, branching_conditions); if result.any_always_false { (None, UnconditionallyVisible::No) @@ -335,7 +337,7 @@ fn declarations_ty<'db>( ) -> DeclaredTypeResult<'db> { let decl_types = declarations .map(|(declaration, branching_conditions)| { - let result = branching_conditions.truthiness(db); + let result = StaticTruthiness::analyze(db, branching_conditions); if result.any_always_false { (Type::Never, UnconditionallyVisible::No) diff --git a/crates/red_knot_python_semantic/src/types/static_truthiness.rs b/crates/red_knot_python_semantic/src/types/static_truthiness.rs new file mode 100644 index 0000000000000..a0551232a5fdf --- /dev/null +++ b/crates/red_knot_python_semantic/src/types/static_truthiness.rs @@ -0,0 +1,103 @@ +use crate::semantic_index::{ + ast_ids::HasScopedExpressionId, + branching_condition::BranchingCondition, + constraint::{Constraint, ConstraintNode, PatternConstraintKind}, + BranchingConditionsIterator, +}; +use crate::types::{infer_expression_types, Truthiness}; +use crate::Db; + +/// The result of a static-truthiness analysis. +/// +/// Consider the following example: +/// ```py +/// a = 1 +/// if True: +/// b = 1 +/// if : +/// c = 1 +/// if False: +/// d = 1 +/// ``` +/// +/// Given an iterator over the branching conditions for each of these bindings, we would get: +/// ``` +/// - a: {any_always_false: false, all_always_true: true, at_least_one_condition: false} +/// - b: {any_always_false: false, all_always_true: true, at_least_one_condition: true} +/// - c: {any_always_false: false, all_always_true: false, at_least_one_condition: true} +/// - d: {any_always_false: true, all_always_true: false, at_least_one_condition: true} +/// ``` +pub(crate) struct StaticTruthiness { + /// Is any of the branching conditions always false? (false if there are no conditions) + pub(crate) any_always_false: bool, + /// Are all of the branching conditions always true? (true if there are no conditions) + pub(crate) all_always_true: bool, + /// Is there at least one branching condition? + pub(crate) at_least_one_condition: bool, +} + +impl StaticTruthiness { + /// Analyze the (statically known) truthiness for a list of branching conditions. + pub(crate) fn analyze<'db>( + db: &'db dyn Db, + branching_conditions: BranchingConditionsIterator<'_, 'db>, + ) -> Self { + let mut result = Self { + any_always_false: false, + all_always_true: true, + at_least_one_condition: false, + }; + + for condition in branching_conditions { + let truthiness = match condition { + BranchingCondition::ConditionalOn(Constraint { + node: ConstraintNode::Expression(test_expr), + is_positive, + }) => { + let inference = infer_expression_types(db, test_expr); + let scope = test_expr.scope(db); + let ty = inference + .expression_ty(test_expr.node_ref(db).scoped_expression_id(db, scope)); + + ty.bool(db).negate_if(!is_positive) + } + BranchingCondition::ConditionalOn(Constraint { + node: ConstraintNode::Pattern(inner), + .. + }) => match inner.kind(db) { + PatternConstraintKind::Value(value) => { + let subject_expression = inner.subject(db); + let inference = infer_expression_types(db, *subject_expression); + let scope = subject_expression.scope(db); + let subject_ty = inference.expression_ty( + subject_expression + .node_ref(db) + .scoped_expression_id(db, scope), + ); + + let inference = infer_expression_types(db, *value); + let scope = value.scope(db); + let value_ty = inference + .expression_ty(value.node_ref(db).scoped_expression_id(db, scope)); + + if subject_ty.is_single_valued(db) { + Truthiness::from_bool(subject_ty.is_equivalent_to(db, value_ty)) + } else { + Truthiness::Ambiguous + } + } + PatternConstraintKind::Singleton(_) | PatternConstraintKind::Unsupported => { + Truthiness::Ambiguous + } + }, + BranchingCondition::Ambiguous => Truthiness::Ambiguous, + }; + + result.any_always_false |= truthiness.is_always_false(); + result.all_always_true &= truthiness.is_always_true(); + result.at_least_one_condition = true; + } + + result + } +} From 02f9ae49a7b8e018b696a14b39d5c8e337f0d5da Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 15:07:08 +0100 Subject: [PATCH 37/68] Move static-truthiness merging to dedicated module --- .../src/semantic_index/use_def.rs | 67 +++++++++---------- .../src/types/static_truthiness.rs | 33 +++++++++ 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index ca496947934a3..ff0b6fb8ed0bf 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -242,40 +242,6 @@ use super::constraint::Constraint; mod bitset; mod symbol_state; -fn compute_boundness<'db, 'map, C>( - db: &dyn crate::db::Db, - conditions_per_binding: C, - may_be_undefined: bool, -) -> Option -where - 'db: 'map, - C: Iterator>, -{ - let mut definitely_bound = false; - let mut definitely_unbound = true; - for conditions in conditions_per_binding { - let result = StaticTruthiness::analyze(db, conditions); - - if !result.any_always_false { - definitely_unbound = false; - } - - if result.all_always_true { - definitely_bound = true; - } - } - - if definitely_unbound { - None - } else { - if definitely_bound || !may_be_undefined { - Some(Boundness::Bound) - } else { - Some(Boundness::PossiblyUnbound) - } - } -} - /// Applicable definitions and constraints for every use of a name. #[derive(Debug, PartialEq, Eq)] pub(crate) struct UseDefMap<'db> { @@ -326,7 +292,7 @@ impl<'db> UseDefMap<'db> { let conditions = self .bindings_iterator(bindings) .map(|binding| binding.branching_conditions); - compute_boundness(db, conditions, bindings.may_be_unbound()) + analyze_boundness(db, conditions, bindings.may_be_unbound()) } pub(crate) fn public_bindings( @@ -345,7 +311,7 @@ impl<'db> UseDefMap<'db> { let conditions = self .bindings_iterator(bindings) .map(|binding| binding.branching_conditions); - compute_boundness(db, conditions, bindings.may_be_unbound()) + analyze_boundness(db, conditions, bindings.may_be_unbound()) } pub(crate) fn bindings_at_declaration( @@ -495,7 +461,7 @@ impl DeclarationsIterator<'_, '_> { pub(crate) fn declaredness(self, db: &dyn crate::db::Db) -> Option { let may_be_undeclared = self.may_be_undeclared; let conditions = self.map(|(_, conditions)| conditions); - compute_boundness(db, conditions, may_be_undeclared) + analyze_boundness(db, conditions, may_be_undeclared) } pub(crate) fn may_be_undeclared(self, db: &dyn crate::db::Db) -> bool { @@ -700,3 +666,30 @@ impl<'db> UseDefMapBuilder<'db> { } } } + +fn analyze_boundness<'db, 'map, C>( + db: &dyn crate::db::Db, + conditions_per_binding: C, + may_be_unbound: bool, +) -> Option +where + 'db: 'map, + C: Iterator>, +{ + let result = conditions_per_binding.fold(StaticTruthiness::no_bindings(), |r, conditions| { + r.merge(StaticTruthiness::analyze(db, conditions)) + }); + + let definitely_unbound = result.any_always_false; + let definitely_bound = result.all_always_true || !may_be_unbound; + + if definitely_unbound { + None + } else { + if definitely_bound { + Some(Boundness::Bound) + } else { + Some(Boundness::PossiblyUnbound) + } + } +} diff --git a/crates/red_knot_python_semantic/src/types/static_truthiness.rs b/crates/red_knot_python_semantic/src/types/static_truthiness.rs index a0551232a5fdf..01b8935d6aedb 100644 --- a/crates/red_knot_python_semantic/src/types/static_truthiness.rs +++ b/crates/red_knot_python_semantic/src/types/static_truthiness.rs @@ -100,4 +100,37 @@ impl StaticTruthiness { result } + + /// Merge two static truthiness results, as if they came from two different control-flow paths. + /// + /// Note that the logical operations are exactly opposite to what one would expect from the names + /// of the fields. The reason for this is that we want to draw conclusions like "this symbol can + /// not be bound because one of the branching conditions is always false". We can only draw this + /// conclusion if this is true in both control-flow paths. Similarly, we want to infer that the + /// binding of a symbol is unconditionally visible, if all branching conditions are known to be + /// statically true. It is enough if this is the case for *any* of the control-flow paths. Other + /// control flow paths will not be taken if this is the case. + pub(crate) fn merge(self, other: Self) -> Self { + Self { + any_always_false: self.any_always_false && other.any_always_false, + all_always_true: self.all_always_true || other.all_always_true, + at_least_one_condition: self.at_least_one_condition && other.at_least_one_condition, + } + } + + /// A static truthiness result that states our knowledge before we have seen any bindings. + /// + /// This is used as a starting point for merging multiple results. + pub(crate) fn no_bindings() -> Self { + Self { + // Corresponds to "definitely unbound". Before we haven't seen any bindings, we + // can conclude that the symbol is not bound. + any_always_false: true, + // Corresponds to "definitely bound". Before we haven't seen any bindings, we + // can not conclude that the symbol is bound. + all_always_true: false, + // Irrelevant for this analysis. + at_least_one_condition: false, + } + } } From a17148ebd058fca59d6babb2aa00db008e1dcf1a Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 15:10:13 +0100 Subject: [PATCH 38/68] Remove Clone --- crates/red_knot_python_semantic/src/semantic_index/use_def.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index ff0b6fb8ed0bf..305b02eeb70e2 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -379,7 +379,7 @@ enum SymbolDefinitions { Declarations(SymbolDeclarations), } -#[derive(Debug, Clone)] +#[derive(Debug)] pub(crate) struct BindingWithConstraintsIterator<'map, 'db> { all_definitions: &'map IndexVec>, all_constraints: &'map IndexVec>, From b5fea25461564fb0c92b0690042e7e8626820286 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 15:15:15 +0100 Subject: [PATCH 39/68] Add documentation --- .../src/semantic_index/use_def.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 305b02eeb70e2..1c54704bbfa44 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -289,10 +289,10 @@ impl<'db> UseDefMap<'db> { use_id: ScopedUseId, ) -> Option { let bindings = &self.bindings_by_use[use_id]; - let conditions = self + let conditions_per_binding = self .bindings_iterator(bindings) .map(|binding| binding.branching_conditions); - analyze_boundness(db, conditions, bindings.may_be_unbound()) + analyze_boundness(db, conditions_per_binding, bindings.may_be_unbound()) } pub(crate) fn public_bindings( @@ -413,7 +413,6 @@ pub(crate) struct BindingWithConstraints<'map, 'db> { pub(crate) branching_conditions: BranchingConditionsIterator<'map, 'db>, } -#[derive(Debug, Clone)] pub(crate) struct ConstraintsIterator<'map, 'db> { all_constraints: &'map IndexVec>, constraint_ids: ConstraintIdIterator<'map>, @@ -431,7 +430,6 @@ impl<'db> Iterator for ConstraintsIterator<'_, 'db> { impl std::iter::FusedIterator for ConstraintsIterator<'_, '_> {} -#[derive(Debug, Clone)] pub(crate) struct BranchingConditionsIterator<'map, 'db> { all_branching_conditions: &'map IndexVec>, branching_condition_ids: BranchingConditionIdIterator<'map>, @@ -460,8 +458,8 @@ pub(crate) struct DeclarationsIterator<'map, 'db> { impl DeclarationsIterator<'_, '_> { pub(crate) fn declaredness(self, db: &dyn crate::db::Db) -> Option { let may_be_undeclared = self.may_be_undeclared; - let conditions = self.map(|(_, conditions)| conditions); - analyze_boundness(db, conditions, may_be_undeclared) + let conditions_per_binding = self.map(|(_, conditions)| conditions); + analyze_boundness(db, conditions_per_binding, may_be_undeclared) } pub(crate) fn may_be_undeclared(self, db: &dyn crate::db::Db) -> bool { @@ -550,6 +548,8 @@ impl<'db> UseDefMapBuilder<'db> { self.record_branching_condition(BranchingCondition::ConditionalOn(constraint)); } + /// Marks a point in control-flow where we branch unconditionally, that is: without any + /// conditions that could be statically analyzed. Examples are `for` loops or `try` blocks. pub(super) fn record_unconditional_branching(&mut self) { self.record_branching_condition(BranchingCondition::Ambiguous); } From d2cd3dea1715acfd1fb4c596cb15a2ca9390f0e7 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 15:27:30 +0100 Subject: [PATCH 40/68] Add doc comment --- .../src/semantic_index/use_def.rs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 1c54704bbfa44..38fcca0ea6dfa 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -667,6 +667,43 @@ impl<'db> UseDefMapBuilder<'db> { } } +/// Analyze the boundness (or declaredness) of a symbol based on all the branching conditions +/// that were active for each of its bindings (or declarations). +/// +/// Returns `None` if the symbol is definitely unbound. +/// +/// Consider this example: +/// ```py +/// if test: +/// x = 1 +/// ``` +/// +/// Depending on the static truthiness of `test`, `x` could either be definitely bound (if `test` +/// is always true), definitely unbound (if `test` is always false), or possibly unbound (if the +/// truthiness of `test` is ambiguous). +/// +/// If there are multiple bindings, the results need to be merged: +/// ```py +/// if test1: +/// x = 1 +/// if test2: +/// x = 2 +/// ``` +/// +/// Here, `x` is definitely bound if `test2` is always true OR if `test1` is always true. `x` is +/// definitely unbound if `test1` is always false AND `test2` is always false. And `x` is possibly +/// unbound in all other cases. +/// +/// Finally, we also need to consider that a symbol could be definitely bound, even if we can not +/// statically infer the truthiness of a test condition. On such example is: +/// ```py +/// if test: +/// x = 1 +/// else: +/// x = 2 +/// ``` +/// Here, `x` is definitely bound, no matter the value of `test`. The `may_be_unbound` flag from +/// semantic index building is used to determine this (with a value of `false` for this case). fn analyze_boundness<'db, 'map, C>( db: &dyn crate::db::Db, conditions_per_binding: C, From fcae0926fa191983486d98a5c49e4bda346d3e09 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 15:29:59 +0100 Subject: [PATCH 41/68] Clean up code in infer.rs --- crates/red_knot_python_semantic/src/types/infer.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index b03ae5ee41480..49fcfd2d12feb 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -3022,7 +3022,9 @@ impl<'db> TypeInferenceBuilder<'db> { use_def.use_boundness(self.db, use_id), ) }; - if boundness == Some(Boundness::PossiblyUnbound) || boundness.is_none() { + if boundness == Some(Boundness::Bound) { + bindings_ty.unwrap_or(Type::Unknown) + } else { match self.lookup_name(name) { Symbol::Type(looked_up_ty, looked_up_boundness) => { if looked_up_boundness == Boundness::PossiblyUnbound { @@ -3046,10 +3048,6 @@ impl<'db> TypeInferenceBuilder<'db> { bindings_ty.unwrap_or(Type::Unknown) } } - } else - /*if boundness == Some(Boundness::Bound) */ - { - bindings_ty.unwrap_or(Type::Unknown) } } From e29f37b4ae69f19f9a105f700ce4f9966c9cc678 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 16:10:51 +0100 Subject: [PATCH 42/68] Update bitset --- .../src/semantic_index/use_def/bitset.rs | 114 ++++++------------ 1 file changed, 40 insertions(+), 74 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs index 09f74f8f72f94..2064ca353898c 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs @@ -93,20 +93,6 @@ impl BitSet { } } - /// Union in-place with another [`BitSet`]. - #[allow(dead_code)] - pub(super) fn union(&mut self, other: &BitSet) { - let mut max_len = self.blocks().len(); - let other_len = other.blocks().len(); - if other_len > max_len { - max_len = other_len; - self.resize_blocks(max_len); - } - for (my_block, other_block) in self.blocks_mut().iter_mut().zip(other.blocks()) { - *my_block |= other_block; - } - } - /// Return an iterator over the values (in ascending order) in this [`BitSet`]. pub(super) fn iter(&self) -> BitSetIterator<'_, B> { let blocks = self.blocks(); @@ -118,11 +104,15 @@ impl BitSet { } pub(super) fn iter_rev(&self) -> ReverseBitSetIterator<'_, B> { + let num_blocks = self.blocks().len(); + + debug_assert!(num_blocks > 0); + let blocks = self.blocks(); ReverseBitSetIterator { blocks, - current_block_index: self.blocks().len() - 1, - current_block: blocks[self.blocks().len() - 1], + current_block_index: num_blocks - 1, + current_block: blocks[num_blocks - 1], } } } @@ -164,7 +154,7 @@ impl Iterator for BitSetIterator<'_, B> { impl std::iter::FusedIterator for BitSetIterator<'_, B> {} -/// Iterator over values in a [`BitSet`]. +/// Iterates over values in a [`BitSet`], in reverse order (highest bit first). #[derive(Debug, Clone)] pub(super) struct ReverseBitSetIterator<'a, const B: usize> { /// The blocks we are iterating over. @@ -188,12 +178,14 @@ impl Iterator for ReverseBitSetIterator<'_, B> { self.current_block_index -= 1; self.current_block = self.blocks[self.current_block_index]; } - - let highest_bit_set = 63 - u64::from(self.current_block.leading_zeros()); - // TODO: efficiency, safety comment, etc. + // SAFETY: current_block is non-zero, so leading_zeros must be + // strictly less than 64. + let highest_bit_set = 63 - self.current_block.leading_zeros(); + // reset the highest bit self.current_block &= !(1u64 << highest_bit_set); + // SAFETY: see above #[allow(clippy::cast_possible_truncation)] - Some(highest_bit_set as u32 + (64 * self.current_block_index) as u32) + Some(highest_bit_set + (64 * self.current_block_index) as u32) } } @@ -279,59 +271,6 @@ mod tests { assert_bitset(&b1, &[89]); } - #[test] - fn union() { - let mut b1 = BitSet::<1>::with(2); - let b2 = BitSet::<1>::with(4); - - b1.union(&b2); - assert_bitset(&b1, &[2, 4]); - } - - #[test] - fn union_mixed_1() { - let mut b1 = BitSet::<1>::with(4); - let mut b2 = BitSet::<1>::with(4); - b1.insert(89); - b2.insert(5); - - b1.union(&b2); - assert_bitset(&b1, &[4, 5, 89]); - } - - #[test] - fn union_mixed_2() { - let mut b1 = BitSet::<1>::with(4); - let mut b2 = BitSet::<1>::with(4); - b1.insert(23); - b2.insert(89); - - b1.union(&b2); - assert_bitset(&b1, &[4, 23, 89]); - } - - #[test] - fn union_heap() { - let mut b1 = BitSet::<1>::with(4); - let mut b2 = BitSet::<1>::with(4); - b1.insert(89); - b2.insert(90); - - b1.union(&b2); - assert_bitset(&b1, &[4, 89, 90]); - } - - #[test] - fn union_heap_2() { - let mut b1 = BitSet::<1>::with(89); - let mut b2 = BitSet::<1>::with(89); - b1.insert(91); - b2.insert(90); - - b1.union(&b2); - assert_bitset(&b1, &[89, 90, 91]); - } - #[test] fn multiple_blocks() { let mut b = BitSet::<2>::with(120); @@ -339,4 +278,31 @@ mod tests { assert!(matches!(b, BitSet::Inline(_))); assert_bitset(&b, &[45, 120]); } + + #[test] + fn reverse_iterator() { + let empty = BitSet::<1>::default(); + assert!(empty.iter_rev().next().is_none()); + + let single_element = BitSet::<1>::with(10); + assert_eq!(single_element.iter_rev().collect::>(), vec![10]); + + let mut single_block = BitSet::<1>::with(1); + single_block.insert(2); + single_block.insert(10); + assert_eq!(single_block.iter_rev().collect::>(), vec![10, 2, 1]); + + let mut multiple_blocks = BitSet::<1>::default(); + multiple_blocks.insert(1); + multiple_blocks.insert(2); + multiple_blocks.insert(3); + multiple_blocks.insert(70); + multiple_blocks.insert(71); + multiple_blocks.insert(1000); + assert!(matches!(multiple_blocks, BitSet::Heap(_))); + assert_eq!( + multiple_blocks.iter_rev().collect::>(), + vec![1000, 71, 70, 3, 2, 1] + ); + } } From 0ec165f9c174c10b1287224d170407b517ca2a23 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 16:28:49 +0100 Subject: [PATCH 43/68] Reinstate tests --- .../src/semantic_index/use_def.rs | 4 +- .../semantic_index/use_def/symbol_state.rs | 449 +++++++++--------- 2 files changed, 237 insertions(+), 216 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 38fcca0ea6dfa..d1aaa620cd3fd 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -355,7 +355,7 @@ impl<'db> UseDefMap<'db> { all_definitions: &self.all_definitions, all_constraints: &self.all_constraints, all_branching_conditions: &self.all_branching_conditions, - inner: bindings.iter(), + inner: bindings.iter_rev(), } } @@ -366,7 +366,7 @@ impl<'db> UseDefMap<'db> { DeclarationsIterator { all_definitions: &self.all_definitions, all_branching_conditions: &self.all_branching_conditions, - inner: declarations.iter(), + inner: declarations.iter_rev(), may_be_undeclared: declarations.may_be_undeclared(), } } diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index ea89967c38895..de04c2ac7d4eb 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -109,6 +109,7 @@ pub(super) struct SymbolDeclarations { /// [`BitSet`]: which declarations (as [`ScopedDefinitionId`]) can reach the current location? live_declarations: Declarations, + /// For each live declaration, which [`BranchingConditions`] were active at that declaration? branching_conditions: BranchingConditionsPerBinding, /// Could the symbol be un-declared at this point? @@ -133,9 +134,9 @@ impl SymbolDeclarations { self.live_declarations = Declarations::with(declaration_id.into()); self.may_be_undeclared = false; - // TODO: unify code with below self.branching_conditions = BranchingConditionsPerBinding::with_capacity(1); - self.branching_conditions.push(BitSet::default()); + self.branching_conditions + .push(BranchingConditions::default()); for active_constraint_id in branching_conditions.iter() { self.branching_conditions[0].insert(active_constraint_id); } @@ -147,7 +148,7 @@ impl SymbolDeclarations { } /// Return an iterator over live declarations for this symbol. - pub(super) fn iter(&self) -> DeclarationIdIterator { + pub(super) fn iter_rev(&self) -> DeclarationIdIterator { DeclarationIdIterator { inner: self.live_declarations.iter_rev(), branching_conditions: self.branching_conditions.iter().rev(), @@ -171,7 +172,7 @@ pub(super) struct SymbolBindings { /// binding in `live_bindings`. constraints: ConstraintsPerBinding, - /// For each live binding, which [`BranchingConditions`] were active *at the time of the binding*? + /// For each live binding, which [`BranchingConditions`] were active at that binding? branching_conditions: BranchingConditionsPerBinding, /// Could the symbol be unbound at this point? @@ -203,10 +204,11 @@ impl SymbolBindings { // constraints. self.live_bindings = Bindings::with(binding_id.into()); self.constraints = ConstraintsPerBinding::with_capacity(1); - self.constraints.push(BitSet::default()); + self.constraints.push(Constraints::default()); self.branching_conditions = BranchingConditionsPerBinding::with_capacity(1); - self.branching_conditions.push(BitSet::default()); + self.branching_conditions + .push(BranchingConditions::default()); for id in branching_conditions.iter() { self.branching_conditions[0].insert(id); } @@ -220,8 +222,8 @@ impl SymbolBindings { } } - /// Iterate over currently live bindings for this symbol. - pub(super) fn iter(&self) -> BindingIdWithConstraintsIterator { + /// Iterate over currently live bindings for this symbol, in reverse order. + pub(super) fn iter_rev(&self) -> BindingIdWithConstraintsIterator { BindingIdWithConstraintsIterator { definitions: self.live_bindings.iter_rev(), constraints: self.constraints.iter().rev(), @@ -303,70 +305,12 @@ impl SymbolState { std::mem::swap(&mut a, self); - let mut a_decls_iter = a.declarations.live_declarations.iter(); - let mut b_decls_iter = b.declarations.live_declarations.iter(); - let mut a_declaration_branching_conditions_iter = - a.declarations.branching_conditions.into_iter(); - let mut b_cdeclaration_branching_conditions_iter = - b.declarations.branching_conditions.into_iter(); - - let mut opt_a_decl: Option = a_decls_iter.next(); - let mut opt_b_decl: Option = b_decls_iter.next(); - - let push = |decl, - declaration_branching_conditions_iter: &mut BranchingConditionsIntoIterator, - merged: &mut Self| { - merged.declarations.live_declarations.insert(decl); - let branching_conditions = declaration_branching_conditions_iter - .next() - .expect("declarations and branching_conditions length mismatch"); - merged - .declarations - .branching_conditions - .push(branching_conditions); - }; - - loop { - match (opt_a_decl, opt_b_decl) { - (Some(a_decl), Some(b_decl)) => match a_decl.cmp(&b_decl) { - std::cmp::Ordering::Less => { - push(a_decl, &mut a_declaration_branching_conditions_iter, self); - opt_a_decl = a_decls_iter.next(); - } - std::cmp::Ordering::Greater => { - push(b_decl, &mut b_cdeclaration_branching_conditions_iter, self); - opt_b_decl = b_decls_iter.next(); - } - std::cmp::Ordering::Equal => { - push(a_decl, &mut b_cdeclaration_branching_conditions_iter, self); - self.declarations - .branching_conditions - .last_mut() - .unwrap() - .intersect(&a_declaration_branching_conditions_iter.next().unwrap()); - - opt_a_decl = a_decls_iter.next(); - opt_b_decl = b_decls_iter.next(); - } - }, - (Some(a_decl), None) => { - push(a_decl, &mut a_declaration_branching_conditions_iter, self); - opt_a_decl = a_decls_iter.next(); - } - (None, Some(b_decl)) => { - push(b_decl, &mut b_cdeclaration_branching_conditions_iter, self); - opt_b_decl = b_decls_iter.next(); - } - (None, None) => break, - } - } - let mut a_defs_iter = a.bindings.live_bindings.iter(); let mut b_defs_iter = b.bindings.live_bindings.iter(); let mut a_constraints_iter = a.bindings.constraints.into_iter(); let mut b_constraints_iter = b.bindings.constraints.into_iter(); - let mut a_binding_branching_conditions_iter = a.bindings.branching_conditions.into_iter(); - let mut b_binding_branching_conditions_iter = b.bindings.branching_conditions.into_iter(); + let mut a_conditions_iter = a.bindings.branching_conditions.into_iter(); + let mut b_conditions_iter = b.bindings.branching_conditions.into_iter(); let mut opt_a_def: Option = a_defs_iter.next(); let mut opt_b_def: Option = b_defs_iter.next(); @@ -406,32 +350,17 @@ impl SymbolState { (Some(a_def), Some(b_def)) => match a_def.cmp(&b_def) { std::cmp::Ordering::Less => { // Next definition ID is only in `a`, push it to `self` and advance `a`. - push( - a_def, - &mut a_constraints_iter, - &mut a_binding_branching_conditions_iter, - self, - ); + push(a_def, &mut a_constraints_iter, &mut a_conditions_iter, self); opt_a_def = a_defs_iter.next(); } std::cmp::Ordering::Greater => { // Next definition ID is only in `b`, push it to `self` and advance `b`. - push( - b_def, - &mut b_constraints_iter, - &mut b_binding_branching_conditions_iter, - self, - ); + push(b_def, &mut b_constraints_iter, &mut b_conditions_iter, self); opt_b_def = b_defs_iter.next(); } std::cmp::Ordering::Equal => { // Next definition is in both; push to `self` and intersect constraints. - push( - a_def, - &mut b_constraints_iter, - &mut b_binding_branching_conditions_iter, - self, - ); + push(a_def, &mut b_constraints_iter, &mut b_conditions_iter, self); // SAFETY: we only ever create SymbolState with either no definitions and // no constraint bitsets (`::unbound`) or one definition and one constraint // bitset (`::with`), and `::merge` always pushes one definition and one @@ -457,27 +386,70 @@ impl SymbolState { }, (Some(a_def), None) => { // We've exhausted `b`, just push the def from `a` and move on to the next. - push( - a_def, - &mut a_constraints_iter, - &mut a_binding_branching_conditions_iter, - self, - ); + push(a_def, &mut a_constraints_iter, &mut a_conditions_iter, self); opt_a_def = a_defs_iter.next(); } (None, Some(b_def)) => { // We've exhausted `a`, just push the def from `b` and move on to the next. - push( - b_def, - &mut b_constraints_iter, - &mut b_binding_branching_conditions_iter, - self, - ); + push(b_def, &mut b_constraints_iter, &mut b_conditions_iter, self); opt_b_def = b_defs_iter.next(); } (None, None) => break, } } + + // Same as above, but for declarations. + let mut a_decls_iter = a.declarations.live_declarations.iter(); + let mut b_decls_iter = b.declarations.live_declarations.iter(); + let mut a_conditions_iter = a.declarations.branching_conditions.into_iter(); + let mut b_conditions_iter = b.declarations.branching_conditions.into_iter(); + + let mut opt_a_decl: Option = a_decls_iter.next(); + let mut opt_b_decl: Option = b_decls_iter.next(); + + let push = + |decl, conditions_iter: &mut BranchingConditionsIntoIterator, merged: &mut Self| { + merged.declarations.live_declarations.insert(decl); + let conditions = conditions_iter + .next() + .expect("declarations and branching_conditions length mismatch"); + merged.declarations.branching_conditions.push(conditions); + }; + + loop { + match (opt_a_decl, opt_b_decl) { + (Some(a_decl), Some(b_decl)) => match a_decl.cmp(&b_decl) { + std::cmp::Ordering::Less => { + push(a_decl, &mut a_conditions_iter, self); + opt_a_decl = a_decls_iter.next(); + } + std::cmp::Ordering::Greater => { + push(b_decl, &mut b_conditions_iter, self); + opt_b_decl = b_decls_iter.next(); + } + std::cmp::Ordering::Equal => { + push(a_decl, &mut b_conditions_iter, self); + self.declarations + .branching_conditions + .last_mut() + .unwrap() + .intersect(&a_conditions_iter.next().unwrap()); + + opt_a_decl = a_decls_iter.next(); + opt_b_decl = b_decls_iter.next(); + } + }, + (Some(a_decl), None) => { + push(a_decl, &mut a_conditions_iter, self); + opt_a_decl = a_decls_iter.next(); + } + (None, Some(b_decl)) => { + push(b_decl, &mut b_conditions_iter, self); + opt_b_decl = b_decls_iter.next(); + } + (None, None) => break, + } + } } pub(super) fn bindings(&self) -> &SymbolBindings { @@ -606,13 +578,14 @@ impl std::iter::FusedIterator for DeclarationIdIterator<'_> {} #[cfg(test)] mod tests { - use super::{ScopedConstraintId, SymbolState}; + use super::*; + #[track_caller] fn assert_bindings(symbol: &SymbolState, may_be_unbound: bool, expected: &[&str]) { assert_eq!(symbol.may_be_unbound(), may_be_unbound); - let actual = symbol + let mut actual = symbol .bindings() - .iter() + .iter_rev() .map(|def_id_with_constraints| { format!( "{}<{}>", @@ -626,20 +599,23 @@ mod tests { ) }) .collect::>(); + actual.reverse(); assert_eq!(actual, expected); } + #[track_caller] pub(crate) fn assert_declarations( symbol: &SymbolState, may_be_undeclared: bool, expected: &[u32], ) { assert_eq!(symbol.declarations.may_be_undeclared(), may_be_undeclared); - let actual = symbol + let mut actual = symbol .declarations() - .iter() - .map(|(d, _)| d.as_u32()) // TODO: constraints + .iter_rev() + .map(|(d, _)| d.as_u32()) .collect::>(); + actual.reverse(); assert_eq!(actual, expected); } @@ -650,76 +626,100 @@ mod tests { assert_bindings(&sym, true, &[]); } - // #[test] - // fn with() { - // let mut sym = SymbolState::undefined(); - // sym.record_binding(ScopedDefinitionId::from_u32(0)); - - // assert_bindings(&sym, false, &["0<>"]); - // } - - // #[test] - // fn set_may_be_unbound() { - // let mut sym = SymbolState::undefined(); - // sym.record_binding(ScopedDefinitionId::from_u32(0)); - // sym.set_may_be_unbound(); - - // assert_bindings(&sym, true, &["0<>"]); - // } - - // #[test] - // fn record_constraint() { - // let mut sym = SymbolState::undefined(); - // sym.record_binding(ScopedDefinitionId::from_u32(0)); - // sym.record_constraint(ScopedConstraintId::from_u32(0)); - - // assert_bindings(&sym, false, &["0<0>"]); - // } - - // #[test] - // fn merge() { - // // merging the same definition with the same constraint keeps the constraint - // let mut sym0a = SymbolState::undefined(); - // sym0a.record_binding(ScopedDefinitionId::from_u32(0)); - // sym0a.record_constraint(ScopedConstraintId::from_u32(0)); - - // let mut sym0b = SymbolState::undefined(); - // sym0b.record_binding(ScopedDefinitionId::from_u32(0)); - // sym0b.record_constraint(ScopedConstraintId::from_u32(0)); - - // sym0a.merge(sym0b); - // let mut sym0 = sym0a; - // assert_bindings(&sym0, false, &["0<0>"]); - - // // merging the same definition with differing constraints drops all constraints - // let mut sym1a = SymbolState::undefined(); - // sym1a.record_binding(ScopedDefinitionId::from_u32(1)); - // sym1a.record_constraint(ScopedConstraintId::from_u32(1)); - - // let mut sym1b = SymbolState::undefined(); - // sym1b.record_binding(ScopedDefinitionId::from_u32(1)); - // sym1b.record_constraint(ScopedConstraintId::from_u32(2)); - - // sym1a.merge(sym1b); - // let sym1 = sym1a; - // assert_bindings(&sym1, false, &["1<>"]); - - // // merging a constrained definition with unbound keeps both - // let mut sym2a = SymbolState::undefined(); - // sym2a.record_binding(ScopedDefinitionId::from_u32(2)); - // sym2a.record_constraint(ScopedConstraintId::from_u32(3)); + #[test] + fn with() { + let mut sym = SymbolState::undefined(); + sym.record_binding( + ScopedDefinitionId::from_u32(0), + &BranchingConditions::default(), + ); + + assert_bindings(&sym, false, &["0<>"]); + } - // let sym2b = SymbolState::undefined(); + #[test] + fn set_may_be_unbound() { + let mut sym = SymbolState::undefined(); + sym.record_binding( + ScopedDefinitionId::from_u32(0), + &BranchingConditions::default(), + ); + sym.set_may_be_unbound(); + + assert_bindings(&sym, true, &["0<>"]); + } - // sym2a.merge(sym2b); - // let sym2 = sym2a; - // assert_bindings(&sym2, true, &["2<3>"]); + #[test] + fn record_constraint() { + let mut sym = SymbolState::undefined(); + sym.record_binding( + ScopedDefinitionId::from_u32(0), + &BranchingConditions::default(), + ); + sym.record_constraint(ScopedConstraintId::from_u32(0)); + + assert_bindings(&sym, false, &["0<0>"]); + } - // // merging different definitions keeps them each with their existing constraints - // sym0.merge(sym2); - // let sym = sym0; - // assert_bindings(&sym, true, &["0<0>", "2<3>"]); - // } + #[test] + fn merge() { + // merging the same definition with the same constraint keeps the constraint + let mut sym0a = SymbolState::undefined(); + sym0a.record_binding( + ScopedDefinitionId::from_u32(0), + &BranchingConditions::default(), + ); + sym0a.record_constraint(ScopedConstraintId::from_u32(0)); + + let mut sym0b = SymbolState::undefined(); + sym0b.record_binding( + ScopedDefinitionId::from_u32(0), + &BranchingConditions::default(), + ); + sym0b.record_constraint(ScopedConstraintId::from_u32(0)); + + sym0a.merge(sym0b); + let mut sym0 = sym0a; + assert_bindings(&sym0, false, &["0<0>"]); + + // merging the same definition with differing constraints drops all constraints + let mut sym1a = SymbolState::undefined(); + sym1a.record_binding( + ScopedDefinitionId::from_u32(1), + &BranchingConditions::default(), + ); + sym1a.record_constraint(ScopedConstraintId::from_u32(1)); + + let mut sym1b = SymbolState::undefined(); + sym1b.record_binding( + ScopedDefinitionId::from_u32(1), + &BranchingConditions::default(), + ); + sym1b.record_constraint(ScopedConstraintId::from_u32(2)); + + sym1a.merge(sym1b); + let sym1 = sym1a; + assert_bindings(&sym1, false, &["1<>"]); + + // merging a constrained definition with unbound keeps both + let mut sym2a = SymbolState::undefined(); + sym2a.record_binding( + ScopedDefinitionId::from_u32(2), + &BranchingConditions::default(), + ); + sym2a.record_constraint(ScopedConstraintId::from_u32(3)); + + let sym2b = SymbolState::undefined(); + + sym2a.merge(sym2b); + let sym2 = sym2a; + assert_bindings(&sym2, true, &["2<3>"]); + + // merging different definitions keeps them each with their existing constraints + sym0.merge(sym2); + let sym = sym0; + assert_bindings(&sym, true, &["0<0>", "2<3>"]); + } #[test] fn no_declaration() { @@ -728,54 +728,75 @@ mod tests { assert_declarations(&sym, true, &[]); } - // #[test] - // fn record_declaration() { - // let mut sym = SymbolState::undefined(); - // sym.record_declaration(ScopedDefinitionId::from_u32(1)); - - // assert_declarations(&sym, false, &[1]); - // } - - // #[test] - // fn record_declaration_override() { - // let mut sym = SymbolState::undefined(); - // sym.record_declaration(ScopedDefinitionId::from_u32(1)); - // sym.record_declaration(ScopedDefinitionId::from_u32(2)); - - // assert_declarations(&sym, false, &[2]); - // } - - // #[test] - // fn record_declaration_merge() { - // let mut sym = SymbolState::undefined(); - // sym.record_declaration(ScopedDefinitionId::from_u32(1)); - - // let mut sym2 = SymbolState::undefined(); - // sym2.record_declaration(ScopedDefinitionId::from_u32(2)); - - // sym.merge(sym2); + #[test] + fn record_declaration() { + let mut sym = SymbolState::undefined(); + sym.record_declaration( + ScopedDefinitionId::from_u32(1), + &BranchingConditions::default(), + ); + + assert_declarations(&sym, false, &[1]); + } - // assert_declarations(&sym, false, &[1, 2]); - // } + #[test] + fn record_declaration_override() { + let mut sym = SymbolState::undefined(); + sym.record_declaration( + ScopedDefinitionId::from_u32(1), + &BranchingConditions::default(), + ); + sym.record_declaration( + ScopedDefinitionId::from_u32(2), + &BranchingConditions::default(), + ); + + assert_declarations(&sym, false, &[2]); + } - // #[test] - // fn record_declaration_merge_partial_undeclared() { - // let mut sym = SymbolState::undefined(); - // sym.record_declaration(ScopedDefinitionId::from_u32(1)); + #[test] + fn record_declaration_merge() { + let mut sym = SymbolState::undefined(); + sym.record_declaration( + ScopedDefinitionId::from_u32(1), + &BranchingConditions::default(), + ); + + let mut sym2 = SymbolState::undefined(); + sym2.record_declaration( + ScopedDefinitionId::from_u32(2), + &BranchingConditions::default(), + ); + + sym.merge(sym2); + + assert_declarations(&sym, false, &[1, 2]); + } - // let sym2 = SymbolState::undefined(); + #[test] + fn record_declaration_merge_partial_undeclared() { + let mut sym = SymbolState::undefined(); + sym.record_declaration( + ScopedDefinitionId::from_u32(1), + &BranchingConditions::default(), + ); - // sym.merge(sym2); + let sym2 = SymbolState::undefined(); - // assert_declarations(&sym, true, &[1]); - // } + sym.merge(sym2); - // #[test] - // fn set_may_be_undeclared() { - // let mut sym = SymbolState::undefined(); - // sym.record_declaration(ScopedDefinitionId::from_u32(0)); - // sym.set_may_be_undeclared(); + assert_declarations(&sym, true, &[1]); + } - // assert_declarations(&sym, true, &[0]); - // } + #[test] + fn set_may_be_undeclared() { + let mut sym = SymbolState::undefined(); + sym.record_declaration( + ScopedDefinitionId::from_u32(0), + &BranchingConditions::default(), + ); + sym.set_may_be_undeclared(); + + assert_declarations(&sym, true, &[0]); + } } From 86747edd6381a3ae43076d5d2562b2773e9c99eb Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 16:44:50 +0100 Subject: [PATCH 44/68] Minor fixes --- .../semantic_index/use_def/symbol_state.rs | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index de04c2ac7d4eb..8558424f8c927 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -43,9 +43,7 @@ //! //! Tracking live declarations is simpler, since constraints are not involved, but otherwise very //! similar to tracking live bindings. -use crate::semantic_index::use_def::bitset::ReverseBitSetIterator; - -use super::bitset::{BitSet, BitSetIterator}; +use super::bitset::{BitSet, BitSetIterator, ReverseBitSetIterator}; use ruff_index::newtype_index; use smallvec::SmallVec; @@ -66,14 +64,14 @@ const INLINE_BINDING_BLOCKS: usize = 3; /// A [`BitSet`] of [`ScopedDefinitionId`], representing live bindings of a symbol in a scope. type Bindings = BitSet; -type BindingsIterator<'a> = ReverseBitSetIterator<'a, INLINE_BINDING_BLOCKS>; +type ReverseBindingsIterator<'a> = ReverseBitSetIterator<'a, INLINE_BINDING_BLOCKS>; /// Can reference this * 64 total declarations inline; more will fall back to the heap. const INLINE_DECLARATION_BLOCKS: usize = 3; /// A [`BitSet`] of [`ScopedDefinitionId`], representing live declarations of a symbol in a scope. type Declarations = BitSet; -type DeclarationsIterator<'a> = ReverseBitSetIterator<'a, INLINE_DECLARATION_BLOCKS>; +type ReverseDeclarationsIterator<'a> = ReverseBitSetIterator<'a, INLINE_DECLARATION_BLOCKS>; /// Can reference this * 64 total constraints inline; more will fall back to the heap. const INLINE_CONSTRAINT_BLOCKS: usize = 2; @@ -369,8 +367,10 @@ impl SymbolState { let a_constraints = a_constraints_iter .next() .expect("definitions and constraints length mismatch"); - // TODO: perform check that we see the same branching_conditions in both paths? - + // SAFETY: The same is true for branching_conditions. + a_conditions_iter + .next() + .expect("branching_conditions length mismatch"); // If the same definition is visible through both paths, any constraint // that applies on only one path is irrelevant to the resulting type from // unioning the two paths, so we intersect the constraints. @@ -418,27 +418,31 @@ impl SymbolState { loop { match (opt_a_decl, opt_b_decl) { - (Some(a_decl), Some(b_decl)) => match a_decl.cmp(&b_decl) { - std::cmp::Ordering::Less => { - push(a_decl, &mut a_conditions_iter, self); - opt_a_decl = a_decls_iter.next(); - } - std::cmp::Ordering::Greater => { - push(b_decl, &mut b_conditions_iter, self); - opt_b_decl = b_decls_iter.next(); - } - std::cmp::Ordering::Equal => { - push(a_decl, &mut b_conditions_iter, self); - self.declarations - .branching_conditions - .last_mut() - .unwrap() - .intersect(&a_conditions_iter.next().unwrap()); - - opt_a_decl = a_decls_iter.next(); - opt_b_decl = b_decls_iter.next(); + (Some(a_decl), Some(b_decl)) => { + match a_decl.cmp(&b_decl) { + std::cmp::Ordering::Less => { + push(a_decl, &mut a_conditions_iter, self); + opt_a_decl = a_decls_iter.next(); + } + std::cmp::Ordering::Greater => { + push(b_decl, &mut b_conditions_iter, self); + opt_b_decl = b_decls_iter.next(); + } + std::cmp::Ordering::Equal => { + push(a_decl, &mut b_conditions_iter, self); + self.declarations + .branching_conditions + .last_mut() + .expect("declarations and branching_conditions length mismatch") + .intersect(&a_conditions_iter.next().expect( + "declarations and branching_conditions length mismatch", + )); + + opt_a_decl = a_decls_iter.next(); + opt_b_decl = b_decls_iter.next(); + } } - }, + } (Some(a_decl), None) => { push(a_decl, &mut a_conditions_iter, self); opt_a_decl = a_decls_iter.next(); @@ -459,12 +463,6 @@ impl SymbolState { pub(super) fn declarations(&self) -> &SymbolDeclarations { &self.declarations } - - /// Could the symbol be unbound? - #[cfg(test)] - pub(super) fn may_be_unbound(&self) -> bool { - self.bindings.may_be_unbound() - } } /// The default state of a symbol, if we've seen no definitions of it, is undefined (that is, @@ -484,9 +482,9 @@ pub(super) struct BindingIdWithConstraints<'a> { pub(super) branching_conditions_ids: BranchingConditionIdIterator<'a>, } -#[derive(Debug, Clone)] +#[derive(Debug)] pub(super) struct BindingIdWithConstraintsIterator<'a> { - definitions: BindingsIterator<'a>, + definitions: ReverseBindingsIterator<'a>, constraints: std::iter::Rev>, branching_conditions: std::iter::Rev>, } @@ -518,7 +516,9 @@ impl<'a> Iterator for BindingIdWithConstraintsIterator<'a> { } } -#[derive(Debug, Clone)] +impl std::iter::FusedIterator for BindingIdWithConstraintsIterator<'_> {} + +#[derive(Debug)] pub(super) struct ConstraintIdIterator<'a> { wrapped: BitSetIterator<'a, INLINE_CONSTRAINT_BLOCKS>, } @@ -550,9 +550,9 @@ impl Iterator for BranchingConditionIdIterator<'_> { impl std::iter::FusedIterator for BranchingConditionIdIterator<'_> {} -#[derive(Debug, Clone)] +#[derive(Clone)] pub(super) struct DeclarationIdIterator<'a> { - inner: DeclarationsIterator<'a>, + inner: ReverseDeclarationsIterator<'a>, branching_conditions: std::iter::Rev>, } @@ -582,7 +582,7 @@ mod tests { #[track_caller] fn assert_bindings(symbol: &SymbolState, may_be_unbound: bool, expected: &[&str]) { - assert_eq!(symbol.may_be_unbound(), may_be_unbound); + assert_eq!(symbol.bindings.may_be_unbound, may_be_unbound); let mut actual = symbol .bindings() .iter_rev() From 42da9ee585ddbd2c515037126973e6e578061621 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 16:49:38 +0100 Subject: [PATCH 45/68] Minor --- crates/red_knot_python_semantic/src/semantic_index/use_def.rs | 2 +- crates/red_knot_python_semantic/src/types/static_truthiness.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index d1aaa620cd3fd..66ac5b1333479 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -714,7 +714,7 @@ where C: Iterator>, { let result = conditions_per_binding.fold(StaticTruthiness::no_bindings(), |r, conditions| { - r.merge(StaticTruthiness::analyze(db, conditions)) + r.merge(&StaticTruthiness::analyze(db, conditions)) }); let definitely_unbound = result.any_always_false; diff --git a/crates/red_knot_python_semantic/src/types/static_truthiness.rs b/crates/red_knot_python_semantic/src/types/static_truthiness.rs index 01b8935d6aedb..18c567150fc29 100644 --- a/crates/red_knot_python_semantic/src/types/static_truthiness.rs +++ b/crates/red_knot_python_semantic/src/types/static_truthiness.rs @@ -110,7 +110,7 @@ impl StaticTruthiness { /// binding of a symbol is unconditionally visible, if all branching conditions are known to be /// statically true. It is enough if this is the case for *any* of the control-flow paths. Other /// control flow paths will not be taken if this is the case. - pub(crate) fn merge(self, other: Self) -> Self { + pub(crate) fn merge(self, other: &Self) -> Self { Self { any_always_false: self.any_always_false && other.any_always_false, all_always_true: self.all_always_true || other.all_always_true, From 33b913610f9fd415d1ba65456b15d8f290f1aeeb Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 16:53:20 +0100 Subject: [PATCH 46/68] Bring back TODO comment --- .../src/semantic_index/use_def/symbol_state.rs | 1 - crates/red_knot_python_semantic/src/types.rs | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 8558424f8c927..6f36593e8463d 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -379,7 +379,6 @@ impl SymbolState { .last_mut() .unwrap() .intersect(&a_constraints); - opt_a_def = a_defs_iter.next(); opt_b_def = b_defs_iter.next(); } diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 41250b5da3770..cdfb9198ef4ee 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -75,6 +75,20 @@ fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolI // on inference from bindings. let declaredness = use_def.public_declarations(symbol).declaredness(db); + // TODO: Our handling of boundness currently only depends on bindings, and ignores + // declarations. This is inconsistent, since we only look at bindings if the symbol + // may be undeclared. Consider the following example: + // ```py + // x: int + // + // if flag: + // y: int + // else + // y = 3 + // ``` + // If we import from this module, we will currently report `x` as a definitely-bound + // symbol (even though it has no bindings at all!) but report `y` as possibly-unbound + // (even though every path has either a binding or a declaration for it.) let undeclared_ty = match declaredness { None => { return bindings_ty(db, use_def.public_bindings(symbol)) From c03be896f30ab3ca423917fff21bc1edbc68fc29 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 16:54:51 +0100 Subject: [PATCH 47/68] Remove Clone --- .../src/semantic_index/use_def/bitset.rs | 2 +- .../src/semantic_index/use_def/symbol_state.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs index 2064ca353898c..92dabe44a41f9 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs @@ -118,7 +118,7 @@ impl BitSet { } /// Iterator over values in a [`BitSet`]. -#[derive(Debug, Clone)] +#[derive(Debug)] pub(super) struct BitSetIterator<'a, const B: usize> { /// The blocks we are iterating over. blocks: &'a [u64], diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs index 6f36593e8463d..3ec9f468475b5 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs @@ -532,7 +532,7 @@ impl Iterator for ConstraintIdIterator<'_> { impl std::iter::FusedIterator for ConstraintIdIterator<'_> {} -#[derive(Debug, Clone)] +#[derive(Debug)] pub(super) struct BranchingConditionIdIterator<'a> { wrapped: BitSetIterator<'a, INLINE_BRANCHING_BLOCKS>, } From 72d09eb0925ab6bee76758014c5171db2d93d502 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 17:26:51 +0100 Subject: [PATCH 48/68] Visibility struct --- crates/red_knot_python_semantic/src/types.rs | 107 +++++++++++++------ 1 file changed, 73 insertions(+), 34 deletions(-) diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index cdfb9198ef4ee..b774fbf51acbd 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -251,10 +251,50 @@ fn definition_expression_ty<'db>( } } +/// The 'visibility' of a binding or declaration. +/// +/// Consider the following example: +/// ```py +/// x = 1 +/// +/// if True: +/// x = 2 +/// +/// if False: +/// x = 3 +/// +/// if flag(): +/// x = 4 +/// ``` +/// When we infer the type of `x`, we look back "through" the bindings in reverse order. +/// The first binding is `x = 4`. It is "transparent" because we could have either taken +/// the `if flag()` branch or not. The second binding `x = 3` is "invisible" because we +/// can statically determine that the `if False` branch is never taken. The third binding +/// `x = 2` is "opaque" because we can statically determine that the `if True` branch is +/// always taken. If the visibility of a binding is "opaque", bindings behind it are not +/// visible. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum UnconditionallyVisible { - Yes, - No, +enum Visibility<'db> { + Invisible, + Transparent(Type<'db>), + Opaque(Type<'db>), +} + +impl<'db> Visibility<'db> { + fn is_not_opaque(&self) -> bool { + !matches!(self, Visibility::Opaque(_)) + } + + fn is_invisible(&self) -> bool { + matches!(self, Visibility::Invisible) + } + + fn unwrap_or_never(self) -> Type<'db> { + match self { + Visibility::Invisible => Type::Never, + Visibility::Transparent(ty) | Visibility::Opaque(ty) => ty, + } + } } /// Infer the combined type of an iterator of bindings. @@ -264,7 +304,7 @@ fn bindings_ty<'db>( db: &'db dyn Db, bindings_with_constraints: BindingWithConstraintsIterator<'_, 'db>, ) -> Option> { - let def_types = bindings_with_constraints + let types = bindings_with_constraints .map( |BindingWithConstraints { binding, @@ -274,51 +314,50 @@ fn bindings_ty<'db>( let result = StaticTruthiness::analyze(db, branching_conditions); if result.any_always_false { - (None, UnconditionallyVisible::No) + Visibility::Invisible } else { - let unconditionally_visible = - if result.at_least_one_condition && result.all_always_true { - UnconditionallyVisible::Yes - } else { - UnconditionallyVisible::No - }; - let mut constraint_tys = constraints .filter_map(|constraint| narrowing_constraint(db, constraint, binding)) .peekable(); let binding_ty = binding_ty(db, binding); - if constraint_tys.peek().is_some() { + let ty = if constraint_tys.peek().is_some() { let intersection_ty = constraint_tys .fold( IntersectionBuilder::new(db).add_positive(binding_ty), IntersectionBuilder::add_positive, ) .build(); - (Some(intersection_ty), unconditionally_visible) + intersection_ty } else { - (Some(binding_ty), unconditionally_visible) + binding_ty + }; + + if result.at_least_one_condition && result.all_always_true { + Visibility::Opaque(ty) + } else { + Visibility::Transparent(ty) } } }, ) - .take_while_inclusive(|(_, uv)| *uv != UnconditionallyVisible::Yes) - .map(|(ty, _)| ty); + .take_while_inclusive(Visibility::is_not_opaque); - // TODO: get rid of all the collects and clean up, obviously - let def_types: Vec<_> = def_types.collect(); + // TODO: try to get rid of the `collect` here + let types: Vec<_> = types.collect(); - if !def_types.is_empty() && def_types.iter().all(Option::is_none) { + if !types.is_empty() && types.iter().all(Visibility::is_invisible) { + // If all bindings are invisible, the symbol is unbound. return Some(Type::Unknown); } - let mut def_types = def_types.iter().map(|ty| ty.unwrap_or(Type::Never)).rev(); + let mut types = types.iter().map(|v| v.unwrap_or_never()).rev(); - if let Some(first) = def_types.next() { - if let Some(second) = def_types.next() { + if let Some(first) = types.next() { + if let Some(second) = types.next() { Some(UnionType::from_elements( db, - [first, second].into_iter().chain(def_types), + [first, second].into_iter().chain(types), )) } else { Some(first) @@ -349,28 +388,28 @@ fn declarations_ty<'db>( declarations: DeclarationsIterator<'_, 'db>, undeclared_ty: Option>, ) -> DeclaredTypeResult<'db> { - let decl_types = declarations + let types = declarations .map(|(declaration, branching_conditions)| { let result = StaticTruthiness::analyze(db, branching_conditions); if result.any_always_false { - (Type::Never, UnconditionallyVisible::No) + Visibility::Invisible } else { if result.at_least_one_condition && result.all_always_true { - (declaration_ty(db, declaration), UnconditionallyVisible::Yes) + Visibility::Opaque(declaration_ty(db, declaration)) } else { - (declaration_ty(db, declaration), UnconditionallyVisible::No) + Visibility::Transparent(declaration_ty(db, declaration)) } } }) - .take_while_inclusive(|(_, uv)| *uv != UnconditionallyVisible::Yes) - .map(|(ty, _)| ty); - - let decl_types: Vec<_> = decl_types.collect(); + .take_while_inclusive(Visibility::is_not_opaque) + .map(|v| v.unwrap_or_never()); - let decl_types = decl_types.into_iter().rev(); + // TODO: try to get rid of the `collect` here (see above) + let types: Vec<_> = types.collect(); + let types = types.into_iter().rev(); - let mut all_types = undeclared_ty.into_iter().chain(decl_types); + let mut all_types = undeclared_ty.into_iter().chain(types); let first = all_types.next().expect( "declarations_ty must not be called with zero declarations and no may-be-undeclared", From 9c78c81f40f0c02c7a77e7f347817023aaf365f7 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 17:48:58 +0100 Subject: [PATCH 49/68] Add introduction section --- .../mdtest/statically-known-branches.md | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 082ace0b5808c..47bc8c20f04fa 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -1,5 +1,46 @@ # Statically-known branches +## Introduction + +We have the ability to infer precise types and boundness information for symbols that are +defined/declared in branches whose conditions we can statically determine to be always true or +always false. This is useful is for `sys.version_info` branches, which are often used to +conditionally make new features available: + +```py path=module1.py +if sys.version_info >= (3, 9): + SomeFeature = "available" +``` + +If we can statically determine that the condition is always true, we can determine that +`SomeFeature` is always bound, without raising any errors: + +```py path=test1.py +from module1 import SomeFeature + +# SomeFeature is unconditionally available here: +reveal_type(SomeFeature) # revealed: Literal["available"] +``` + +Another scenario where this is useful is for `typing.TYPE_CHECKING` branches, which are often used +for conditional imports: + +```py path=module2.py +class SomeType: ... +``` + +```py path=test2.py +import typing + +if typing.TYPE_CHECKING: + from module2 import SomeType + +# `SomeType` is unconditionally available here for type checkers: +def f(s: SomeType) -> None: ... +``` + +The rest of this document contains tests for various cases where this feature can be used. + ## If statements ### Always false From acf2ce6da9ce1989a85bf3a220d111194fd21e60 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 17:50:37 +0100 Subject: [PATCH 50/68] Fix clippy suggestions --- crates/red_knot_python_semantic/src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index b774fbf51acbd..8013cf68f7409 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -403,7 +403,7 @@ fn declarations_ty<'db>( } }) .take_while_inclusive(Visibility::is_not_opaque) - .map(|v| v.unwrap_or_never()); + .map(Visibility::unwrap_or_never); // TODO: try to get rid of the `collect` here (see above) let types: Vec<_> = types.collect(); From 04e80a8feba710a6500cff4f7553b5795efd0ac9 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 17:56:55 +0100 Subject: [PATCH 51/68] Fix doc test --- crates/red_knot_python_semantic/src/types/static_truthiness.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/red_knot_python_semantic/src/types/static_truthiness.rs b/crates/red_knot_python_semantic/src/types/static_truthiness.rs index 18c567150fc29..63e70bb02e539 100644 --- a/crates/red_knot_python_semantic/src/types/static_truthiness.rs +++ b/crates/red_knot_python_semantic/src/types/static_truthiness.rs @@ -21,7 +21,7 @@ use crate::Db; /// ``` /// /// Given an iterator over the branching conditions for each of these bindings, we would get: -/// ``` +/// ```txt /// - a: {any_always_false: false, all_always_true: true, at_least_one_condition: false} /// - b: {any_always_false: false, all_always_true: true, at_least_one_condition: true} /// - c: {any_always_false: false, all_always_true: false, at_least_one_condition: true} From bee7cf1d78be1ab3c6fb0000dbadd8eb61e80a33 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 19:31:28 +0100 Subject: [PATCH 52/68] Minor doc update --- .../resources/mdtest/statically-known-branches.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 47bc8c20f04fa..9483e517ee986 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -2,23 +2,23 @@ ## Introduction -We have the ability to infer precise types and boundness information for symbols that are -defined/declared in branches whose conditions we can statically determine to be always true or -always false. This is useful is for `sys.version_info` branches, which are often used to -conditionally make new features available: +We have the ability to infer precise types and boundness information for symbols that are defined in +branches whose conditions we can statically determine to be always true or always false. This is +useful for `sys.version_info` branches, which can make new features available based on the Python +version: ```py path=module1.py if sys.version_info >= (3, 9): SomeFeature = "available" ``` -If we can statically determine that the condition is always true, we can determine that +If we can statically determine that the condition is always true, then we can also understand that `SomeFeature` is always bound, without raising any errors: ```py path=test1.py from module1 import SomeFeature -# SomeFeature is unconditionally available here: +# SomeFeature is unconditionally available here, because we are on Python 3.9 or newer: reveal_type(SomeFeature) # revealed: Literal["available"] ``` From 0746cec3e708d47a8431b3dd10acc1a32b9d8688 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 19:49:04 +0100 Subject: [PATCH 53/68] Add tests for elif branches --- .../mdtest/statically-known-branches.md | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 9483e517ee986..812b28f61b1b8 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -123,6 +123,101 @@ else: reveal_type(x) # revealed: Literal[2] ``` +### `elif` branches + +#### Always false + +```py +def flag() -> bool: ... + +x = 1 + +if flag(): + x = 2 +elif False: + x = 3 +else: + x = 4 + +reveal_type(x) # revealed: Literal[2, 4] +``` + +#### Always true + +```py +def flag() -> bool: ... + +x = 1 + +if flag(): + x = 2 +elif True: + x = 3 +else: + x = 4 + +reveal_type(x) # revealed: Literal[2, 3] +``` + +#### Ambiguous + +```py +def flag() -> bool: ... + +x = 1 + +if flag(): + x = 2 +elif flag(): + x = 3 +else: + x = 4 + +reveal_type(x) # revealed: Literal[2, 3, 4] +``` + +#### Multiple `elif` branches, always false + +```py +def flag() -> bool: ... + +x = 1 + +if flag(): + x = 2 +elif flag(): + x = 3 +elif False: + x = 4 +elif flag(): + x = 5 +else: + x = 6 + +reveal_type(x) # revealed: Literal[2, 3, 5, 6] +``` + +#### Multiple `elif` branches, always true + +```py +def flag() -> bool: ... + +x = 1 + +if flag(): + x = 2 +elif flag(): + x = 3 +elif True: + x = 4 +elif flag(): + x = 5 +else: + x = 6 + +reveal_type(x) # revealed: Literal[2, 3, 4] +``` + ### Nested conditionals #### `if True` inside `if True` From 6ffa9e832caa7736e13c6a49097b20ba22190574 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 19:57:41 +0100 Subject: [PATCH 54/68] Minor doc updates --- .../src/semantic_index/use_def.rs | 8 ++++---- .../src/types/static_truthiness.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 66ac5b1333479..5f19d67840450 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -690,9 +690,9 @@ impl<'db> UseDefMapBuilder<'db> { /// x = 2 /// ``` /// -/// Here, `x` is definitely bound if `test2` is always true OR if `test1` is always true. `x` is -/// definitely unbound if `test1` is always false AND `test2` is always false. And `x` is possibly -/// unbound in all other cases. +/// Here, `x` is definitely bound if `test1` is always true OR if `test2` is always true. `x` is +/// definitely unbound if `test1` is always false AND `test2` is always false. `x` is possibly +/// unbound in all other cases. This logic is handled in [`StaticTruthiness::flow_merge`]. /// /// Finally, we also need to consider that a symbol could be definitely bound, even if we can not /// statically infer the truthiness of a test condition. On such example is: @@ -714,7 +714,7 @@ where C: Iterator>, { let result = conditions_per_binding.fold(StaticTruthiness::no_bindings(), |r, conditions| { - r.merge(&StaticTruthiness::analyze(db, conditions)) + r.flow_merge(&StaticTruthiness::analyze(db, conditions)) }); let definitely_unbound = result.any_always_false; diff --git a/crates/red_knot_python_semantic/src/types/static_truthiness.rs b/crates/red_knot_python_semantic/src/types/static_truthiness.rs index 63e70bb02e539..3e1c06cbeeceb 100644 --- a/crates/red_knot_python_semantic/src/types/static_truthiness.rs +++ b/crates/red_knot_python_semantic/src/types/static_truthiness.rs @@ -107,10 +107,10 @@ impl StaticTruthiness { /// of the fields. The reason for this is that we want to draw conclusions like "this symbol can /// not be bound because one of the branching conditions is always false". We can only draw this /// conclusion if this is true in both control-flow paths. Similarly, we want to infer that the - /// binding of a symbol is unconditionally visible, if all branching conditions are known to be - /// statically true. It is enough if this is the case for *any* of the control-flow paths. Other - /// control flow paths will not be taken if this is the case. - pub(crate) fn merge(self, other: &Self) -> Self { + /// binding of a symbol is unconditionally visible if all branching conditions are known to be + /// statically true. It is enough if this is the case for either of the two control-flow paths. + /// The other paths can not be taken if this is the case. + pub(crate) fn flow_merge(self, other: &Self) -> Self { Self { any_always_false: self.any_always_false && other.any_always_false, all_always_true: self.all_always_true || other.all_always_true, From 57c91023cb90d9b968f7a593412aaa8cd2996675 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 11 Dec 2024 20:29:00 +0100 Subject: [PATCH 55/68] Update TODO --- crates/red_knot_python_semantic/src/types.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 8013cf68f7409..51e05ac8a94ad 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -75,9 +75,9 @@ fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolI // on inference from bindings. let declaredness = use_def.public_declarations(symbol).declaredness(db); - // TODO: Our handling of boundness currently only depends on bindings, and ignores - // declarations. This is inconsistent, since we only look at bindings if the symbol - // may be undeclared. Consider the following example: + // TODO (ticket: https://github.com/astral-sh/ruff/issues/14297) Our handling of boundness + // currently only depends on bindings, and ignores declarations. This is inconsistent, since + // we only look at bindings if the symbol may be undeclared. Consider the following example: // ```py // x: int // @@ -86,9 +86,9 @@ fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolI // else // y = 3 // ``` - // If we import from this module, we will currently report `x` as a definitely-bound - // symbol (even though it has no bindings at all!) but report `y` as possibly-unbound - // (even though every path has either a binding or a declaration for it.) + // If we import from this module, we will currently report `x` as a definitely-bound symbol + // (even though it has no bindings at all!) but report `y` as possibly-unbound (even though + // every path has either a binding or a declaration for it.) let undeclared_ty = match declaredness { None => { return bindings_ty(db, use_def.public_bindings(symbol)) From 37359bba25935817ce8140597f32a64b4d0a16eb Mon Sep 17 00:00:00 2001 From: David Peter Date: Thu, 12 Dec 2024 08:39:30 +0100 Subject: [PATCH 56/68] Add 'import sys' --- .../resources/mdtest/statically-known-branches.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 812b28f61b1b8..a505876e795e1 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -8,6 +8,8 @@ useful for `sys.version_info` branches, which can make new features available ba version: ```py path=module1.py +import sys + if sys.version_info >= (3, 9): SomeFeature = "available" ``` From 0372684f727b5ed88203056e2bb7e9212c31f7d1 Mon Sep 17 00:00:00 2001 From: David Peter Date: Thu, 12 Dec 2024 08:40:11 +0100 Subject: [PATCH 57/68] Fix x=1,2,3 values in test --- .../resources/mdtest/statically-known-branches.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index a505876e795e1..f199678ea5ae4 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -660,9 +660,9 @@ x = 1 match "something else": case "a": - x = 1 - case "b": x = 2 + case "b": + x = 3 reveal_type(x) # revealed: Literal[1] ``` From eae479600091fa73aae6fd2bd55ebcf80d0c6f2b Mon Sep 17 00:00:00 2001 From: David Peter Date: Thu, 12 Dec 2024 09:10:06 +0100 Subject: [PATCH 58/68] Add tests for multiple 'elif True' / 'elif False' branches --- .../mdtest/statically-known-branches.md | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index f199678ea5ae4..1406f0db755b1 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -180,6 +180,8 @@ reveal_type(x) # revealed: Literal[2, 3, 4] #### Multiple `elif` branches, always false +Make sure that we include bindings from all non-`False` branches: + ```py def flag() -> bool: ... @@ -191,16 +193,22 @@ elif flag(): x = 3 elif False: x = 4 -elif flag(): +elif False: x = 5 -else: +elif flag(): x = 6 +elif flag(): + x = 7 +else: + x = 8 -reveal_type(x) # revealed: Literal[2, 3, 5, 6] +reveal_type(x) # revealed: Literal[2, 3, 6, 7, 8] ``` #### Multiple `elif` branches, always true +Make sure that we only include the binding from the first `elif True` branch: + ```py def flag() -> bool: ... @@ -212,10 +220,12 @@ elif flag(): x = 3 elif True: x = 4 -elif flag(): +elif True: x = 5 -else: +elif flag(): x = 6 +else: + x = 7 reveal_type(x) # revealed: Literal[2, 3, 4] ``` From e59e8c95d0816bc3a9dea36f22ca48d57fc5f5fb Mon Sep 17 00:00:00 2001 From: David Peter Date: Thu, 12 Dec 2024 09:12:53 +0100 Subject: [PATCH 59/68] Rename to record_ambiguous_branching --- .../src/semantic_index/builder.rs | 17 ++++++++--------- .../src/semantic_index/use_def.rs | 6 +++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index 302fa6fb56fcb..4617b4027a1b3 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -302,9 +302,8 @@ impl<'db> SemanticIndexBuilder<'db> { self.current_use_def_map_mut().record_constraint(constraint); } - fn record_unconditional_branching(&mut self) { - self.current_use_def_map_mut() - .record_unconditional_branching(); + fn record_ambiguous_branching(&mut self) { + self.current_use_def_map_mut().record_ambiguous_branching(); } fn build_constraint(&mut self, constraint_node: &Expr) -> Constraint<'db> { @@ -931,7 +930,7 @@ where let pre_loop_conditions = self.branching_conditions_snapshot(); let saved_break_states = std::mem::take(&mut self.loop_break_states); - self.record_unconditional_branching(); + self.record_ambiguous_branching(); debug_assert_eq!(&self.current_assignments, &[]); self.push_assignment(for_stmt.into()); @@ -952,7 +951,7 @@ where // We may execute the `else` clause without ever executing the body, so merge in // the pre-loop state before visiting `else`. self.flow_merge(pre_loop, pre_loop_conditions.clone()); - self.record_unconditional_branching(); + self.record_ambiguous_branching(); self.visit_body(orelse); // Breaking out of a `for` loop bypasses the `else` clause, so merge in the break @@ -1011,7 +1010,7 @@ where let pre_try_block_state = self.flow_snapshot(); let pre_try_block_conditions = self.branching_conditions_snapshot(); - self.record_unconditional_branching(); + self.record_ambiguous_branching(); self.try_node_context_stack_manager.push_context(); @@ -1051,7 +1050,7 @@ where range: _, } = except_handler; - self.record_unconditional_branching(); + self.record_ambiguous_branching(); if let Some(handled_exceptions) = handled_exceptions { self.visit_expr(handled_exceptions); @@ -1092,7 +1091,7 @@ where self.flow_restore(post_try_block_state, pre_try_block_conditions.clone()); } - self.record_unconditional_branching(); + self.record_ambiguous_branching(); self.visit_body(orelse); @@ -1110,7 +1109,7 @@ where // For more details, see: // - https://astral-sh.notion.site/Exception-handler-control-flow-11348797e1ca80bb8ce1e9aedbbe439d // - https://github.com/astral-sh/ruff/pull/13633#discussion_r1788626702 - self.record_unconditional_branching(); + self.record_ambiguous_branching(); self.visit_body(finalbody); diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs index 5f19d67840450..a85876be5f7a5 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def.rs @@ -548,9 +548,9 @@ impl<'db> UseDefMapBuilder<'db> { self.record_branching_condition(BranchingCondition::ConditionalOn(constraint)); } - /// Marks a point in control-flow where we branch unconditionally, that is: without any - /// conditions that could be statically analyzed. Examples are `for` loops or `try` blocks. - pub(super) fn record_unconditional_branching(&mut self) { + /// Marks a point in control-flow where we branch on a condition that we can not (or choose + /// not to) analyze statically. Examples are `try` blocks or `for` loops. + pub(super) fn record_ambiguous_branching(&mut self) { self.record_branching_condition(BranchingCondition::Ambiguous); } From 88fef0bf702e1c4b22d1453f05a6da614aefe245 Mon Sep 17 00:00:00 2001 From: David Peter Date: Thu, 12 Dec 2024 10:30:56 +0100 Subject: [PATCH 60/68] Hard assert --- .../src/semantic_index/use_def/bitset.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs index 92dabe44a41f9..69c052e4ae164 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs @@ -106,7 +106,7 @@ impl BitSet { pub(super) fn iter_rev(&self) -> ReverseBitSetIterator<'_, B> { let num_blocks = self.blocks().len(); - debug_assert!(num_blocks > 0); + assert!(num_blocks > 0); let blocks = self.blocks(); ReverseBitSetIterator { From b43e9f29edaa85a5696565e9d850c7f348d347cc Mon Sep 17 00:00:00 2001 From: David Peter Date: Thu, 12 Dec 2024 10:33:58 +0100 Subject: [PATCH 61/68] Resolve TODO in tuple.md --- .../resources/mdtest/subscript/tuple.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/subscript/tuple.md b/crates/red_knot_python_semantic/resources/mdtest/subscript/tuple.md index 1d27885567df2..88dd39144a6ae 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/subscript/tuple.md +++ b/crates/red_knot_python_semantic/resources/mdtest/subscript/tuple.md @@ -81,10 +81,7 @@ python-version = "3.9" ``` ```py -# TODO: -# * `tuple.__class_getitem__` is always bound on 3.9 (`sys.version_info`) -# * `tuple[int, str]` is a valid base (generics) -# error: [call-possibly-unbound-method] "Method `__class_getitem__` of type `Literal[tuple]` is possibly unbound" +# TODO: `tuple[int, str]` is a valid base (generics) # error: [invalid-base] "Invalid class base with type `GenericAlias` (all bases must be a class, `Any`, `Unknown` or `Todo`)" class A(tuple[int, str]): ... From ae470f0093f080a63e9dd800ef2e7e6e820a7270 Mon Sep 17 00:00:00 2001 From: David Peter Date: Thu, 12 Dec 2024 14:44:10 +0100 Subject: [PATCH 62/68] Document current limitations --- .../mdtest/statically-known-branches.md | 129 +++++++++++++++++- 1 file changed, 126 insertions(+), 3 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md index 1406f0db755b1..ff6118a99290a 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md +++ b/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md @@ -230,6 +230,22 @@ else: reveal_type(x) # revealed: Literal[2, 3, 4] ``` +#### `elif` without `else` branch + +```py +def flag() -> bool: ... + +x = 1 + +if flag(): + x = 2 +elif True: + x = 3 + +# TODO: This should be Literal[2, 3] +reveal_type(x) # revealed: Literal[1, 2, 3] +``` + ### Nested conditionals #### `if True` inside `if True` @@ -237,6 +253,113 @@ reveal_type(x) # revealed: Literal[2, 3, 4] ```py x = 1 +if True: + if True: + x = 2 +else: + x = 3 + +reveal_type(x) # revealed: Literal[2] +``` + +#### `if False` inside `if True` + +```py +x = 1 + +if True: + if False: + x = 2 +else: + x = 3 + +reveal_type(x) # revealed: Literal[1] +``` + +#### `if ` inside `if True` + +```py +def flag() -> bool: ... + +x = 1 + +if True: + if flag(): + x = 2 +else: + x = 3 + +reveal_type(x) # revealed: Literal[1, 2] +``` + +#### `if True` inside `if ` + +```py +def flag() -> bool: ... + +x = 1 + +if flag(): + if True: + x = 2 +else: + x = 3 + +# TODO: This should be Literal[2, 3] +reveal_type(x) # revealed: Literal[1, 2, 3] +``` + +#### `if True` inside `if False` ... `else` + +```py +x = 1 + +if False: + x = 2 +else: + if True: + x = 3 + +reveal_type(x) # revealed: Literal[3] +``` + +#### `if False` inside `if False` ... `else` + +```py +x = 1 + +if False: + x = 2 +else: + if False: + x = 3 + +reveal_type(x) # revealed: Literal[1] +``` + +#### `if ` inside `if False` ... `else` + +```py +def flag() -> bool: ... + +x = 1 + +if False: + x = 2 +else: + if flag(): + x = 3 + +reveal_type(x) # revealed: Literal[1, 3] +``` + +### Nested conditionals (with inner `else`) + +#### `if True` inside `if True` + +```py +x = 1 + if True: if True: x = 2 @@ -448,12 +571,12 @@ def iterable() -> list[object]: ... x = 1 for _ in iterable(): + x = 2 if True: - x = 2 - else: x = 3 -reveal_type(x) # revealed: Literal[1, 2] +# TODO: This should be Literal[1, 3] +reveal_type(x) # revealed: Literal[1, 2, 3] ``` ##### `if True` inside `for` ... `else` From 513795e540de096ad706545f69e756378b684373 Mon Sep 17 00:00:00 2001 From: David Peter Date: Fri, 13 Dec 2024 09:27:40 +0100 Subject: [PATCH 63/68] Snake case --- ...{statically-known-branches.md => statically_known_branches.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename crates/red_knot_python_semantic/resources/mdtest/{statically-known-branches.md => statically_known_branches.md} (100%) diff --git a/crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md b/crates/red_knot_python_semantic/resources/mdtest/statically_known_branches.md similarity index 100% rename from crates/red_knot_python_semantic/resources/mdtest/statically-known-branches.md rename to crates/red_knot_python_semantic/resources/mdtest/statically_known_branches.md From 9b4bb86eca52af8e2ca823bddf5331f71963c464 Mon Sep 17 00:00:00 2001 From: David Peter Date: Fri, 13 Dec 2024 09:28:44 +0100 Subject: [PATCH 64/68] Remove Truthiness::from_bool --- crates/red_knot_python_semantic/src/types.rs | 8 -------- .../src/types/static_truthiness.rs | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 51e05ac8a94ad..049390016eec9 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -2604,14 +2604,6 @@ pub enum Truthiness { } impl Truthiness { - pub(crate) const fn from_bool(value: bool) -> Self { - if value { - Self::AlwaysTrue - } else { - Self::AlwaysFalse - } - } - const fn is_ambiguous(self) -> bool { matches!(self, Truthiness::Ambiguous) } diff --git a/crates/red_knot_python_semantic/src/types/static_truthiness.rs b/crates/red_knot_python_semantic/src/types/static_truthiness.rs index 3e1c06cbeeceb..441d355c2753a 100644 --- a/crates/red_knot_python_semantic/src/types/static_truthiness.rs +++ b/crates/red_knot_python_semantic/src/types/static_truthiness.rs @@ -81,7 +81,7 @@ impl StaticTruthiness { .expression_ty(value.node_ref(db).scoped_expression_id(db, scope)); if subject_ty.is_single_valued(db) { - Truthiness::from_bool(subject_ty.is_equivalent_to(db, value_ty)) + Truthiness::from(subject_ty.is_equivalent_to(db, value_ty)) } else { Truthiness::Ambiguous } From 741beb7ba68a802ab28c0ec847556bb60ce13942 Mon Sep 17 00:00:00 2001 From: David Peter Date: Fri, 13 Dec 2024 09:29:11 +0100 Subject: [PATCH 65/68] Make is_ambiguous pub(crate) --- crates/red_knot_python_semantic/src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 049390016eec9..c39757994bf6a 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -2604,7 +2604,7 @@ pub enum Truthiness { } impl Truthiness { - const fn is_ambiguous(self) -> bool { + pub(crate) const fn is_ambiguous(self) -> bool { matches!(self, Truthiness::Ambiguous) } From d3f460c2527ca831147a9407d3a1aaaa0786dadb Mon Sep 17 00:00:00 2001 From: David Peter Date: Fri, 13 Dec 2024 09:29:44 +0100 Subject: [PATCH 66/68] Derive Debug --- crates/red_knot_python_semantic/src/types/static_truthiness.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/red_knot_python_semantic/src/types/static_truthiness.rs b/crates/red_knot_python_semantic/src/types/static_truthiness.rs index 441d355c2753a..99fc94d5a0d65 100644 --- a/crates/red_knot_python_semantic/src/types/static_truthiness.rs +++ b/crates/red_knot_python_semantic/src/types/static_truthiness.rs @@ -27,6 +27,7 @@ use crate::Db; /// - c: {any_always_false: false, all_always_true: false, at_least_one_condition: true} /// - d: {any_always_false: true, all_always_true: false, at_least_one_condition: true} /// ``` +#[derive(Debug)] pub(crate) struct StaticTruthiness { /// Is any of the branching conditions always false? (false if there are no conditions) pub(crate) any_always_false: bool, From 006a5370d94c32f68a9fc3c789ac56b94969674e Mon Sep 17 00:00:00 2001 From: David Peter Date: Fri, 13 Dec 2024 09:31:01 +0100 Subject: [PATCH 67/68] Fix rogue conflict resolution --- .../resources/mdtest/narrow/while.md | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/narrow/while.md b/crates/red_knot_python_semantic/resources/mdtest/narrow/while.md index 17f630d023ddf..ac91a576c9de1 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/narrow/while.md +++ b/crates/red_knot_python_semantic/resources/mdtest/narrow/while.md @@ -1,4 +1,3 @@ -<<<<<<< HEAD # Narrowing in `while` loops We only make sure that narrowing works for `while` loops in general, we do not exhaustively test all @@ -57,24 +56,3 @@ while x != 1: x = next_item() ``` -||||||| parent of 8247164bf (Add tests for 'while'-narrowing) -======= -# Narrowing in `while` loops - -We only make sure that narrowing works for `while` loops in general, we do not exhaustively test all -narrowing forms here, as they are covered in other tests. - -## Basic example - -```py -def next_item() -> int | None: ... - -x = next_item() - -while x is not None: - reveal_type(x) # revealed: int - x = next_item() -else: - reveal_type(x) # revealed: None -``` ->>>>>>> 8247164bf (Add tests for 'while'-narrowing) From 0a0e5e686f65a34b7b409555baac9ad60c590ac2 Mon Sep 17 00:00:00 2001 From: David Peter Date: Fri, 13 Dec 2024 10:43:21 +0100 Subject: [PATCH 68/68] Minor --- .../resources/mdtest/pep695_type_aliases.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md b/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md index dd94e6b2d55c4..de2eccf81ea81 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md +++ b/crates/red_knot_python_semantic/resources/mdtest/pep695_type_aliases.md @@ -7,13 +7,6 @@ PEP 695 type aliases are only available in Python 3.12 and later: python-version = "3.12" ``` -Type aliases are only available in Python 3.12 and later: - -```toml -[environment] -python-version = "3.12" -``` - ## Basic ```py