Workspace discovery (#14308)

Author: MichaReiser

.editorconfig

@@ -17,4 +17,7 @@ indent_size = 4
 trim_trailing_whitespace = false
 
 [*.md]
-max_line_length = 100
+max_line_length = 100
+
+[*.toml]
+indent_size = 4

Cargo.lock

@@ -66,6 +66,7 @@ criterion = { version = "0.5.1", default-features = false }
 crossbeam = { version = "0.8.4" }
 dashmap = { version = "6.0.1" }
 dir-test = { version = "0.3.0" }
+dunce = { version = "1.0.5" }
 drop_bomb = { version = "0.1.5" }
 env_logger = { version = "0.11.0" }
 etcetera = { version = "0.8.0" }
@@ -81,7 +82,7 @@ hashbrown = { version = "0.15.0", default-features = false, features = [
 ignore = { version = "0.4.22" }
 imara-diff = { version = "0.1.5" }
 imperative = { version = "1.0.4" }
-indexmap = {version = "2.6.0" }
+indexmap = { version = "2.6.0" }
 indicatif = { version = "0.17.8" }
 indoc = { version = "2.0.4" }
 insta = { version = "1.35.1" }

Cargo.toml

@@ -1,6 +1,11 @@
 [files]
 # https://github.com/crate-ci/typos/issues/868
-extend-exclude = ["crates/red_knot_vendored/vendor/**/*", "**/resources/**/*", "**/snapshots/**/*"]
+extend-exclude = [
+  "crates/red_knot_vendored/vendor/**/*",
+  "**/resources/**/*",
+  "**/snapshots/**/*",
+  "crates/red_knot_workspace/src/workspace/pyproject/package_name.rs"
+]
 
 [default.extend-words]
 "arange" = "arange"  # e.g. `numpy.arange`

_typos.toml

@@ -34,6 +34,7 @@ tracing-tree = { workspace = true }
 [dev-dependencies]
 filetime = { workspace = true }
 tempfile = { workspace = true }
+ruff_db = { workspace = true, features = ["testing"] }
 
 [lints]
 workspace = true

crates/red_knot/Cargo.toml

@@ -183,10 +183,10 @@ fn run() -> anyhow::Result<ExitStatus> {
 
     let system = OsSystem::new(cwd.clone());
     let cli_configuration = args.to_configuration(&cwd);
-    let workspace_metadata = WorkspaceMetadata::from_path(
+    let workspace_metadata = WorkspaceMetadata::discover(
         system.current_directory(),
         &system,
-        Some(cli_configuration.clone()),
+        Some(&cli_configuration),
     )?;
 
     // TODO: Use the `program_settings` to compute the key for the database's persistent

crates/red_knot/src/main.rs

@@ -6,14 +6,15 @@ use std::time::Duration;
 use anyhow::{anyhow, Context};
 
 use red_knot_python_semantic::{resolve_module, ModuleName, Program, PythonVersion, SitePackages};
-use red_knot_workspace::db::RootDatabase;
+use red_knot_workspace::db::{Db, RootDatabase};
 use red_knot_workspace::watch;
 use red_knot_workspace::watch::{directory_watcher, WorkspaceWatcher};
 use red_knot_workspace::workspace::settings::{Configuration, SearchPathConfiguration};
 use red_knot_workspace::workspace::WorkspaceMetadata;
 use ruff_db::files::{system_path_to_file, File, FileError};
 use ruff_db::source::source_text;
 use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf};
+use ruff_db::testing::setup_logging;
 use ruff_db::Upcast;
 
 struct TestCase {
@@ -69,7 +70,6 @@ impl TestCase {
         Some(all_events)
     }
 
-    #[cfg(unix)]
     fn take_watch_changes(&self) -> Vec<watch::ChangeEvent> {
         self.try_take_watch_changes(Duration::from_secs(10))
             .expect("Expected watch changes but observed none")
@@ -110,8 +110,8 @@ impl TestCase {
     ) -> anyhow::Result<()> {
         let program = Program::get(self.db());
 
-        self.configuration.search_paths = configuration.clone();
-        let new_settings = configuration.into_settings(self.db.workspace().root(&self.db));
+        let new_settings = configuration.to_settings(self.db.workspace().root(&self.db));
+        self.configuration.search_paths = configuration;
 
         program.update_search_paths(&mut self.db, &new_settings)?;
 
@@ -204,7 +204,9 @@ where
             .as_utf8_path()
             .canonicalize_utf8()
             .with_context(|| "Failed to canonicalize root path.")?,
-    );
+    )
+    .simplified()
+    .to_path_buf();
 
     let workspace_path = root_path.join("workspace");
 
@@ -241,8 +243,7 @@ where
         search_paths,
     };
 
-    let workspace =
-        WorkspaceMetadata::from_path(&workspace_path, &system, Some(configuration.clone()))?;
+    let workspace = WorkspaceMetadata::discover(&workspace_path, &system, Some(&configuration))?;
 
     let db = RootDatabase::new(workspace, system)?;
 
@@ -1311,3 +1312,138 @@ mod unix {
         Ok(())
     }
 }
+
+#[test]
+fn nested_packages_delete_root() -> anyhow::Result<()> {
+    let mut case = setup(|root: &SystemPath, workspace_root: &SystemPath| {
+        std::fs::write(
+            workspace_root.join("pyproject.toml").as_std_path(),
+            r#"
+            [project]
+            name = "inner"
+            "#,
+        )?;
+
+        std::fs::write(
+            root.join("pyproject.toml").as_std_path(),
+            r#"
+            [project]
+            name = "outer"
+            "#,
+        )?;
+
+        Ok(())
+    })?;
+
+    assert_eq!(
+        case.db().workspace().root(case.db()),
+        &*case.workspace_path("")
+    );
+
+    std::fs::remove_file(case.workspace_path("pyproject.toml").as_std_path())?;
+
+    let changes = case.stop_watch();
+
+    case.apply_changes(changes);
+
+    // It should now pick up the outer workspace.
+    assert_eq!(case.db().workspace().root(case.db()), case.root_path());
+
+    Ok(())
+}
+
+#[test]
+fn added_package() -> anyhow::Result<()> {
+    let _ = setup_logging();
+    let mut case = setup([
+        (
+            "pyproject.toml",
+            r#"
+            [project]
+            name = "inner"
+
+            [tool.knot.workspace]
+            members = ["packages/*"]
+            "#,
+        ),
+        (
+            "packages/a/pyproject.toml",
+            r#"
+            [project]
+            name = "a"
+            "#,
+        ),
+    ])?;
+
+    assert_eq!(case.db().workspace().packages(case.db()).len(), 2);
+
+    std::fs::create_dir(case.workspace_path("packages/b").as_std_path())
+        .context("failed to create folder for package 'b'")?;
+
+    // It seems that the file watcher won't pick up on file changes shortly after the folder
+    // was created... I suspect this is because most file watchers don't support recursive
+    // file watching. Instead, file-watching libraries manually implement recursive file watching
+    // by setting a watcher for each directory. But doing this obviously "lags" behind.
+    case.take_watch_changes();
+
+    std::fs::write(
+        case.workspace_path("packages/b/pyproject.toml")
+            .as_std_path(),
+        r#"
+            [project]
+            name = "b"
+            "#,
+    )
+    .context("failed to write pyproject.toml for package b")?;
+
+    let changes = case.stop_watch();
+
+    case.apply_changes(changes);
+
+    assert_eq!(case.db().workspace().packages(case.db()).len(), 3);
+
+    Ok(())
+}
+
+#[test]
+fn removed_package() -> anyhow::Result<()> {
+    let mut case = setup([
+        (
+            "pyproject.toml",
+            r#"
+            [project]
+            name = "inner"
+
+            [tool.knot.workspace]
+            members = ["packages/*"]
+            "#,
+        ),
+        (
+            "packages/a/pyproject.toml",
+            r#"
+            [project]
+            name = "a"
+            "#,
+        ),
+        (
+            "packages/b/pyproject.toml",
+            r#"
+            [project]
+            name = "b"
+            "#,
+        ),
+    ])?;
+
+    assert_eq!(case.db().workspace().packages(case.db()).len(), 3);
+
+    std::fs::remove_dir_all(case.workspace_path("packages/b").as_std_path())
+        .context("failed to remove package 'b'")?;
+
+    let changes = case.stop_watch();
+
+    case.apply_changes(changes);
+
+    assert_eq!(case.db().workspace().packages(case.db()).len(), 2);
+
+    Ok(())
+}

crates/red_knot/tests/file_watching.rs

@@ -33,6 +33,7 @@ thiserror = { workspace = true }
 tracing = { workspace = true }
 rustc-hash = { workspace = true }
 hashbrown = { workspace = true }
+serde = { workspace = true, optional = true }
 smallvec = { workspace = true }
 static_assertions = { workspace = true }
 test-case = { workspace = true }

crates/red_knot_python_semantic/Cargo.toml

@@ -54,13 +54,15 @@ impl Program {
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize))]
 pub struct ProgramSettings {
     pub target_version: PythonVersion,
     pub search_paths: SearchPathSettings,
 }
 
 /// Configures the search paths for module resolution.
 #[derive(Eq, PartialEq, Debug, Clone)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize))]
 pub struct SearchPathSettings {
     /// List of user-provided paths that should take first priority in the module resolution.
     /// Examples in other type checkers are mypy's MYPYPATH environment variable,
@@ -91,6 +93,7 @@ impl SearchPathSettings {
 }
 
 #[derive(Debug, Clone, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize))]
 pub enum SitePackages {
     Derived {
         venv_path: SystemPathBuf,

crates/red_knot_python_semantic/src/program.rs

@@ -5,6 +5,7 @@ use std::fmt;
 /// Unlike the `TargetVersion` enums in the CLI crates,
 /// this does not necessarily represent a Python version that we actually support.
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize))]
 pub struct PythonVersion {
     pub major: u8,
     pub minor: u8,