Use an empty vendored file system in Ruff (#13436)

## Summary

This PR changes removes the typeshed stubs from the vendored file system
shipped with ruff
and instead ships an empty "typeshed".

Making the typeshed files optional required extracting the typshed files
into a new `ruff_vendored` crate. I do like this even if all our builds
always include typeshed because it means `red_knot_python_semantic`
contains less code that needs compiling.

This also allows us to use deflate because the compression algorithm
doesn't matter for an archive containing a single, empty file.

## Test Plan

`cargo test`

I verified with ` cargo tree -f "{p} {f}" -p <package> ` that:

* red_knot_wasm: enables `deflate` compression
* red_knot: enables `zstd` compression
* `ruff`: uses stored


I'm not quiet sure how to build the binary that maturin builds but
comparing the release artifact size with `strip = true` shows a `1.5MB`
size reduction

---------

Co-authored-by: Charlie Marsh <[email protected]>

Author: MichaReiser

.github/workflows/sync_typeshed.yaml

@@ -37,13 +37,13 @@ jobs:
       - name: Sync typeshed
         id: sync
         run: |
-          rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed
-          mkdir ruff/crates/red_knot_python_semantic/vendor/typeshed
-          cp typeshed/README.md ruff/crates/red_knot_python_semantic/vendor/typeshed
-          cp typeshed/LICENSE ruff/crates/red_knot_python_semantic/vendor/typeshed
-          cp -r typeshed/stdlib ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib
-          rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib/@tests
-          git -C typeshed rev-parse HEAD > ruff/crates/red_knot_python_semantic/vendor/typeshed/source_commit.txt
+          rm -rf ruff/crates/ruff_vendored/vendor/typeshed
+          mkdir ruff/crates/ruff_vendored/vendor/typeshed
+          cp typeshed/README.md ruff/crates/ruff_vendored/vendor/typeshed
+          cp typeshed/LICENSE ruff/crates/ruff_vendored/vendor/typeshed
+          cp -r typeshed/stdlib ruff/crates/ruff_vendored/vendor/typeshed/stdlib
+          rm -rf ruff/crates/ruff_vendored/vendor/typeshed/stdlib/@tests
+          git -C typeshed rev-parse HEAD > ruff/crates/ruff_vendored/vendor/typeshed/source_commit.txt
       - name: Commit the changes
         id: commit
         if: ${{ steps.sync.outcome == 'success' }}

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ fail_fast: true
 
 exclude: |
   (?x)^(
-    crates/red_knot_python_semantic/vendor/.*|
+    crates/ruff_vendored/vendor/.*|
     crates/red_knot_workspace/resources/.*|
     crates/ruff_linter/resources/.*|
     crates/ruff_linter/src/rules/.*/snapshots/.*|

Cargo.lock

@@ -14,7 +14,7 @@ license = "MIT"
 [workspace.dependencies]
 ruff = { path = "crates/ruff" }
 ruff_cache = { path = "crates/ruff_cache" }
-ruff_db = { path = "crates/ruff_db" }
+ruff_db = { path = "crates/ruff_db", default-features = false }
 ruff_diagnostics = { path = "crates/ruff_diagnostics" }
 ruff_formatter = { path = "crates/ruff_formatter" }
 ruff_graph = { path = "crates/ruff_graph" }
@@ -34,11 +34,12 @@ ruff_python_trivia = { path = "crates/ruff_python_trivia" }
 ruff_server = { path = "crates/ruff_server" }
 ruff_source_file = { path = "crates/ruff_source_file" }
 ruff_text_size = { path = "crates/ruff_text_size" }
+ruff_vendored = { path = "crates/ruff_vendored" }
 ruff_workspace = { path = "crates/ruff_workspace" }
 
 red_knot_python_semantic = { path = "crates/red_knot_python_semantic" }
 red_knot_server = { path = "crates/red_knot_server" }
-red_knot_workspace = { path = "crates/red_knot_workspace" }
+red_knot_workspace = { path = "crates/red_knot_workspace", default-features = false }
 
 aho-corasick = { version = "1.1.3" }
 annotate-snippets = { version = "0.9.2", features = ["color"] }

Cargo.toml

@@ -13,9 +13,8 @@ license.workspace = true
 
 [dependencies]
 red_knot_python_semantic = { workspace = true }
-red_knot_workspace = { workspace = true }
+red_knot_workspace = { workspace = true, features = ["zstd"] }
 red_knot_server = { workspace = true }
-
 ruff_db = { workspace = true, features = ["os", "cache"] }
 
 anyhow = { workspace = true }

crates/red_knot/Cargo.toml

@@ -24,7 +24,6 @@ bitflags = { workspace = true }
 camino = { workspace = true }
 compact_str = { workspace = true }
 countme = { workspace = true }
-once_cell = { workspace = true }
 ordermap = { workspace = true }
 salsa = { workspace = true }
 thiserror = { workspace = true }
@@ -35,20 +34,14 @@ smallvec = { workspace = true }
 static_assertions = { workspace = true }
 test-case = { workspace = true }
 
-[build-dependencies]
-path-slash = { workspace = true }
-walkdir = { workspace = true }
-zip = { workspace = true, features = ["zstd", "deflate"] }
-
 [dev-dependencies]
 ruff_db = { workspace = true, features = ["os", "testing"] }
 ruff_python_parser = { workspace = true }
+ruff_vendored = { workspace = true }
 
 anyhow = { workspace = true }
 insta = { workspace = true }
 tempfile = { workspace = true }
-walkdir = { workspace = true }
-zip = { workspace = true }
 
 [lints]
 workspace = true

crates/red_knot_python_semantic/Cargo.toml

@@ -11,7 +11,6 @@ pub trait Db: SourceDb + Upcast<dyn SourceDb> {
 pub(crate) mod tests {
     use std::sync::Arc;
 
-    use crate::module_resolver::vendored_typeshed_stubs;
     use ruff_db::files::{File, Files};
     use ruff_db::system::{DbWithTestSystem, System, TestSystem};
     use ruff_db::vendored::VendoredFileSystem;
@@ -33,7 +32,7 @@ pub(crate) mod tests {
             Self {
                 storage: salsa::Storage::default(),
                 system: TestSystem::default(),
-                vendored: vendored_typeshed_stubs().clone(),
+                vendored: ruff_vendored::file_system().clone(),
                 events: std::sync::Arc::default(),
                 files: Files::default(),
             }

crates/red_knot_python_semantic/src/db.rs

@@ -4,9 +4,7 @@ use rustc_hash::FxHasher;
 
 pub use db::Db;
 pub use module_name::ModuleName;
-pub use module_resolver::{
-    resolve_module, system_module_search_paths, vendored_typeshed_stubs, Module,
-};
+pub use module_resolver::{resolve_module, system_module_search_paths, Module};
 pub use program::{Program, ProgramSettings, SearchPathSettings, SitePackages};
 pub use python_version::PythonVersion;
 pub use semantic_model::{HasTy, SemanticModel};

crates/red_knot_python_semantic/src/lib.rs

@@ -4,7 +4,6 @@ pub use module::Module;
 pub use resolver::resolve_module;
 pub(crate) use resolver::{file_to_module, SearchPaths};
 use ruff_db::system::SystemPath;
-pub use typeshed::vendored_typeshed_stubs;
 
 use crate::module_resolver::resolver::search_paths;
 use crate::Db;

crates/red_knot_python_semantic/src/module_resolver/mod.rs

@@ -390,7 +390,8 @@ mod tests {
     fn typeshed_versions_consistent_with_vendored_stubs() {
         let db = TestDb::new();
         let vendored_typeshed_versions = vendored_typeshed_versions(&db);
-        let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap();
+        let vendored_typeshed_dir =
+            Path::new(env!("CARGO_MANIFEST_DIR")).join("../ruff_vendored/vendor/typeshed");
 
         let mut empty_iterator = true;
 

crates/red_knot_python_semantic/src/module_resolver/typeshed/versions.rs renamed to crates/red_knot_python_semantic/src/module_resolver/typeshed.rs

@@ -701,12 +701,12 @@ enum CallOutcome<'db> {
 
 impl<'db> CallOutcome<'db> {
     /// Create a new `CallOutcome::Callable` with given return type.
-    fn callable(return_ty: Type<'db>) -> CallOutcome {
+    fn callable(return_ty: Type<'db>) -> CallOutcome<'db> {
         CallOutcome::Callable { return_ty }
     }
 
     /// Create a new `CallOutcome::NotCallable` with given not-callable type.
-    fn not_callable(not_callable_ty: Type<'db>) -> CallOutcome {
+    fn not_callable(not_callable_ty: Type<'db>) -> CallOutcome<'db> {
         CallOutcome::NotCallable { not_callable_ty }
     }
 
@@ -719,7 +719,10 @@ impl<'db> CallOutcome<'db> {
     }
 
     /// Create a new `CallOutcome::Union` with given wrapped outcomes.
-    fn union(called_ty: Type<'db>, outcomes: impl Into<Box<[CallOutcome<'db>]>>) -> CallOutcome {
+    fn union(
+        called_ty: Type<'db>,
+        outcomes: impl Into<Box<[CallOutcome<'db>]>>,
+    ) -> CallOutcome<'db> {
         CallOutcome::Union {
             called_ty,
             outcomes: outcomes.into(),

crates/red_knot_python_semantic/src/module_resolver/typeshed/mod.rs

@@ -20,9 +20,9 @@ default = ["console_error_panic_hook"]
 
 [dependencies]
 red_knot_python_semantic = { workspace = true }
-red_knot_workspace = { workspace = true }
+red_knot_workspace = { workspace = true, default-features = false, features = ["deflate"] }
 
-ruff_db = { workspace = true }
+ruff_db = { workspace = true, features = [] }
 ruff_notebook = { workspace = true }
 
 console_error_panic_hook = { workspace = true, optional = true }

crates/red_knot_python_semantic/src/types.rs

@@ -18,6 +18,7 @@ ruff_cache = { workspace = true }
 ruff_db = { workspace = true, features = ["os", "cache"] }
 ruff_python_ast = { workspace = true }
 ruff_text_size = { workspace = true }
+ruff_vendored = { workspace = true }
 
 anyhow = { workspace = true }
 crossbeam = { workspace = true }
@@ -31,5 +32,10 @@ tracing = { workspace = true }
 ruff_db = { workspace = true, features = ["testing"] }
 tempfile = { workspace = true }
 
+[features]
+default = ["zstd"]
+zstd = ["ruff_vendored/zstd"]
+deflate = ["ruff_vendored/deflate"]
+
 [lints]
 workspace = true

crates/red_knot_wasm/Cargo.toml

@@ -4,7 +4,7 @@ use std::sync::Arc;
 use salsa::plumbing::ZalsaDatabase;
 use salsa::{Cancelled, Event};
 
-use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb, Program};
+use red_knot_python_semantic::{Db as SemanticDb, Program};
 use ruff_db::files::{File, Files};
 use ruff_db::system::System;
 use ruff_db::vendored::VendoredFileSystem;
@@ -124,7 +124,7 @@ impl SemanticDb for RootDatabase {
 #[salsa::db]
 impl SourceDb for RootDatabase {
     fn vendored(&self) -> &VendoredFileSystem {
-        vendored_typeshed_stubs()
+        ruff_vendored::file_system()
     }
 
     fn system(&self) -> &dyn System {
@@ -161,7 +161,7 @@ pub(crate) mod tests {
 
     use salsa::Event;
 
-    use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb};
+    use red_knot_python_semantic::Db as SemanticDb;
     use ruff_db::files::Files;
     use ruff_db::system::{DbWithTestSystem, System, TestSystem};
     use ruff_db::vendored::VendoredFileSystem;
@@ -183,7 +183,7 @@ pub(crate) mod tests {
             Self {
                 storage: salsa::Storage::default(),
                 system: TestSystem::default(),
-                vendored: vendored_typeshed_stubs().clone(),
+                vendored: ruff_vendored::file_system().clone(),
                 files: Files::default(),
                 events: Arc::default(),
             }

crates/red_knot_workspace/Cargo.toml

@@ -14,7 +14,7 @@ default-run = "ruff"
 
 [dependencies]
 ruff_cache = { workspace = true }
-ruff_db = { workspace = true }
+ruff_db = { workspace = true, default-features = false, features = ["os"] }
 ruff_diagnostics = { workspace = true }
 ruff_graph = { workspace = true, features = ["serde", "clap"] }
 ruff_linter = { workspace = true, features = ["clap"] }

crates/red_knot_workspace/src/db.rs

@@ -33,19 +33,17 @@ tracing = { workspace = true }
 tracing-subscriber = { workspace = true, optional = true }
 tracing-tree = { workspace = true, optional = true }
 rustc-hash = { workspace = true }
-
-[target.'cfg(not(target_arch="wasm32"))'.dependencies]
-zip = { workspace = true, features = ["zstd"] }
+zip = { workspace = true }
 
 [target.'cfg(target_arch="wasm32")'.dependencies]
 web-time = { version = "1.1.0" }
-zip = { workspace = true, features = ["deflate"] }
 
 [dev-dependencies]
 insta = { workspace = true }
 tempfile = { workspace = true }
 
 [features]
+default = ["os"]
 cache = ["ruff_cache"]
 os = ["ignore"]
 serde = ["dep:serde", "camino/serde1"]

crates/ruff/Cargo.toml

@@ -503,7 +503,8 @@ mod tests {
     use crate::files::{system_path_to_file, vendored_path_to_file, FileError};
     use crate::system::DbWithTestSystem;
     use crate::tests::TestDb;
-    use crate::vendored::tests::VendoredFileSystemBuilder;
+    use crate::vendored::VendoredFileSystemBuilder;
+    use zip::CompressionMethod;
 
     #[test]
     fn system_existing_file() -> crate::system::Result<()> {
@@ -548,7 +549,7 @@ mod tests {
     fn stubbed_vendored_file() -> crate::system::Result<()> {
         let mut db = TestDb::new();
 
-        let mut vendored_builder = VendoredFileSystemBuilder::new();
+        let mut vendored_builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
         vendored_builder
             .add_file("test.pyi", "def foo() -> str")
             .unwrap();

crates/ruff_db/Cargo.toml

@@ -79,8 +79,9 @@ mod tests {
     use crate::parsed::parsed_module;
     use crate::system::{DbWithTestSystem, SystemPath, SystemVirtualPath};
     use crate::tests::TestDb;
-    use crate::vendored::{tests::VendoredFileSystemBuilder, VendoredPath};
+    use crate::vendored::{VendoredFileSystemBuilder, VendoredPath};
     use crate::Db;
+    use zip::CompressionMethod;
 
     #[test]
     fn python_file() -> crate::system::Result<()> {
@@ -150,7 +151,7 @@ mod tests {
     fn vendored_file() {
         let mut db = TestDb::new();
 
-        let mut vendored_builder = VendoredFileSystemBuilder::new();
+        let mut vendored_builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
         vendored_builder
             .add_file(
                 "path.pyi",