feat(npm): add flag for creating and resolving npm packages to a local node_modules folder (#15971)

Author: dsherret

Cargo.lock

@@ -85,7 +85,6 @@ monch = "=0.2.0"
 notify = "=5.0.0"
 once_cell = "=1.14.0"
 os_pipe = "=1.0.1"
-path-clean = "=0.1.0"
 percent-encoding = "=2.2.0"
 pin-project = "1.0.11" # don't pin because they yank crates from cargo
 rand = { version = "=0.8.5", features = ["small_rng"] }

cli/Cargo.toml

@@ -301,6 +301,7 @@ pub struct Flags {
   pub cached_only: bool,
   pub type_check_mode: TypeCheckMode,
   pub config_flag: ConfigFlag,
+  pub node_modules_dir: bool,
   pub coverage_dir: Option<String>,
   pub enable_testing_features: bool,
   pub ignore: Vec<PathBuf>,
@@ -1732,6 +1733,7 @@ fn compile_args(app: Command) -> Command {
   app
     .arg(import_map_arg())
     .arg(no_remote_arg())
+    .arg(local_npm_arg())
     .arg(no_config_arg())
     .arg(config_arg())
     .arg(no_check_arg())
@@ -1746,6 +1748,7 @@ fn compile_args_without_check_args(app: Command) -> Command {
   app
     .arg(import_map_arg())
     .arg(no_remote_arg())
+    .arg(local_npm_arg())
     .arg(config_arg())
     .arg(no_config_arg())
     .arg(reload_arg())
@@ -2144,6 +2147,12 @@ fn no_remote_arg<'a>() -> Arg<'a> {
     .help("Do not resolve remote modules")
 }
 
+fn local_npm_arg<'a>() -> Arg<'a> {
+  Arg::new("node-modules-dir")
+    .long("node-modules-dir")
+    .help("Creates a local node_modules folder")
+}
+
 fn unsafely_ignore_certificate_errors_arg<'a>() -> Arg<'a> {
   Arg::new("unsafely-ignore-certificate-errors")
     .long("unsafely-ignore-certificate-errors")
@@ -2789,6 +2798,7 @@ fn vendor_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
 fn compile_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   import_map_arg_parse(flags, matches);
   no_remote_arg_parse(flags, matches);
+  local_npm_args_parse(flags, matches);
   config_args_parse(flags, matches);
   no_check_arg_parse(flags, matches);
   check_arg_parse(flags, matches);
@@ -2803,6 +2813,7 @@ fn compile_args_without_no_check_parse(
 ) {
   import_map_arg_parse(flags, matches);
   no_remote_arg_parse(flags, matches);
+  local_npm_args_parse(flags, matches);
   config_args_parse(flags, matches);
   reload_arg_parse(flags, matches);
   lock_args_parse(flags, matches);
@@ -3044,6 +3055,12 @@ fn no_remote_arg_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   }
 }
 
+fn local_npm_args_parse(flags: &mut Flags, matches: &ArgMatches) {
+  if matches.is_present("node-modules-dir") {
+    flags.node_modules_dir = true;
+  }
+}
+
 fn inspect_arg_validate(val: &str) -> Result<(), String> {
   match val.parse::<SocketAddr>() {
     Ok(_) => Ok(()),
@@ -5002,6 +5019,22 @@ mod tests {
     );
   }
 
+  #[test]
+  fn local_npm() {
+    let r =
+      flags_from_vec(svec!["deno", "run", "--node-modules-dir", "script.ts"]);
+    assert_eq!(
+      r.unwrap(),
+      Flags {
+        subcommand: DenoSubcommand::Run(RunFlags {
+          script: "script.ts".to_string(),
+        }),
+        node_modules_dir: true,
+        ..Flags::default()
+      }
+    );
+  }
+
   #[test]
   fn cached_only() {
     let r = flags_from_vec(svec!["deno", "run", "--cached-only", "script.ts"]);

cli/args/flags.rs

@@ -42,6 +42,7 @@ use crate::emit::TsConfigWithIgnoredOptions;
 use crate::emit::TsTypeLib;
 use crate::file_fetcher::get_root_cert_store;
 use crate::file_fetcher::CacheSetting;
+use crate::fs_util;
 use crate::lockfile::Lockfile;
 use crate::version;
 
@@ -146,6 +147,24 @@ impl CliOptions {
     self.overrides.import_map_specifier = Some(path);
   }
 
+  /// Resolves the path to use for a local node_modules folder.
+  pub fn resolve_local_node_modules_folder(
+    &self,
+  ) -> Result<Option<PathBuf>, AnyError> {
+    let path = if !self.flags.node_modules_dir {
+      return Ok(None);
+    } else if let Some(config_path) = self
+      .maybe_config_file
+      .as_ref()
+      .and_then(|c| c.specifier.to_file_path().ok())
+    {
+      config_path.parent().unwrap().join("node_modules")
+    } else {
+      std::env::current_dir()?.join("node_modules")
+    };
+    Ok(Some(fs_util::canonicalize_path_maybe_not_exists(&path)?))
+  }
+
   pub fn resolve_root_cert_store(&self) -> Result<RootCertStore, AnyError> {
     get_root_cert_store(
       None,

cli/args/mod.rs

@@ -5,10 +5,11 @@ use deno_core::error::{uri_error, AnyError};
 pub use deno_core::normalize_path;
 use deno_core::ModuleSpecifier;
 use deno_runtime::deno_crypto::rand;
+use deno_runtime::deno_node::PathClean;
 use std::borrow::Cow;
 use std::env::current_dir;
 use std::fs::OpenOptions;
-use std::io::{Error, Write};
+use std::io::{Error, ErrorKind, Write};
 use std::path::{Path, PathBuf};
 use walkdir::WalkDir;
 
@@ -75,6 +76,35 @@ pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
   return Ok(path);
 }
 
+/// Canonicalizes a path which might be non-existent by going up the
+/// ancestors until it finds a directory that exists, canonicalizes
+/// that path, then adds back the remaining path components.
+///
+/// Note: When using this, you should be aware that a symlink may
+/// subsequently be created along this path by some other code.
+pub fn canonicalize_path_maybe_not_exists(
+  path: &Path,
+) -> Result<PathBuf, Error> {
+  let path = path.to_path_buf().clean();
+  let mut path = path.as_path();
+  let mut names_stack = Vec::new();
+  loop {
+    match canonicalize_path(path) {
+      Ok(mut canonicalized_path) => {
+        for name in names_stack.into_iter().rev() {
+          canonicalized_path = canonicalized_path.join(name);
+        }
+        return Ok(canonicalized_path);
+      }
+      Err(err) if err.kind() == ErrorKind::NotFound => {
+        names_stack.push(path.file_name().unwrap());
+        path = path.parent().unwrap();
+      }
+      Err(err) => return Err(err),
+    }
+  }
+}
+
 #[cfg(windows)]
 fn strip_unc_prefix(path: PathBuf) -> PathBuf {
   use std::path::Component;
@@ -294,6 +324,60 @@ pub async fn remove_dir_all_if_exists(path: &Path) -> std::io::Result<()> {
   }
 }
 
+/// Copies a directory to another directory.
+///
+/// Note: Does not handle symlinks.
+pub fn copy_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> {
+  std::fs::create_dir_all(&to)
+    .with_context(|| format!("Creating {}", to.display()))?;
+  let read_dir = std::fs::read_dir(&from)
+    .with_context(|| format!("Reading {}", from.display()))?;
+
+  for entry in read_dir {
+    let entry = entry?;
+    let file_type = entry.file_type()?;
+    let new_from = from.join(entry.file_name());
+    let new_to = to.join(entry.file_name());
+
+    if file_type.is_dir() {
+      copy_dir_recursive(&new_from, &new_to).with_context(|| {
+        format!("Dir {} to {}", new_from.display(), new_to.display())
+      })?;
+    } else if file_type.is_file() {
+      std::fs::copy(&new_from, &new_to).with_context(|| {
+        format!("Copying {} to {}", new_from.display(), new_to.display())
+      })?;
+    }
+  }
+
+  Ok(())
+}
+
+pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), AnyError> {
+  let err_mapper = |err: Error| {
+    Error::new(
+      err.kind(),
+      format!(
+        "{}, symlink '{}' -> '{}'",
+        err,
+        oldpath.display(),
+        newpath.display()
+      ),
+    )
+  };
+  #[cfg(unix)]
+  {
+    use std::os::unix::fs::symlink;
+    symlink(&oldpath, &newpath).map_err(err_mapper)?;
+  }
+  #[cfg(not(unix))]
+  {
+    use std::os::windows::fs::symlink_dir;
+    symlink_dir(&oldpath, &newpath).map_err(err_mapper)?;
+  }
+  Ok(())
+}
+
 /// Attempts to convert a specifier to a file path. By default, uses the Url
 /// crate's `to_file_path()` method, but falls back to try and resolve unix-style
 /// paths on Windows.

cli/fs_util.rs

@@ -26,11 +26,11 @@ use deno_runtime::deno_node::package_imports_resolve;
 use deno_runtime::deno_node::package_resolve;
 use deno_runtime::deno_node::NodeModuleKind;
 use deno_runtime::deno_node::PackageJson;
+use deno_runtime::deno_node::PathClean;
 use deno_runtime::deno_node::RequireNpmResolver;
 use deno_runtime::deno_node::DEFAULT_CONDITIONS;
 use deno_runtime::deno_node::NODE_GLOBAL_THIS_NAME;
 use once_cell::sync::Lazy;
-use path_clean::PathClean;
 use regex::Regex;
 
 use crate::file_fetcher::FileFetcher;
@@ -433,9 +433,8 @@ pub fn node_resolve_npm_reference(
   reference: &NpmPackageReference,
   npm_resolver: &NpmPackageResolver,
 ) -> Result<Option<NodeResolution>, AnyError> {
-  let package_folder = npm_resolver
-    .resolve_package_from_deno_module(&reference.req)?
-    .folder_path;
+  let package_folder =
+    npm_resolver.resolve_package_folder_from_deno_module(&reference.req)?;
   let resolved_path = package_config_resolve(
     &reference
       .sub_path
@@ -462,15 +461,28 @@ pub fn node_resolve_binary_export(
   bin_name: Option<&str>,
   npm_resolver: &NpmPackageResolver,
 ) -> Result<NodeResolution, AnyError> {
-  let pkg = npm_resolver.resolve_package_from_deno_module(pkg_req)?;
-  let package_folder = pkg.folder_path;
+  fn get_package_display_name(package_json: &PackageJson) -> String {
+    package_json
+      .name
+      .as_ref()
+      .and_then(|name| {
+        package_json
+          .version
+          .as_ref()
+          .map(|version| format!("{}@{}", name, version))
+      })
+      .unwrap_or_else(|| format!("{}", package_json.path.display()))
+  }
+
+  let package_folder =
+    npm_resolver.resolve_package_folder_from_deno_module(pkg_req)?;
   let package_json_path = package_folder.join("package.json");
   let package_json = PackageJson::load(npm_resolver, package_json_path)?;
   let bin = match &package_json.bin {
     Some(bin) => bin,
     None => bail!(
       "package {} did not have a 'bin' property in its package.json",
-      pkg.id
+      get_package_display_name(&package_json),
     ),
   };
   let bin_entry = match bin {
@@ -490,21 +502,21 @@ pub fn node_resolve_binary_export(
         o.get(&pkg_req.name)
       }
     },
-    _ => bail!("package {} did not have a 'bin' property with a string or object value in its package.json", pkg.id),
+    _ => bail!("package {} did not have a 'bin' property with a string or object value in its package.json", get_package_display_name(&package_json)),
   };
   let bin_entry = match bin_entry {
     Some(e) => e,
     None => bail!(
       "package {} did not have a 'bin' entry for {} in its package.json",
-      pkg.id,
+      get_package_display_name(&package_json),
       bin_name.unwrap_or(&pkg_req.name),
     ),
   };
   let bin_entry = match bin_entry {
     Value::String(s) => s,
     _ => bail!(
       "package {} had a non-string sub property of 'bin' in its package.json",
-      pkg.id
+      get_package_display_name(&package_json),
     ),
   };
 

cli/node/mod.rs

@@ -2,6 +2,7 @@
 
 use std::cmp::Ordering;
 use std::collections::HashMap;
+use std::collections::HashSet;
 use std::collections::VecDeque;
 
 use deno_ast::ModuleSpecifier;
@@ -185,6 +186,26 @@ impl NpmResolutionSnapshot {
     }
   }
 
+  pub fn top_level_packages(&self) -> Vec<NpmPackageId> {
+    self
+      .package_reqs
+      .iter()
+      .map(|(req, version)| NpmPackageId {
+        name: req.name.clone(),
+        version: version.clone(),
+      })
+      .collect::<HashSet<_>>()
+      .into_iter()
+      .collect::<Vec<_>>()
+  }
+
+  pub fn package_from_id(
+    &self,
+    id: &NpmPackageId,
+  ) -> Option<&NpmResolutionPackage> {
+    self.packages.get(id)
+  }
+
   pub fn resolve_package_from_package(
     &self,
     name: &str,
@@ -471,8 +492,6 @@ impl NpmResolution {
     !self.snapshot.read().packages.is_empty()
   }
 
-  // todo(dsherret): for use in the lsp
-  #[allow(dead_code)]
   pub fn snapshot(&self) -> NpmResolutionSnapshot {
     self.snapshot.read().clone()
   }