feat(vendor): support for npm specifiers

cli/args/flags.rs

@@ -1343,7 +1343,7 @@ TypeScript compiler cache: Subdirectory containing TS compiler output.",
     .arg(lock_arg())
     .arg(config_arg())
     .arg(import_map_arg())
-    .arg(local_npm_arg())
+    .arg(node_modules_dir_arg())
     .arg(
       Arg::new("json")
         .long("json")
@@ -1862,6 +1862,7 @@ Remote modules and multiple modules may also be specified:
     .arg(config_arg())
     .arg(import_map_arg())
     .arg(lock_arg())
+    .arg(node_modules_dir_arg())
     .arg(reload_arg())
     .arg(ca_file_arg())
 }
@@ -1875,7 +1876,7 @@ fn compile_args_without_check_args(app: Command) -> Command {
     .arg(import_map_arg())
     .arg(no_remote_arg())
     .arg(no_npm_arg())
-    .arg(local_npm_arg())
+    .arg(node_modules_dir_arg())
     .arg(config_arg())
     .arg(no_config_arg())
     .arg(reload_arg())
@@ -2424,7 +2425,7 @@ fn no_npm_arg() -> Arg {
     .help("Do not resolve npm modules")
 }
 
-fn local_npm_arg() -> Arg {
+fn node_modules_dir_arg() -> Arg {
   Arg::new("node-modules-dir")
     .long("node-modules-dir")
     .num_args(0..=1)
@@ -2719,7 +2720,7 @@ fn info_parse(flags: &mut Flags, matches: &mut ArgMatches) {
   import_map_arg_parse(flags, matches);
   location_arg_parse(flags, matches);
   ca_file_arg_parse(flags, matches);
-  local_npm_args_parse(flags, matches);
+  node_modules_dir_arg_parse(flags, matches);
   lock_arg_parse(flags, matches);
   no_lock_arg_parse(flags, matches);
   no_remote_arg_parse(flags, matches);
@@ -2975,6 +2976,7 @@ fn vendor_parse(flags: &mut Flags, matches: &mut ArgMatches) {
   config_args_parse(flags, matches);
   import_map_arg_parse(flags, matches);
   lock_arg_parse(flags, matches);
+  node_modules_dir_arg_parse(flags, matches);
   reload_arg_parse(flags, matches);
 
   flags.subcommand = DenoSubcommand::Vendor(VendorFlags {
@@ -3000,7 +3002,7 @@ fn compile_args_without_check_parse(
   import_map_arg_parse(flags, matches);
   no_remote_arg_parse(flags, matches);
   no_npm_arg_parse(flags, matches);
-  local_npm_args_parse(flags, matches);
+  node_modules_dir_arg_parse(flags, matches);
   config_args_parse(flags, matches);
   reload_arg_parse(flags, matches);
   lock_args_parse(flags, matches);
@@ -3254,7 +3256,7 @@ fn no_npm_arg_parse(flags: &mut Flags, matches: &mut ArgMatches) {
   }
 }
 
-fn local_npm_args_parse(flags: &mut Flags, matches: &mut ArgMatches) {
+fn node_modules_dir_arg_parse(flags: &mut Flags, matches: &mut ArgMatches) {
   flags.node_modules_dir = matches.remove_one::<bool>("node-modules-dir");
 }
 

cli/args/mod.rs

@@ -881,6 +881,15 @@ impl CliOptions {
     self.maybe_node_modules_folder.clone()
   }
 
+  pub fn node_modules_dir_enablement(&self) -> Option<bool> {
+    self.flags.node_modules_dir.or_else(|| {
+      self
+        .maybe_config_file
+        .as_ref()
+        .and_then(|c| c.node_modules_dir())
+    })
+  }
+
   pub fn node_modules_dir_specifier(&self) -> Option<ModuleSpecifier> {
     self
       .maybe_node_modules_folder

cli/factory.rs

@@ -29,6 +29,7 @@ use crate::npm::create_npm_fs_resolver;
 use crate::npm::CliNpmRegistryApi;
 use crate::npm::CliNpmResolver;
 use crate::npm::NpmCache;
+use crate::npm::NpmPackageFsResolver;
 use crate::npm::NpmResolution;
 use crate::npm::PackageJsonDepsInstaller;
 use crate::resolver::CliGraphResolver;
@@ -325,6 +326,23 @@ impl CliFactory {
       .await
   }
 
+  pub async fn create_node_modules_npm_fs_resolver(
+    &self,
+    node_modules_dir_path: PathBuf,
+  ) -> Result<Arc<dyn NpmPackageFsResolver>, AnyError> {
+    Ok(create_npm_fs_resolver(
+      self.fs().clone(),
+      self.npm_cache()?.clone(),
+      self.text_only_progress_bar(),
+      CliNpmRegistryApi::default_url().to_owned(),
+      self.npm_resolution().await?.clone(),
+      // when an explicit path is provided here, it will create the
+      // local node_modules variant of an npm fs resolver
+      Some(node_modules_dir_path),
+      self.options.npm_system_info(),
+    ))
+  }
+
   pub fn package_json_deps_provider(&self) -> &Arc<PackageJsonDepsProvider> {
     self.services.package_json_deps_provider.get_or_init(|| {
       Arc::new(PackageJsonDepsProvider::new(

cli/npm/mod.rs

@@ -14,4 +14,5 @@ pub use registry::CliNpmRegistryApi;
 pub use resolution::NpmResolution;
 pub use resolvers::create_npm_fs_resolver;
 pub use resolvers::CliNpmResolver;
+pub use resolvers::NpmPackageFsResolver;
 pub use resolvers::NpmProcessState;

cli/npm/resolvers/mod.rs

@@ -36,11 +36,12 @@ use crate::args::Lockfile;
 use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
 use crate::util::progress_bar::ProgressBar;
 
-use self::common::NpmPackageFsResolver;
 use self::local::LocalNpmPackageResolver;
 use super::resolution::NpmResolution;
 use super::NpmCache;
 
+pub use self::common::NpmPackageFsResolver;
+
 /// State provided to the process via an environment variable.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct NpmProcessState {

cli/tests/integration/vendor_tests.rs

@@ -10,6 +10,7 @@ use test_util as util;
 use test_util::TempDir;
 use util::http_server;
 use util::new_deno_dir;
+use util::TestContextBuilder;
 
 #[test]
 fn output_dir_exists() {
@@ -186,15 +187,13 @@ fn import_map_output_dir() {
     String::from_utf8_lossy(&output.stderr).trim(),
     format!(
       concat!(
-        "Ignoring import map. Specifying an import map file ({}) in the deno ",
-        "vendor output directory is not supported. If you wish to use an ",
-        "import map while vendoring, please specify one located outside this ",
-        "directory.\n",
+        "{}\n",
         "Download http://localhost:4545/vendor/logger.ts\n",
-        "{}",
+        "{}\n\n{}",
       ),
-      PathBuf::from("vendor").join("import_map.json").display(),
-      success_text_updated_deno_json("1 module", "vendor/"),
+      ignoring_import_map_text(),
+      vendored_text("1 module", "vendor/"),
+      success_text_updated_deno_json("vendor/"),
     )
   );
   assert!(output.status.success());
@@ -511,8 +510,9 @@ fn update_existing_config_test() {
   assert_eq!(
     String::from_utf8_lossy(&output.stderr).trim(),
     format!(
-      "Download http://localhost:4545/vendor/logger.ts\n{}",
-      success_text_updated_deno_json("1 module", "vendor2",)
+      "Download http://localhost:4545/vendor/logger.ts\n{}\n\n{}",
+      vendored_text("1 module", "vendor2"),
+      success_text_updated_deno_json("vendor2",)
     )
   );
   assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "");
@@ -537,38 +537,158 @@ fn update_existing_config_test() {
   assert!(output.status.success());
 }
 
+#[test]
+fn vendor_npm_node_specifiers() {
+  let context = TestContextBuilder::for_npm().use_temp_cwd().build();
+  let temp_dir = context.temp_dir();
+  temp_dir.write(
+    "my_app.ts",
+    concat!(
+      "import { path, getValue, setValue } from 'http://localhost:4545/vendor/npm_and_node_specifier.ts';\n",
+      "setValue(5);\n",
+      "console.log(path.isAbsolute(Deno.cwd()), getValue());",
+    ),
+  );
+  temp_dir.write("deno.json", "{}");
+
+  let output = context.new_command().args("vendor my_app.ts").run();
+  output.assert_matches_text(
+    format!(
+      concat!(
+        "Download http://localhost:4545/vendor/npm_and_node_specifier.ts\n",
+        "Download http://localhost:4545/npm/registry/@denotest/esm-basic\n",
+        "Download http://localhost:4545/npm/registry/@denotest/esm-basic/1.0.0.tgz\n",
+        "{}\n",
+        "Initialize @denotest/[email protected]\n",
+        "{}\n\n",
+        "{}\n",
+      ),
+      vendored_text("1 module", "vendor/"),
+      vendored_npm_package_text("1 npm package"),
+      success_text_updated_deno_json("vendor/")
+    )
+  );
+  let output = context.new_command().args("run -A my_app.ts").run();
+  output.assert_matches_text("true 5\n");
+  assert!(temp_dir.path().join("node_modules").exists());
+  assert!(temp_dir.path().join("deno.lock").exists());
+
+  // now try re-vendoring with a lockfile
+  let output = context.new_command().args("vendor --force my_app.ts").run();
+  output.assert_matches_text(format!(
+    "{}\n{}\n\n{}\n",
+    ignoring_import_map_text(),
+    vendored_text("1 module", "vendor/"),
+    success_text_updated_deno_json("vendor/"),
+  ));
+
+  // delete the node_modules folder
+  temp_dir.remove_dir_all("node_modules");
+
+  // vendor with --node-modules-dir=false
+  let output = context
+    .new_command()
+    .args("vendor --node-modules-dir=false --force my_app.ts")
+    .run();
+  output.assert_matches_text(format!(
+    "{}\n{}\n\n{}\n",
+    ignoring_import_map_text(),
+    vendored_text("1 module", "vendor/"),
+    success_text_updated_deno_json("vendor/")
+  ));
+  assert!(!temp_dir.path().join("node_modules").exists());
+
+  // delete the deno.json
+  temp_dir.remove_file("deno.json");
+
+  // vendor with --node-modules-dir
+  let output = context
+    .new_command()
+    .args("vendor --node-modules-dir --force my_app.ts")
+    .run();
+  output.assert_matches_text(format!(
+    "Initialize @denotest/[email protected]\n{}\n\n{}\n",
+    vendored_text("1 module", "vendor/"),
+    use_import_map_text("vendor/")
+  ));
+}
+
+#[test]
+fn vendor_only_npm_specifiers() {
+  let context = TestContextBuilder::for_npm().use_temp_cwd().build();
+  let temp_dir = context.temp_dir();
+  temp_dir.write(
+    "my_app.ts",
+    concat!(
+      "import { getValue, setValue } from 'npm:@denotest/esm-basic';\n",
+      "setValue(5);\n",
+      "console.log(path.isAbsolute(Deno.cwd()), getValue());",
+    ),
+  );
+  temp_dir.write("deno.json", "{}");
+
+  let output = context.new_command().args("vendor my_app.ts").run();
+  output.assert_matches_text(
+    format!(
+      concat!(
+        "Download http://localhost:4545/npm/registry/@denotest/esm-basic\n",
+        "Download http://localhost:4545/npm/registry/@denotest/esm-basic/1.0.0.tgz\n",
+        "{}\n",
+        "Initialize @denotest/[email protected]\n",
+        "{}\n",
+      ),
+      vendored_text("0 modules", "vendor/"),
+      vendored_npm_package_text("1 npm package"),
+    )
+  );
+}
+
 fn success_text(module_count: &str, dir: &str, has_import_map: bool) -> String {
   let mut text = format!("Vendored {module_count} into {dir} directory.");
   if has_import_map {
-    let f = format!(
-      concat!(
-        "\n\nTo use vendored modules, specify the `--import-map {}import_map.json` flag when ",
-        r#"invoking Deno subcommands or add an `"importMap": "<path_to_vendored_import_map>"` "#,
-        "entry to a deno.json file.",
-      ),
-      if dir != "vendor/" {
-        format!("{}{}", dir.trim_end_matches('/'), if cfg!(windows) { '\\' } else {'/'})
-      } else {
-        dir.to_string()
-      }
-    );
-    write!(text, "{f}").unwrap();
+    write!(text, "\n\n{}", use_import_map_text(dir)).unwrap();
   }
   text
 }
 
-fn success_text_updated_deno_json(module_count: &str, dir: &str) -> String {
+fn use_import_map_text(dir: &str) -> String {
+  format!(
+    concat!(
+      "To use vendored modules, specify the `--import-map {}import_map.json` flag when ",
+      r#"invoking Deno subcommands or add an `"importMap": "<path_to_vendored_import_map>"` "#,
+      "entry to a deno.json file.",
+    ),
+    if dir != "vendor/" {
+      format!("{}{}", dir.trim_end_matches('/'), if cfg!(windows) { '\\' } else {'/'})
+    } else {
+      dir.to_string()
+    }
+  )
+}
+
+fn vendored_text(module_count: &str, dir: &str) -> String {
+  format!("Vendored {} into {} directory.", module_count, dir)
+}
+
+fn vendored_npm_package_text(package_count: &str) -> String {
+  format!(
+    concat!(
+     "Vendored {} into node_modules directory. Set `nodeModulesDir: false` ",
+     "in the Deno configuration file to disable vendoring npm packages in the future.",
+    ),
+    package_count
+  )
+}
+
+fn success_text_updated_deno_json(dir: &str) -> String {
   format!(
     concat!(
-      "Vendored {} into {} directory.\n\n",
       "Updated your local Deno configuration file with a reference to the ",
       "new vendored import map at {}import_map.json. Invoking Deno subcommands will ",
       "now automatically resolve using the vendored modules. You may override ",
       "this by providing the `--import-map <other-import-map>` flag or by ",
       "manually editing your Deno configuration file.",
     ),
-    module_count,
-    dir,
     if dir != "vendor/" {
       format!(
         "{}{}",
@@ -580,3 +700,15 @@ fn success_text_updated_deno_json(module_count: &str, dir: &str) -> String {
     }
   )
 }
+
+fn ignoring_import_map_text() -> String {
+  format!(
+    concat!(
+      "Ignoring import map. Specifying an import map file ({}) in the deno ",
+      "vendor output directory is not supported. If you wish to use an ",
+      "import map while vendoring, please specify one located outside this ",
+      "directory.",
+    ),
+    PathBuf::from("vendor").join("import_map.json").display(),
+  )
+}

cli/tests/testdata/vendor/npm_and_node_specifier.ts

@@ -0,0 +1,2 @@
+export { default as path } from "node:path";
+export { getValue, setValue } from "npm:@denotest/esm-basic";

cli/tools/vendor/import_map.rs

@@ -304,7 +304,7 @@ fn handle_dep_specifier(
       referrer,
       mappings,
     )
-  } else {
+  } else if specifier.scheme() == "file" {
     handle_local_dep_specifier(
       text,
       unresolved_specifier,