fix(npm): reload an npm package's dependency's information when version not found (#18622)

This reloads an npm package's dependency's information when a
version/version req/tag is not found.

This PR applies only to dependencies of npm packages. It does NOT yet
cause npm specifiers to have their dependency information cache busted.
That requires a different solution, but this should help cache bust in
more scenarios.

Part of #16901, but doesn't close it yet

Author: dsherret

Cargo.lock

@@ -47,7 +47,7 @@ deno_emit = "0.18.0"
 deno_graph = "=0.46.0"
 deno_lint = { version = "0.43.0", features = ["docs"] }
 deno_lockfile.workspace = true
-deno_npm = "0.1.0"
+deno_npm = "0.2.0"
 deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "include_js_files_for_snapshotting"] }
 deno_semver = "0.2.0"
 deno_task_shell = "0.10.0"

cli/Cargo.toml

@@ -117,10 +117,11 @@ pub async fn snapshot_from_lockfile(
   let mut version_infos =
       FuturesOrdered::from_iter(packages.iter().map(|p| p.pkg_id.nv.clone()).map(
         |nv| async move {
-          match api.package_version_info(&nv).await? {
-            Some(version_info) => Ok(version_info),
-            None => {
-              bail!("could not find '{}' specified in the lockfile. Maybe try again with --reload", nv);
+          let package_info = api.package_info(&nv.name).await?;
+          match package_info.version_info(&nv) {
+            Ok(version_info) => Ok(version_info),
+            Err(err) => {
+              bail!("Could not find '{}' specified in the lockfile. Maybe try again with --reload", err.0);
             }
           }
         },

cli/args/lockfile.rs

@@ -5,6 +5,8 @@ use std::collections::HashSet;
 use std::fs;
 use std::io::ErrorKind;
 use std::path::PathBuf;
+use std::sync::atomic::AtomicBool;
+use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
 use async_trait::async_trait;
@@ -21,6 +23,7 @@ use deno_core::url::Url;
 use deno_core::TaskQueue;
 use deno_npm::registry::NpmPackageInfo;
 use deno_npm::registry::NpmRegistryApi;
+use deno_npm::registry::NpmRegistryPackageInfoLoadError;
 use once_cell::sync::Lazy;
 
 use crate::args::CacheSetting;
@@ -67,6 +70,7 @@ impl NpmRegistry {
     Self(Some(Arc::new(NpmRegistryApiInner {
       base_url,
       cache,
+      force_reload: AtomicBool::new(false),
       mem_cache: Default::default(),
       previously_reloaded_packages: Default::default(),
       http_client,
@@ -96,6 +100,18 @@ impl NpmRegistry {
     &self.inner().base_url
   }
 
+  /// Marks that new requests for package information should retrieve it
+  /// from the npm registry
+  ///
+  /// Returns true if it was successfully set for the first time.
+  pub fn mark_force_reload(&self) -> bool {
+    // never force reload the registry information if reloading is disabled
+    if matches!(self.inner().cache.cache_setting(), CacheSetting::Only) {
+      return false;
+    }
+    !self.inner().force_reload.swap(true, Ordering::SeqCst)
+  }
+
   fn inner(&self) -> &Arc<NpmRegistryApiInner> {
     // this panicking indicates a bug in the code where this
     // wasn't initialized
@@ -108,17 +124,26 @@ static SYNC_DOWNLOAD_TASK_QUEUE: Lazy<TaskQueue> =
 
 #[async_trait]
 impl NpmRegistryApi for NpmRegistry {
-  async fn maybe_package_info(
+  async fn package_info(
     &self,
     name: &str,
-  ) -> Result<Option<Arc<NpmPackageInfo>>, AnyError> {
-    if should_sync_download() {
+  ) -> Result<Arc<NpmPackageInfo>, NpmRegistryPackageInfoLoadError> {
+    let result = if should_sync_download() {
       let inner = self.inner().clone();
       SYNC_DOWNLOAD_TASK_QUEUE
         .queue(async move { inner.maybe_package_info(name).await })
         .await
     } else {
       self.inner().maybe_package_info(name).await
+    };
+    match result {
+      Ok(Some(info)) => Ok(info),
+      Ok(None) => Err(NpmRegistryPackageInfoLoadError::PackageNotExists {
+        package_name: name.to_string(),
+      }),
+      Err(err) => {
+        Err(NpmRegistryPackageInfoLoadError::LoadError(Arc::new(err)))
+      }
     }
   }
 }
@@ -135,6 +160,7 @@ enum CacheItem {
 struct NpmRegistryApiInner {
   base_url: Url,
   cache: NpmCache,
+  force_reload: AtomicBool,
   mem_cache: Mutex<HashMap<String, CacheItem>>,
   previously_reloaded_packages: Mutex<HashSet<String>>,
   http_client: HttpClient,
@@ -154,10 +180,10 @@ impl NpmRegistryApiInner {
         }
         Some(CacheItem::Pending(future)) => (false, future.clone()),
         None => {
-          if self.cache.cache_setting().should_use_for_npm_package(name)
-        // if this has been previously reloaded, then try loading from the
-        // file system cache
-        || !self.previously_reloaded_packages.lock().insert(name.to_string())
+          if (self.cache.cache_setting().should_use_for_npm_package(name) && !self.force_reload())
+            // if this has been previously reloaded, then try loading from the
+            // file system cache
+            || !self.previously_reloaded_packages.lock().insert(name.to_string())
           {
             // attempt to load from the file cache
             if let Some(info) = self.load_file_cached_package_info(name) {
@@ -203,6 +229,10 @@ impl NpmRegistryApiInner {
     }
   }
 
+  fn force_reload(&self) -> bool {
+    self.force_reload.load(Ordering::SeqCst)
+  }
+
   fn load_file_cached_package_info(
     &self,
     name: &str,

cli/npm/registry.rs

@@ -10,8 +10,13 @@ use deno_core::TaskQueue;
 use deno_lockfile::NpmPackageDependencyLockfileInfo;
 use deno_lockfile::NpmPackageLockfileInfo;
 use deno_npm::registry::NpmPackageInfo;
+use deno_npm::resolution::NpmPackageVersionResolutionError;
 use deno_npm::resolution::NpmPackagesPartitioned;
+use deno_npm::resolution::NpmResolutionError;
 use deno_npm::resolution::NpmResolutionSnapshot;
+use deno_npm::resolution::PackageNotFoundFromReferrerError;
+use deno_npm::resolution::PackageNvNotFoundError;
+use deno_npm::resolution::PackageReqNotFoundError;
 use deno_npm::NpmPackageCacheFolderId;
 use deno_npm::NpmPackageId;
 use deno_npm::NpmResolutionPackage;
@@ -70,13 +75,11 @@ impl NpmResolution {
 
     // only allow one thread in here at a time
     let _permit = inner.update_queue.acquire().await;
-    let snapshot = inner.snapshot.read().clone();
-
     let snapshot = add_package_reqs_to_snapshot(
       &inner.api,
       package_reqs,
-      snapshot,
       self.0.maybe_lockfile.clone(),
+      || inner.snapshot.read().clone(),
     )
     .await?;
 
@@ -91,24 +94,25 @@ impl NpmResolution {
     let inner = &self.0;
     // only allow one thread in here at a time
     let _permit = inner.update_queue.acquire().await;
-    let snapshot = inner.snapshot.read().clone();
-
-    let reqs_set = package_reqs.iter().collect::<HashSet<_>>();
-    let has_removed_package = !snapshot
-      .package_reqs()
-      .keys()
-      .all(|req| reqs_set.contains(req));
-    // if any packages were removed, we need to completely recreate the npm resolution snapshot
-    let snapshot = if has_removed_package {
-      NpmResolutionSnapshot::default()
-    } else {
-      snapshot
-    };
+
+    let reqs_set = package_reqs.iter().cloned().collect::<HashSet<_>>();
     let snapshot = add_package_reqs_to_snapshot(
       &inner.api,
       package_reqs,
-      snapshot,
       self.0.maybe_lockfile.clone(),
+      || {
+        let snapshot = inner.snapshot.read().clone();
+        let has_removed_package = !snapshot
+          .package_reqs()
+          .keys()
+          .all(|req| reqs_set.contains(req));
+        // if any packages were removed, we need to completely recreate the npm resolution snapshot
+        if has_removed_package {
+          NpmResolutionSnapshot::default()
+        } else {
+          snapshot
+        }
+      },
     )
     .await?;
 
@@ -121,13 +125,12 @@ impl NpmResolution {
     let inner = &self.0;
     // only allow one thread in here at a time
     let _permit = inner.update_queue.acquire().await;
-    let snapshot = inner.snapshot.read().clone();
 
     let snapshot = add_package_reqs_to_snapshot(
       &inner.api,
       Vec::new(),
-      snapshot,
       self.0.maybe_lockfile.clone(),
+      || inner.snapshot.read().clone(),
     )
     .await?;
 
@@ -139,7 +142,7 @@ impl NpmResolution {
   pub fn pkg_req_ref_to_nv_ref(
     &self,
     req_ref: NpmPackageReqReference,
-  ) -> Result<NpmPackageNvReference, AnyError> {
+  ) -> Result<NpmPackageNvReference, PackageReqNotFoundError> {
     let node_id = self.resolve_pkg_id_from_pkg_req(&req_ref.req)?;
     Ok(NpmPackageNvReference {
       nv: node_id.nv,
@@ -163,7 +166,7 @@ impl NpmResolution {
     &self,
     name: &str,
     referrer: &NpmPackageCacheFolderId,
-  ) -> Result<NpmResolutionPackage, AnyError> {
+  ) -> Result<NpmResolutionPackage, Box<PackageNotFoundFromReferrerError>> {
     self
       .0
       .snapshot
@@ -176,7 +179,7 @@ impl NpmResolution {
   pub fn resolve_pkg_id_from_pkg_req(
     &self,
     req: &NpmPackageReq,
-  ) -> Result<NpmPackageId, AnyError> {
+  ) -> Result<NpmPackageId, PackageReqNotFoundError> {
     self
       .0
       .snapshot
@@ -188,7 +191,7 @@ impl NpmResolution {
   pub fn resolve_pkg_id_from_deno_module(
     &self,
     id: &NpmPackageNv,
-  ) -> Result<NpmPackageId, AnyError> {
+  ) -> Result<NpmPackageId, PackageNvNotFoundError> {
     self
       .0
       .snapshot
@@ -203,7 +206,7 @@ impl NpmResolution {
   pub fn resolve_package_req_as_pending(
     &self,
     pkg_req: &NpmPackageReq,
-  ) -> Result<NpmPackageNv, AnyError> {
+  ) -> Result<NpmPackageNv, NpmPackageVersionResolutionError> {
     // we should always have this because it should have been cached before here
     let package_info =
       self.0.api.get_cached_package_info(&pkg_req.name).unwrap();
@@ -217,7 +220,7 @@ impl NpmResolution {
     &self,
     pkg_req: &NpmPackageReq,
     package_info: &NpmPackageInfo,
-  ) -> Result<NpmPackageNv, AnyError> {
+  ) -> Result<NpmPackageNv, NpmPackageVersionResolutionError> {
     debug_assert_eq!(pkg_req.name, package_info.name);
     let inner = &self.0;
     let mut snapshot = inner.snapshot.write();
@@ -245,10 +248,14 @@ impl NpmResolution {
 
 async fn add_package_reqs_to_snapshot(
   api: &NpmRegistry,
+  // todo(18079): it should be possible to pass &[NpmPackageReq] in here
+  // and avoid all these clones, but the LSP complains because of its
+  // `Send` requirement
   package_reqs: Vec<NpmPackageReq>,
-  snapshot: NpmResolutionSnapshot,
   maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
+  get_new_snapshot: impl Fn() -> NpmResolutionSnapshot,
 ) -> Result<NpmResolutionSnapshot, AnyError> {
+  let snapshot = get_new_snapshot();
   if !snapshot.has_pending()
     && package_reqs
       .iter()
@@ -257,9 +264,23 @@ async fn add_package_reqs_to_snapshot(
     return Ok(snapshot); // already up to date
   }
 
-  let result = snapshot.resolve_pending(package_reqs, api).await;
+  let result = snapshot.resolve_pending(package_reqs.clone(), api).await;
   api.clear_memory_cache();
-  let snapshot = result?; // propagate the error after clearing the memory cache
+  let snapshot = match result {
+    Ok(snapshot) => snapshot,
+    Err(NpmResolutionError::Resolution(err)) if api.mark_force_reload() => {
+      log::debug!("{err:#}");
+      log::debug!("npm resolution failed. Trying again...");
+
+      // try again
+      let snapshot = get_new_snapshot();
+      let result = snapshot.resolve_pending(package_reqs, api).await;
+      api.clear_memory_cache();
+      // now surface the result after clearing the cache
+      result?
+    }
+    Err(err) => return Err(err.into()),
+  };
 
   if let Some(lockfile_mutex) = maybe_lockfile {
     let mut lockfile = lockfile_mutex.lock();

cli/npm/resolution.rs

@@ -9,6 +9,7 @@ use async_trait::async_trait;
 use deno_ast::ModuleSpecifier;
 use deno_core::error::AnyError;
 use deno_core::url::Url;
+use deno_npm::resolution::PackageNotFoundFromReferrerError;
 use deno_npm::NpmPackageCacheFolderId;
 use deno_npm::NpmPackageId;
 use deno_npm::NpmResolutionPackage;
@@ -58,7 +59,7 @@ impl GlobalNpmPackageResolver {
     &self,
     package_name: &str,
     referrer_pkg_id: &NpmPackageCacheFolderId,
-  ) -> Result<NpmResolutionPackage, AnyError> {
+  ) -> Result<NpmResolutionPackage, Box<PackageNotFoundFromReferrerError>> {
     let types_name = types_package_name(package_name);
     self
       .resolution

cli/npm/resolvers/global.rs

@@ -15,6 +15,7 @@ use deno_core::parking_lot::Mutex;
 use deno_core::serde_json;
 use deno_core::url::Url;
 use deno_npm::resolution::NpmResolutionSnapshot;
+use deno_npm::resolution::PackageReqNotFoundError;
 use deno_npm::NpmPackageId;
 use deno_runtime::deno_node::NodePermissions;
 use deno_runtime::deno_node::NodeResolutionMode;
@@ -82,14 +83,14 @@ impl NpmPackageResolver {
   pub fn resolve_pkg_id_from_pkg_req(
     &self,
     req: &NpmPackageReq,
-  ) -> Result<NpmPackageId, AnyError> {
+  ) -> Result<NpmPackageId, PackageReqNotFoundError> {
     self.resolution.resolve_pkg_id_from_pkg_req(req)
   }
 
   pub fn pkg_req_ref_to_nv_ref(
     &self,
     req_ref: NpmPackageReqReference,
-  ) -> Result<NpmPackageNvReference, AnyError> {
+  ) -> Result<NpmPackageNvReference, PackageReqNotFoundError> {
     self.resolution.pkg_req_ref_to_nv_ref(req_ref)
   }
 
@@ -225,10 +226,8 @@ impl NpmPackageResolver {
     &self,
   ) -> Result<(), AnyError> {
     // add and ensure this isn't added to the lockfile
-    self
-      .resolution
-      .add_package_reqs(vec![NpmPackageReq::from_str("@types/node").unwrap()])
-      .await?;
+    let package_reqs = vec![NpmPackageReq::from_str("@types/node").unwrap()];
+    self.resolution.add_package_reqs(package_reqs).await?;
     self.fs_resolver.cache_packages().await?;
 
     Ok(())

cli/npm/resolvers/mod.rs

@@ -237,9 +237,15 @@ impl NpmResolver for CliGraphResolver {
     if self.no_npm {
       bail!("npm specifiers were requested; but --no-npm is specified")
     }
-    self
+    match self
       .npm_resolution
       .resolve_package_req_as_pending(package_req)
+    {
+      Ok(nv) => Ok(nv),
+      Err(err) => {
+        bail!("{:#} Try retrieving the latest npm package information by running with --reload", err)
+      }
+    }
   }
 }