Make uv’s first-index strategy more secure by default by failing early on authentication failure (#12805)

uv’s default index strategy was designed with dependency confusion
attacks in mind. [According to the
docs](https://docs.astral.sh/uv/configuration/indexes/#searching-across-multiple-indexes),
“if a package exists on an internal index, it should always be installed
from the internal index, and never from PyPI”. Unfortunately, this is
not true in the case where authentication fails on that internal index.
In that case, uv will simply try the next index (even on the
`first-index` strategy). This means that uv is not secure by default in
this common scenario.

This PR causes uv to stop searching for a package if it encounters an
authentication failure at an index. It is possible to opt out of this
behavior for an index with a new `pyproject.toml` option
`ignore-error-codes`. For example:

```
[[tool.uv.index]]
name = "my-index"
url = "<index-url>"
ignore-error-codes = [401, 403]
```

This will also enable users to handle idiosyncratic registries in a more
fine-grained way. For example, PyTorch registries return a 403 when a
package is not found. In this PR, we special-case PyTorch registries to
ignore 403s, but users can use `ignore-error-codes` to handle similar
behaviors if they encounter them on internal registries.

Depends on #12651

Closes #9429
Closes #12362

Author: jtfmumm

Cargo.lock

@@ -1,3 +1,4 @@
+use std::fmt::Display;
 use std::fmt::Formatter;
 use std::hash::BuildHasherDefault;
 use std::sync::Arc;
@@ -14,11 +15,28 @@ use crate::Realm;
 
 type FxOnceMap<K, V> = OnceMap<K, V, BuildHasherDefault<FxHasher>>;
 
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub(crate) enum FetchUrl {
+    /// A full index URL
+    Index(Url),
+    /// A realm URL
+    Realm(Realm),
+}
+
+impl Display for FetchUrl {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        match self {
+            Self::Index(index) => Display::fmt(index, f),
+            Self::Realm(realm) => Display::fmt(realm, f),
+        }
+    }
+}
+
 pub struct CredentialsCache {
     /// A cache per realm and username
     realms: RwLock<FxHashMap<(Realm, Username), Arc<Credentials>>>,
     /// A cache tracking the result of realm or index URL fetches from external services
-    pub(crate) fetches: FxOnceMap<(String, Username), Option<Arc<Credentials>>>,
+    pub(crate) fetches: FxOnceMap<(FetchUrl, Username), Option<Arc<Credentials>>>,
     /// A cache per URL, uses a trie for efficient prefix queries.
     urls: RwLock<UrlTrie>,
 }

crates/uv-auth/src/cache.rs

@@ -60,6 +60,23 @@ pub struct Index {
     pub auth_policy: AuthPolicy,
 }
 
+impl Index {
+    pub fn is_prefix_for(&self, url: &Url) -> bool {
+        if self.root_url.scheme() != url.scheme()
+            || self.root_url.host_str() != url.host_str()
+            || self.root_url.port_or_known_default() != url.port_or_known_default()
+        {
+            return false;
+        }
+
+        url.path().starts_with(self.root_url.path())
+    }
+}
+
+// TODO(john): Multiple methods in this struct need to iterate over
+// all the indexes in the set. There are probably not many URLs to
+// iterate through, but we could use a trie instead of a HashSet here
+// for more efficient search.
 #[derive(Debug, Default, Clone, Eq, PartialEq)]
 pub struct Indexes(FxHashSet<Index>);
 
@@ -79,36 +96,17 @@ impl Indexes {
 
     /// Get the index URL prefix for a URL if one exists.
     pub fn index_url_for(&self, url: &Url) -> Option<&Url> {
-        // TODO(john): There are probably not many URLs to iterate through,
-        // but we could use a trie instead of a HashSet here for more
-        // efficient search.
-        self.0
-            .iter()
-            .find(|index| is_url_prefix(&index.root_url, url))
-            .map(|index| &index.url)
+        self.find_prefix_index(url).map(|index| &index.url)
     }
 
     /// Get the [`AuthPolicy`] for a URL.
-    pub fn policy_for(&self, url: &Url) -> AuthPolicy {
-        // TODO(john): There are probably not many URLs to iterate through,
-        // but we could use a trie instead of a HashMap here for more
-        // efficient search.
-        for index in &self.0 {
-            if is_url_prefix(&index.root_url, url) {
-                return index.auth_policy;
-            }
-        }
-        AuthPolicy::Auto
+    pub fn auth_policy_for(&self, url: &Url) -> AuthPolicy {
+        self.find_prefix_index(url)
+            .map(|index| index.auth_policy)
+            .unwrap_or(AuthPolicy::Auto)
     }
-}
 
-fn is_url_prefix(base: &Url, url: &Url) -> bool {
-    if base.scheme() != url.scheme()
-        || base.host_str() != url.host_str()
-        || base.port_or_known_default() != url.port_or_known_default()
-    {
-        return false;
+    fn find_prefix_index(&self, url: &Url) -> Option<&Index> {
+        self.0.iter().find(|&index| index.is_prefix_for(url))
     }
-
-    url.path().starts_with(base.path())
 }

crates/uv-auth/src/index.rs

@@ -4,6 +4,7 @@ use http::{Extensions, StatusCode};
 use url::Url;
 
 use crate::{
+    cache::FetchUrl,
     credentials::{Credentials, Username},
     index::{AuthPolicy, Indexes},
     realm::Realm,
@@ -182,7 +183,7 @@ impl Middleware for AuthMiddleware {
         // to the headers so for display purposes we restore some information
         let url = tracing_url(&request, request_credentials.as_ref());
         let maybe_index_url = self.indexes.index_url_for(request.url());
-        let auth_policy = self.indexes.policy_for(request.url());
+        let auth_policy = self.indexes.auth_policy_for(request.url());
         trace!("Handling request for {url} with authentication policy {auth_policy}");
 
         let credentials: Option<Arc<Credentials>> = if matches!(auth_policy, AuthPolicy::Never) {
@@ -384,7 +385,7 @@ impl AuthMiddleware {
         extensions: &mut Extensions,
         next: Next<'_>,
         url: &str,
-        maybe_index_url: Option<&Url>,
+        index_url: Option<&Url>,
         auth_policy: AuthPolicy,
     ) -> reqwest_middleware::Result<Response> {
         let credentials = Arc::new(credentials);
@@ -402,7 +403,7 @@ impl AuthMiddleware {
         // There's just a username, try to find a password.
         // If we have an index URL, check the cache for that URL. Otherwise,
         // check for the realm.
-        let maybe_cached_credentials = if let Some(index_url) = maybe_index_url {
+        let maybe_cached_credentials = if let Some(index_url) = index_url {
             self.cache()
                 .get_url(index_url, credentials.as_username().as_ref())
         } else {
@@ -426,17 +427,12 @@ impl AuthMiddleware {
             // Do not insert already-cached credentials
             None
         } else if let Some(credentials) = self
-            .fetch_credentials(
-                Some(&credentials),
-                request.url(),
-                maybe_index_url,
-                auth_policy,
-            )
+            .fetch_credentials(Some(&credentials), request.url(), index_url, auth_policy)
             .await
         {
             request = credentials.authenticate(request);
             Some(credentials)
-        } else if maybe_index_url.is_some() {
+        } else if index_url.is_some() {
             // If this is a known index, we fall back to checking for the realm.
             self.cache()
                 .get_realm(Realm::from(request.url()), credentials.to_username())
@@ -468,9 +464,9 @@ impl AuthMiddleware {
         // Fetches can be expensive, so we will only run them _once_ per realm or index URL and username combination
         // All other requests for the same realm or index URL will wait until the first one completes
         let key = if let Some(index_url) = maybe_index_url {
-            (index_url.to_string(), username)
+            (FetchUrl::Index(index_url.clone()), username)
         } else {
-            (Realm::from(url).to_string(), username)
+            (FetchUrl::Realm(Realm::from(url)), username)
         };
         if !self.cache().fetches.register(key.clone()) {
             let credentials = self
@@ -520,7 +516,7 @@ impl AuthMiddleware {
                         debug!("Checking keyring for credentials for index URL {}@{}", username, index_url);
                         keyring.fetch(index_url, Some(username)).await
                     } else {
-                        debug!("Checking keyring for credentials for full URL {}@{}", username, *url);
+                        debug!("Checking keyring for credentials for full URL {}@{}", username, url);
                         keyring.fetch(url, Some(username)).await
                     }
                 } else if matches!(auth_policy, AuthPolicy::Always) {
@@ -530,10 +526,7 @@ impl AuthMiddleware {
                         );
                         keyring.fetch(index_url, None).await
                     } else {
-                        debug!(
-                            "Checking keyring for credentials for full URL {url} without username due to `authenticate = always`"
-                        );
-                        keyring.fetch(url, None).await
+                        None
                     }
                 } else {
                     debug!("Skipping keyring fetch for {url} without username; use `authenticate = always` to force");

crates/uv-auth/src/middleware.rs

@@ -7,13 +7,13 @@ use std::time::Duration;
 
 use async_http_range_reader::AsyncHttpRangeReader;
 use futures::{FutureExt, StreamExt, TryStreamExt};
-use http::HeaderMap;
+use http::{HeaderMap, StatusCode};
 use itertools::Either;
-use reqwest::{Proxy, Response, StatusCode};
+use reqwest::{Proxy, Response};
 use reqwest_middleware::ClientWithMiddleware;
 use rustc_hash::FxHashMap;
 use tokio::sync::{Mutex, Semaphore};
-use tracing::{info_span, instrument, trace, warn, Instrument};
+use tracing::{debug, info_span, instrument, trace, warn, Instrument};
 use url::Url;
 
 use uv_auth::Indexes;
@@ -23,7 +23,7 @@ use uv_configuration::{IndexStrategy, TrustedHost};
 use uv_distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
 use uv_distribution_types::{
     BuiltDist, File, FileLocation, IndexCapabilities, IndexFormat, IndexLocations,
-    IndexMetadataRef, IndexUrl, IndexUrls, Name,
+    IndexMetadataRef, IndexStatusCodeDecision, IndexStatusCodeStrategy, IndexUrl, IndexUrls, Name,
 };
 use uv_metadata::{read_metadata_async_seek, read_metadata_async_stream};
 use uv_normalize::PackageName;
@@ -331,12 +331,29 @@ impl RegistryClient {
                     let _permit = download_concurrency.acquire().await;
                     match index.format {
                         IndexFormat::Simple => {
-                            if let Some(metadata) = self
-                                .simple_single_index(package_name, index.url, capabilities)
+                            let status_code_strategy =
+                                self.index_urls.status_code_strategy_for(index.url);
+                            match self
+                                .simple_single_index(
+                                    package_name,
+                                    index.url,
+                                    capabilities,
+                                    &status_code_strategy,
+                                )
                                 .await?
                             {
-                                results.push((index.url, MetadataFormat::Simple(metadata)));
-                                break;
+                                SimpleMetadataSearchOutcome::Found(metadata) => {
+                                    results.push((index.url, MetadataFormat::Simple(metadata)));
+                                    break;
+                                }
+                                // Package not found, so we will continue on to the next index (if there is one)
+                                SimpleMetadataSearchOutcome::NotFound => {}
+                                // The search failed because of an HTTP status code that we don't ignore for
+                                // this index. We end our search here.
+                                SimpleMetadataSearchOutcome::StatusCodeFailure(status_code) => {
+                                    debug!("Indexes search failed because of status code failure: {status_code}");
+                                    break;
+                                }
                             }
                         }
                         IndexFormat::Flat => {
@@ -357,9 +374,21 @@ impl RegistryClient {
                         let _permit = download_concurrency.acquire().await;
                         match index.format {
                             IndexFormat::Simple => {
-                                let metadata = self
-                                    .simple_single_index(package_name, index.url, capabilities)
-                                    .await?;
+                                // For unsafe matches, ignore authentication failures.
+                                let status_code_strategy =
+                                    IndexStatusCodeStrategy::ignore_authentication_error_codes();
+                                let metadata = match self
+                                    .simple_single_index(
+                                        package_name,
+                                        index.url,
+                                        capabilities,
+                                        &status_code_strategy,
+                                    )
+                                    .await?
+                                {
+                                    SimpleMetadataSearchOutcome::Found(metadata) => Some(metadata),
+                                    _ => None,
+                                };
                                 Ok((index.url, metadata.map(MetadataFormat::Simple)))
                             }
                             IndexFormat::Flat => {
@@ -438,14 +467,13 @@ impl RegistryClient {
     ///
     /// The index can either be a PEP 503-compatible remote repository, or a local directory laid
     /// out in the same format.
-    ///
-    /// Returns `Ok(None)` if the package is not found in the index.
     async fn simple_single_index(
         &self,
         package_name: &PackageName,
         index: &IndexUrl,
         capabilities: &IndexCapabilities,
-    ) -> Result<Option<OwnedArchive<SimpleMetadata>>, Error> {
+        status_code_strategy: &IndexStatusCodeStrategy,
+    ) -> Result<SimpleMetadataSearchOutcome, Error> {
         // Format the URL for PyPI.
         let mut url = index.url().clone();
         url.path_segments_mut()
@@ -487,27 +515,31 @@ impl RegistryClient {
         };
 
         match result {
-            Ok(metadata) => Ok(Some(metadata)),
+            Ok(metadata) => Ok(SimpleMetadataSearchOutcome::Found(metadata)),
             Err(err) => match err.into_kind() {
                 // The package could not be found in the remote index.
-                ErrorKind::WrappedReqwestError(url, err) => match err.status() {
-                    Some(StatusCode::NOT_FOUND) => Ok(None),
-                    Some(StatusCode::UNAUTHORIZED) => {
-                        capabilities.set_unauthorized(index.clone());
-                        Ok(None)
-                    }
-                    Some(StatusCode::FORBIDDEN) => {
-                        capabilities.set_forbidden(index.clone());
-                        Ok(None)
+                ErrorKind::WrappedReqwestError(url, err) => {
+                    let Some(status_code) = err.status() else {
+                        return Err(ErrorKind::WrappedReqwestError(url, err).into());
+                    };
+                    let decision =
+                        status_code_strategy.handle_status_code(status_code, index, capabilities);
+                    if let IndexStatusCodeDecision::Fail(status_code) = decision {
+                        if !matches!(
+                            status_code,
+                            StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
+                        ) {
+                            return Err(ErrorKind::WrappedReqwestError(url, err).into());
+                        }
                     }
-                    _ => Err(ErrorKind::WrappedReqwestError(url, err).into()),
-                },
+                    Ok(SimpleMetadataSearchOutcome::from(decision))
+                }
 
                 // The package is unavailable due to a lack of connectivity.
-                ErrorKind::Offline(_) => Ok(None),
+                ErrorKind::Offline(_) => Ok(SimpleMetadataSearchOutcome::NotFound),
 
                 // The package could not be found in the local index.
-                ErrorKind::FileNotFound(_) => Ok(None),
+                ErrorKind::FileNotFound(_) => Ok(SimpleMetadataSearchOutcome::NotFound),
 
                 err => Err(err.into()),
             },
@@ -960,6 +992,26 @@ impl RegistryClient {
     }
 }
 
+#[derive(Debug)]
+pub(crate) enum SimpleMetadataSearchOutcome {
+    /// Simple metadata was found
+    Found(OwnedArchive<SimpleMetadata>),
+    /// Simple metadata was not found
+    NotFound,
+    /// A status code failure was encountered when searching for
+    /// simple metadata and our strategy did not ignore it
+    StatusCodeFailure(StatusCode),
+}
+
+impl From<IndexStatusCodeDecision> for SimpleMetadataSearchOutcome {
+    fn from(item: IndexStatusCodeDecision) -> Self {
+        match item {
+            IndexStatusCodeDecision::Ignore => Self::NotFound,
+            IndexStatusCodeDecision::Fail(status_code) => Self::StatusCodeFailure(status_code),
+        }
+    }
+}
+
 /// A map from [`IndexUrl`] to [`FlatIndexEntry`] entries found at the given URL, indexed by
 /// [`PackageName`].
 #[derive(Default, Debug, Clone)]

crates/uv-client/src/registry_client.rs

@@ -32,6 +32,7 @@ uv-small-str = { workspace = true }
 arcstr = { workspace = true }
 bitflags = { workspace = true }
 fs-err = { workspace = true }
+http = { workspace = true }
 itertools = { workspace = true }
 jiff = { workspace = true }
 owo-colors = { workspace = true }
@@ -47,7 +48,5 @@ tracing = { workspace = true }
 url = { workspace = true }
 version-ranges = { workspace = true }
 
-
 [dev-dependencies]
 toml = { workspace = true }
-