refactor: Improve AuthenticationStorage, update rattler (#2909)

Author: pavelzw

Cargo.lock

@@ -113,20 +113,20 @@ which = "7.0.1"
 
 # Rattler crates
 file_url = "0.2.2"
-rattler = { version = "0.28.11", default-features = false }
-rattler_cache = { version = "0.3.3", default-features = false }
-rattler_conda_types = { version = "0.29.9", default-features = false, features = [
+rattler = { version = "0.29.0", default-features = false }
+rattler_cache = { version = "0.3.5", default-features = false }
+rattler_conda_types = { version = "0.30.0", default-features = false, features = [
   "rayon",
 ] }
 rattler_digest = { version = "1.0.5", default-features = false }
-rattler_lock = { version = "0.22.38", default-features = false }
-rattler_networking = { version = "0.21.10", default-features = false, features = [
+rattler_lock = { version = "0.22.40", default-features = false }
+rattler_networking = { version = "0.22.0", default-features = false, features = [
   "google-cloud-auth",
 ] }
-rattler_repodata_gateway = { version = "0.21.31", default-features = false }
-rattler_shell = { version = "0.22.14", default-features = false }
-rattler_solve = { version = "1.3.3", default-features = false }
-rattler_virtual_packages = { version = "1.1.17", default-features = false }
+rattler_repodata_gateway = { version = "0.21.33", default-features = false }
+rattler_shell = { version = "0.22.16", default-features = false }
+rattler_solve = { version = "1.3.5", default-features = false }
+rattler_virtual_packages = { version = "2.0.0", default-features = false }
 
 
 # Bumping this to a higher version breaks the Windows path handling.

Cargo.toml

@@ -38,7 +38,7 @@ impl Default for EnabledProtocols {
 #[derive(Debug)]
 pub(crate) enum ProtocolBuilder {
     /// A pixi project.
-    Pixi(pixi_protocol::ProtocolBuilder),
+    Pixi(Box<pixi_protocol::ProtocolBuilder>),
 
     /// A directory containing a `meta.yaml` that can be interpreted by
     /// conda-build.
@@ -51,7 +51,7 @@ pub(crate) enum ProtocolBuilder {
 
 impl From<pixi_protocol::ProtocolBuilder> for ProtocolBuilder {
     fn from(value: pixi_protocol::ProtocolBuilder) -> Self {
-        Self::Pixi(value)
+        Self::Pixi(Box::new(value))
     }
 }
 
@@ -125,7 +125,9 @@ impl ProtocolBuilder {
     /// Sets the channel configuration used by the protocol.
     pub fn with_channel_config(self, channel_config: ChannelConfig) -> Self {
         match self {
-            Self::Pixi(protocol) => Self::Pixi(protocol.with_channel_config(channel_config)),
+            Self::Pixi(protocol) => {
+                Self::Pixi(Box::new(protocol.with_channel_config(channel_config)))
+            }
             Self::CondaBuild(protocol) => {
                 Self::CondaBuild(protocol.with_channel_config(channel_config))
             }
@@ -139,7 +141,7 @@ impl ProtocolBuilder {
         if let Some(backend_override) = backend_override {
             match self {
                 Self::Pixi(protocol) => {
-                    Self::Pixi(protocol.with_backend_override(backend_override))
+                    Self::Pixi(Box::new(protocol.with_backend_override(backend_override)))
                 }
                 Self::CondaBuild(protocol) => {
                     Self::CondaBuild(protocol.with_backend_override(backend_override))
@@ -156,7 +158,9 @@ impl ProtocolBuilder {
     /// Sets the cache directory to use for any caching.
     pub fn with_opt_cache_dir(self, cache_directory: Option<PathBuf>) -> Self {
         match self {
-            Self::Pixi(protocol) => Self::Pixi(protocol.with_opt_cache_dir(cache_directory)),
+            Self::Pixi(protocol) => {
+                Self::Pixi(Box::new(protocol.with_opt_cache_dir(cache_directory)))
+            }
             Self::CondaBuild(protocol) => {
                 Self::CondaBuild(protocol.with_opt_cache_dir(cache_directory))
             }

crates/pixi_build_frontend/src/protocol_builder.rs

@@ -405,7 +405,12 @@ mod tests {
             .await
             .unwrap();
 
-        tool.command().arg("--version").spawn().unwrap();
+        tool.command()
+            .arg("--version")
+            .spawn()
+            .unwrap()
+            .wait()
+            .unwrap();
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]

crates/pixi_build_frontend/src/tool/cache.rs

@@ -301,7 +301,7 @@ impl ToolInstaller for ToolContext {
                 self.cache_dir
                     .join(pixi_consts::consts::CONDA_PACKAGE_CACHE_DIR),
             ))
-            .install(&cached_dir, solved_records)
+            .install(&cached_dir, solved_records.records)
             .await
             .into_diagnostic()?;
 

crates/pixi_build_frontend/src/tool/installer.rs

@@ -59,7 +59,7 @@ fn to_pixi_spec_v1(
         }
         itertools::Either::Right(binary) => {
             let nameless = binary.try_into_nameless_match_spec(channel_config)?;
-            pbt::PackageSpecV1::Binary(pbt::BinaryPackageSpecV1 {
+            pbt::PackageSpecV1::Binary(Box::new(pbt::BinaryPackageSpecV1 {
                 version: nameless.version,
                 build: nameless.build,
                 build_number: nameless.build_number,
@@ -68,7 +68,7 @@ fn to_pixi_spec_v1(
                 subdir: nameless.subdir,
                 md5: nameless.md5.map(Into::into),
                 sha256: nameless.sha256.map(Into::into),
-            })
+            }))
         }
     };
     Ok(pbt_spec)

crates/pixi_build_type_conversions/src/project_model.rs

@@ -152,7 +152,7 @@ pub struct TargetV1 {
 #[serde(rename_all = "camelCase")]
 pub enum PackageSpecV1 {
     /// This is a binary dependency
-    Binary(BinaryPackageSpecV1),
+    Binary(Box<BinaryPackageSpecV1>),
     /// This is a dependency on a source package
     Source(SourcePackageSpecV1),
 }

crates/pixi_build_types/src/project_model.rs

@@ -98,7 +98,7 @@ pub struct ConfigCli {
     tls_no_verify: bool,
 
     /// Path to the file containing the authentication token.
-    #[arg(long, env = "RATTLER_AUTH_FILE")]
+    #[arg(long)]
     auth_file: Option<PathBuf>,
 
     /// Specifies if we want to use uv keyring provider

crates/pixi_config/src/lib.rs

@@ -10,6 +10,11 @@ pub fn default_linux_version() -> Version {
     "4.18".parse().unwrap()
 }
 
+/// The default Windows version to use. This is used when no system requirements are specified.
+pub fn default_windows_version() -> Version {
+    "10.0".parse().unwrap()
+}
+
 /// Returns the default Mac OS version for the specified platform. The platform must refer to a
 /// MacOS platform.
 pub fn default_mac_os_version(platform: Platform) -> Version {

crates/pixi_default_versions/src/lib.rs

@@ -1,9 +1,11 @@
 ---
 source: crates/pixi_manifest/src/manifests/workspace.rs
+assertion_line: 298
 expression: "expect_parse_failure(&format!(\"{PROJECT_BOILERPLATE}\\n{}\", examples[0]))"
+snapshot_kind: text
 ---
-  × 'foobar' is not a known platform. Valid platforms are 'noarch', 'unknown', 'linux-32', 'linux-64', 'linux-aarch64', 'linux-armv6l', 'linux-armv7l', 'linux-ppc64le', 'linux-ppc64', 'linux-s390x',
-  │ 'linux-riscv32', 'linux-riscv64', 'osx-64', 'osx-arm64', 'win-32', 'win-64', 'win-arm64', 'emscripten-wasm32', 'wasi-wasm32', 'zos-z'
+  × 'foobar' is not a known platform. Valid platforms are 'noarch', 'unknown', 'linux-32', 'linux-64', 'linux-aarch64', 'linux-armv6l', 'linux-armv7l', 'linux-ppc64le', 'linux-ppc64', 'linux-ppc',
+  │ 'linux-s390x', 'linux-riscv32', 'linux-riscv64', 'osx-64', 'osx-arm64', 'win-32', 'win-64', 'win-arm64', 'emscripten-wasm32', 'wasi-wasm32', 'zos-z'
    ╭─[pixi.toml:8:9]
  7 │
  8 │ [target.foobar.dependencies]

crates/pixi_manifest/src/manifests/snapshots/pixi_manifest__manifests__workspace__tests__invalid_target_specific.snap

@@ -91,7 +91,7 @@ impl<'de> Deserialize<'de> for UniquePackageMap {
 
 struct PackageMap<'a>(&'a IndexMap<rattler_conda_types::PackageName, PixiSpec>);
 
-impl<'de, 'a> DeserializeSeed<'de> for PackageMap<'a> {
+impl<'de> DeserializeSeed<'de> for PackageMap<'_> {
     type Value = PixiSpanned<rattler_conda_types::PackageName>;
 
     fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>

crates/pixi_manifest/src/utils/package_map.rs

@@ -37,7 +37,6 @@ impl PixiRecord {
     }
 
     /// Converts this instance into a binary record if it is a binary record.
-
     pub fn into_binary(self) -> Option<RepoDataRecord> {
         match self {
             PixiRecord::Binary(record) => Some(record),

crates/pixi_record/src/lib.rs

@@ -69,12 +69,13 @@ impl DetailedSpec {
             md5: self.md5,
             sha256: self.sha256,
             url: None,
+            extras: Default::default(),
         })
     }
 }
 
 impl From<DetailedSpec> for BinarySpec {
     fn from(value: DetailedSpec) -> Self {
-        Self::DetailedVersion(value)
+        Self::DetailedVersion(Box::new(value))
     }
 }

crates/pixi_spec/src/detailed.rs

@@ -65,7 +65,7 @@ pub enum PixiSpec {
 
     /// The spec is represented by a detailed version spec. The package should
     /// be retrieved from a channel.
-    DetailedVersion(DetailedSpec),
+    DetailedVersion(Box<DetailedSpec>),
 
     /// The spec is represented as an archive that can be downloaded from the
     /// specified URL. The package should be retrieved from the URL and can
@@ -117,7 +117,7 @@ impl PixiSpec {
         {
             Self::Version(spec.version.unwrap_or(VersionSpec::Any))
         } else {
-            Self::DetailedVersion(DetailedSpec {
+            Self::DetailedVersion(Box::new(DetailedSpec {
                 version: spec.version,
                 build: spec.build,
                 build_number: spec.build_number,
@@ -129,7 +129,7 @@ impl PixiSpec {
                 subdir: spec.subdir,
                 md5: spec.md5,
                 sha256: spec.sha256,
-            })
+            }))
         }
     }
 
@@ -189,17 +189,15 @@ impl PixiSpec {
     pub fn into_version(self) -> Option<VersionSpec> {
         match self {
             Self::Version(v) => Some(v),
-            Self::DetailedVersion(DetailedSpec {
-                version: Some(v), ..
-            }) => Some(v),
+            Self::DetailedVersion(v) => v.version,
             _ => None,
         }
     }
 
     /// Converts this instance into a [`DetailedSpec`] if possible.
     pub fn into_detailed(self) -> Option<DetailedSpec> {
         match self {
-            Self::DetailedVersion(v) => Some(v),
+            Self::DetailedVersion(v) => Some(*v),
             Self::Version(v) => Some(DetailedSpec {
                 version: Some(v),
                 ..DetailedSpec::default()
@@ -351,7 +349,7 @@ impl From<SourceSpec> for PixiSpec {
 
 impl From<DetailedSpec> for PixiSpec {
     fn from(value: DetailedSpec) -> Self {
-        Self::DetailedVersion(value)
+        Self::DetailedVersion(Box::new(value))
     }
 }
 
@@ -407,7 +405,7 @@ pub enum BinarySpec {
 
     /// The spec is represented by a detailed version spec. The package should
     /// be retrieved from a channel.
-    DetailedVersion(DetailedSpec),
+    DetailedVersion(Box<DetailedSpec>),
 
     /// The spec is represented as an archive that can be downloaded from the
     /// specified URL. The package should be retrieved from the URL and can

crates/pixi_spec/src/lib.rs

@@ -286,7 +286,7 @@ impl TomlSpec {
                     return Err(SpecError::MissingDetailedIdentifier);
                 }
 
-                PixiSpec::DetailedVersion(DetailedSpec {
+                PixiSpec::DetailedVersion(Box::new(DetailedSpec {
                     version: self.version,
                     build: self.build,
                     build_number: self.build_number,
@@ -295,7 +295,7 @@ impl TomlSpec {
                     subdir: self.subdir,
                     md5: self.md5,
                     sha256: self.sha256,
-                })
+                }))
             }
             (_, _, _) => return Err(SpecError::MultipleIdentifiers),
         };
@@ -344,7 +344,7 @@ impl TomlSpec {
                     return Err(SpecError::MissingDetailedIdentifier);
                 }
 
-                BinarySpec::DetailedVersion(DetailedSpec {
+                BinarySpec::DetailedVersion(Box::new(DetailedSpec {
                     version: self.version,
                     build: self.build,
                     build_number: self.build_number,
@@ -353,7 +353,7 @@ impl TomlSpec {
                     subdir: self.subdir,
                     md5: self.md5,
                     sha256: self.sha256,
-                })
+                }))
             }
             (_, _, _) => return Err(SpecError::MultipleIdentifiers),
         };

crates/pixi_spec/src/toml.rs

@@ -1,4 +1,4 @@
-use std::{path::PathBuf, sync::Arc, time::Duration};
+use std::{any::Any, path::PathBuf, sync::Arc, time::Duration};
 
 use pixi_consts::consts;
 use rattler_networking::{
@@ -22,23 +22,41 @@ pub fn default_retry_policy() -> ExponentialBackoff {
     ExponentialBackoff::builder().build_with_max_retries(3)
 }
 
-fn auth_middleware(config: &Config) -> Result<AuthenticationMiddleware, FileStorageError> {
+fn auth_store(config: &Config) -> Result<AuthenticationStorage, FileStorageError> {
+    let mut store = AuthenticationStorage::from_env_and_defaults()?;
     if let Some(auth_file) = config.authentication_override_file() {
         tracing::info!("Loading authentication from file: {:?}", auth_file);
 
         if !auth_file.exists() {
             tracing::warn!("Authentication file does not exist: {:?}", auth_file);
         }
 
-        let mut store = AuthenticationStorage::new();
-        store.add_backend(Arc::from(
-            authentication_storage::backends::file::FileStorage::new(PathBuf::from(&auth_file))?,
-        ));
-
-        return Ok(AuthenticationMiddleware::new(store));
+        // this should be the first place before the keyring authentication
+        // i.e. either index 0 if RATTLER_AUTH_FILE is not set or index 1 if it is
+        let first_storage = store.backends.first().unwrap();
+        let index = if first_storage.type_id()
+            == std::any::TypeId::of::<authentication_storage::backends::file::FileStorage>()
+        {
+            1
+        } else {
+            0
+        };
+        store.backends.insert(
+            index,
+            Arc::from(
+                authentication_storage::backends::file::FileStorage::from_path(PathBuf::from(
+                    &auth_file,
+                ))?,
+            ),
+        );
     }
+    Ok(store)
+}
 
-    Ok(AuthenticationMiddleware::default())
+fn auth_middleware(config: &Config) -> Result<AuthenticationMiddleware, FileStorageError> {
+    Ok(AuthenticationMiddleware::from_auth_storage(auth_store(
+        config,
+    )?))
 }
 
 pub fn mirror_middleware(config: &Config) -> MirrorMiddleware {