feat: add proxy-config and activate https/http proxy for pixi commands (#3320)

Author: gzm55

Cargo.lock

@@ -98,6 +98,7 @@ impl Tool {
     }
 
     /// Construct a new command that enables invocation of the tool.
+    /// TODO: whether to inject proxy config
     pub fn command(&self) -> std::process::Command {
         match self {
             Tool::Isolated(tool) => {

crates/pixi_build_frontend/src/tool/mod.rs

@@ -21,6 +21,7 @@ rattler = { workspace = true }
 rattler_conda_types = { workspace = true }
 rattler_networking = { workspace = true, features = ["s3"] }
 rattler_repodata_gateway = { workspace = true, features = ["gateway"] }
+reqwest = { workspace = true }
 serde = { workspace = true }
 serde_ignored = { workspace = true }
 serde_json = { workspace = true }

crates/pixi_config/Cargo.toml

@@ -3,6 +3,7 @@ use std::{
     path::{Path, PathBuf},
     process::{Command, Stdio},
     str::FromStr,
+    sync::LazyLock,
 };
 
 use clap::{ArgAction, Parser};
@@ -15,6 +16,7 @@ use rattler_conda_types::{
 };
 use rattler_networking::s3_middleware;
 use rattler_repodata_gateway::{Gateway, GatewayBuilder, SourceConfig};
+use reqwest::{NoProxy, Proxy};
 use serde::{de::IntoDeserializer, Deserialize, Serialize};
 use url::Url;
 
@@ -55,6 +57,25 @@ pub fn get_default_author() -> Option<(String, String)> {
     Some((name?, email.unwrap_or_else(|| "".into())))
 }
 
+// detect proxy env vars like curl: https://curl.se/docs/manpage.html
+static ENV_HTTP_PROXY: LazyLock<Option<String>> = LazyLock::new(|| {
+    ["http_proxy", "all_proxy", "ALL_PROXY"]
+        .iter()
+        .find_map(|&k| std::env::var(k).ok().filter(|v| !v.is_empty()))
+});
+static ENV_HTTPS_PROXY: LazyLock<Option<String>> = LazyLock::new(|| {
+    ["https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"]
+        .iter()
+        .find_map(|&k| std::env::var(k).ok().filter(|v| !v.is_empty()))
+});
+static ENV_NO_PROXY: LazyLock<Option<String>> = LazyLock::new(|| {
+    ["no_proxy", "NO_PROXY"]
+        .iter()
+        .find_map(|&k| std::env::var(k).ok().filter(|v| !v.is_empty()))
+});
+static USE_PROXY_FROM_ENV: LazyLock<bool> =
+    LazyLock::new(|| (*ENV_HTTPS_PROXY).is_some() || (*ENV_HTTP_PROXY).is_some());
+
 /// Get pixi home directory, default to `$HOME/.pixi`
 ///
 /// It may be overridden by the `PIXI_HOME` environment variable.
@@ -651,6 +672,11 @@ pub struct Config {
     #[serde(skip_serializing_if = "ConcurrencyConfig::is_default")]
     pub concurrency: ConcurrencyConfig,
 
+    /// Https/Http proxy configuration for pixi
+    #[serde(default)]
+    #[serde(skip_serializing_if = "ProxyConfig::is_default")]
+    pub proxy_config: ProxyConfig,
+
     //////////////////////
     // Deprecated fields //
     //////////////////////
@@ -681,6 +707,7 @@ impl Default for Config {
             shell: ShellConfig::default(),
             experimental: ExperimentalConfig::default(),
             concurrency: ConcurrencyConfig::default(),
+            proxy_config: ProxyConfig::default(),
 
             // Deprecated fields
             change_ps1: None,
@@ -783,6 +810,40 @@ impl ShellConfig {
     }
 }
 
+#[derive(Clone, Debug, Deserialize, Serialize, Default, PartialEq, Eq)]
+#[serde(rename_all = "kebab-case")]
+pub struct ProxyConfig {
+    /// https proxy.
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub https: Option<Url>,
+    /// http proxy.
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub http: Option<Url>,
+    /// A list of no proxy pattern
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub non_proxy_hosts: Vec<String>,
+}
+
+impl ProxyConfig {
+    pub fn is_default(&self) -> bool {
+        self.https.is_none() && self.https.is_none() && self.non_proxy_hosts.is_empty()
+    }
+    pub fn merge(&self, other: Self) -> Self {
+        Self {
+            https: other.https.as_ref().or(self.https.as_ref()).cloned(),
+            http: other.http.as_ref().or(self.http.as_ref()).cloned(),
+            non_proxy_hosts: if other.is_default() {
+                self.non_proxy_hosts.clone()
+            } else {
+                other.non_proxy_hosts.clone()
+            },
+        }
+    }
+}
+
 #[derive(thiserror::Error, Debug)]
 pub enum ConfigError {
     #[error("no file was found at {0}")]
@@ -909,6 +970,23 @@ impl Config {
             .validate()
             .map_err(|e| ConfigError::ValidationError(e, path.to_path_buf()))?;
 
+        // check proxy config
+        if config.proxy_config.https.is_none() && config.proxy_config.http.is_none() {
+            if !config.proxy_config.non_proxy_hosts.is_empty() {
+                tracing::warn!("proxy-config.non-proxy-hosts is not empty but will be ignored, as no https or http config is set.")
+            }
+        } else if *USE_PROXY_FROM_ENV {
+            let config_no_proxy = Some(config.proxy_config.non_proxy_hosts.iter().join(","))
+                .filter(|v| !v.is_empty());
+            if (*ENV_HTTPS_PROXY).as_deref() != config.proxy_config.https.as_ref().map(Url::as_str)
+                || (*ENV_HTTP_PROXY).as_deref()
+                    != config.proxy_config.http.as_ref().map(Url::as_str)
+                || *ENV_NO_PROXY != config_no_proxy
+            {
+                tracing::info!("proxy configs are overridden by proxy environment vars.")
+            }
+        }
+
         Ok(config)
     }
 
@@ -1047,6 +1125,10 @@ impl Config {
             "s3-options.<bucket>.region",
             "s3-options.<bucket>.force-path-style",
             "experimental.use-environment-activation-cache",
+            "proxy-config",
+            "proxy-config.https",
+            "proxy-config.http",
+            "proxy-config.non-proxy-hosts",
         ]
     }
 
@@ -1087,6 +1169,8 @@ impl Config {
             // Make other take precedence over self to allow for setting the value through the CLI
             concurrency: self.concurrency.merge(other.concurrency),
 
+            proxy_config: self.proxy_config.merge(other.proxy_config),
+
             // Deprecated fields that we can ignore as we handle them inside `shell.` field
             change_ps1: None,
             force_activate: None,
@@ -1162,6 +1246,43 @@ impl Config {
         self.concurrency.downloads
     }
 
+    pub fn get_proxies(&self) -> reqwest::Result<Vec<Proxy>> {
+        if (self.proxy_config.https.is_none() && self.proxy_config.http.is_none())
+            || *USE_PROXY_FROM_ENV
+        {
+            return Ok(vec![]);
+        }
+
+        let config_no_proxy =
+            Some(self.proxy_config.non_proxy_hosts.iter().join(",")).filter(|v| !v.is_empty());
+
+        let mut result: Vec<Proxy> = Vec::new();
+        let config_no_proxy: Option<NoProxy> =
+            config_no_proxy.as_deref().and_then(NoProxy::from_string);
+
+        if self.proxy_config.https == self.proxy_config.http {
+            result.push(
+                Proxy::all(
+                    self.proxy_config
+                        .https
+                        .as_ref()
+                        .expect("must be some")
+                        .as_str(),
+                )?
+                .no_proxy(config_no_proxy),
+            );
+        } else {
+            if let Some(url) = &self.proxy_config.http {
+                result.push(Proxy::http(url.as_str())?.no_proxy(config_no_proxy.clone()));
+            }
+            if let Some(url) = &self.proxy_config.https {
+                result.push(Proxy::https(url.as_str())?.no_proxy(config_no_proxy));
+            }
+        }
+
+        Ok(result)
+    }
+
     /// Modify this config with the given key and value
     ///
     /// # Note
@@ -1426,6 +1547,42 @@ impl Config {
                     _ => return Err(err),
                 }
             }
+            key if key.starts_with("proxy-config") => {
+                if key == "proxy-config" {
+                    if let Some(value) = value {
+                        self.proxy_config = serde_json::de::from_str(&value).into_diagnostic()?;
+                    } else {
+                        self.proxy_config = ProxyConfig::default();
+                    }
+                    return Ok(());
+                } else if !key.starts_with("proxy-config.") {
+                    return Err(err);
+                }
+
+                let subkey = key.strip_prefix("proxy-config.").unwrap();
+                match subkey {
+                    "https" => {
+                        self.proxy_config.https = value
+                            .map(|v| Url::parse(&v))
+                            .transpose()
+                            .into_diagnostic()?;
+                    }
+                    "http" => {
+                        self.proxy_config.http = value
+                            .map(|v| Url::parse(&v))
+                            .transpose()
+                            .into_diagnostic()?;
+                    }
+                    "non-proxy-hosts" => {
+                        self.proxy_config.non_proxy_hosts = value
+                            .map(|v| serde_json::de::from_str(&v))
+                            .transpose()
+                            .into_diagnostic()?
+                            .unwrap_or_default();
+                    }
+                    _ => return Err(err),
+                }
+            }
             _ => return Err(err),
         }
 
@@ -1728,6 +1885,7 @@ UNUSED = "unused"
                     RepodataChannelConfig::default(),
                 )]),
             },
+            proxy_config: ProxyConfig::default(),
             // Deprecated keys
             change_ps1: None,
             force_activate: None,
@@ -2148,4 +2306,25 @@ UNUSED = "unused"
         assert_eq!(anaconda_config.disable_zstd, Some(false));
         assert_eq!(anaconda_config.disable_sharded, None);
     }
+
+    #[test]
+    fn test_proxy_config_parse() {
+        let toml = r#"
+            [proxy-config]
+            https = "http://proxy-for-https"
+            http = "http://proxy-for-http"
+            non-proxy-hosts = [ "a.com" ]
+        "#;
+        let (config, _) = Config::from_toml(toml, None).unwrap();
+        assert_eq!(
+            config.proxy_config.https,
+            Some(Url::parse("http://proxy-for-https").unwrap())
+        );
+        assert_eq!(
+            config.proxy_config.http,
+            Some(Url::parse("http://proxy-for-http").unwrap())
+        );
+        assert_eq!(config.proxy_config.non_proxy_hosts.len(), 1);
+        assert_eq!(config.proxy_config.non_proxy_hosts[0], "a.com");
+    }
 }

crates/pixi_config/src/lib.rs

@@ -133,6 +133,11 @@ Config {
         solves: 1,
         downloads: 50,
     },
+    proxy_config: ProxyConfig {
+        https: None,
+        http: None,
+        non_proxy_hosts: [],
+    },
     change_ps1: None,
     force_activate: None,
 }

crates/pixi_config/src/snapshots/pixi_config__tests__config_merge_multiple.snap

@@ -117,14 +117,18 @@ pub fn build_reqwest_clients(
     }
 
     let timeout = 5 * 60;
-    let client = Client::builder()
+    let mut builder = Client::builder()
         .pool_max_idle_per_host(20)
         .user_agent(app_user_agent)
         .danger_accept_invalid_certs(config.tls_no_verify())
         .read_timeout(Duration::from_secs(timeout))
-        .use_rustls_tls()
-        .build()
-        .expect("failed to create reqwest Client");
+        .use_rustls_tls();
+
+    for p in config.get_proxies().into_diagnostic()? {
+        builder = builder.proxy(p);
+    }
+
+    let client = builder.build().expect("failed to create reqwest Client");
 
     let mut client_builder = ClientBuilder::new(client.clone());
 

crates/pixi_utils/src/reqwest.rs

@@ -4,6 +4,7 @@ use flate2::read::GzDecoder;
 use tar::Archive;
 
 use miette::IntoDiagnostic;
+use pixi_config::Config;
 use pixi_consts::consts;
 use reqwest::redirect::Policy;
 use reqwest::Client;
@@ -59,10 +60,11 @@ async fn latest_version() -> miette::Result<Version> {
     let url = format!("{}/latest", consts::RELEASES_URL);
 
     // Create a client with a redirect policy
-    let no_redirect_client = Client::builder()
-        .redirect(Policy::none()) // Prevent automatic redirects
-        .build()
-        .into_diagnostic()?;
+    let mut no_redirect_client_builder = Client::builder().redirect(Policy::none()); // Prevent automatic redirects
+    for p in Config::load_global().get_proxies().into_diagnostic()? {
+        no_redirect_client_builder = no_redirect_client_builder.proxy(p);
+    }
+    let no_redirect_client = no_redirect_client_builder.build().into_diagnostic()?;
 
     let version: String = match no_redirect_client
         .head(&url)
@@ -170,6 +172,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
     // Create a temp file to download the archive
     let mut archived_tempfile = tempfile::NamedTempFile::new().into_diagnostic()?;
 
+    // TODO proxy inject in https://github.com/prefix-dev/pixi/pull/3346
     let client = Client::new();
     let mut res = client
         .get(&download_url)

pixi_docs/Cargo.lock

@@ -15,6 +15,7 @@ use thiserror::Error;
 use tokio::fs::File;
 use tokio_util::io::ReaderStream;
 
+use pixi_config::Config;
 use pixi_progress;
 
 #[allow(rustdoc::bare_urls)]
@@ -53,7 +54,12 @@ pub async fn execute(args: Args) -> miette::Result<()> {
         HumanBytes(filesize)
     );
 
-    let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new())
+    let mut raw_builder = reqwest::Client::builder();
+    for p in Config::load_global().get_proxies().into_diagnostic()? {
+        raw_builder = raw_builder.proxy(p);
+    }
+
+    let client = reqwest_middleware::ClientBuilder::new(raw_builder.build().into_diagnostic()?)
         .with_arc(Arc::new(
             AuthenticationMiddleware::from_env_and_defaults().into_diagnostic()?,
         ))