Merge pull request #754 from messense/cross-repair

Add support for repairing cross compiled linux wheels

Author: messense

Cargo.lock

@@ -57,7 +57,8 @@ ignore = "0.4.18"
 dialoguer = "0.9.0"
 console = "0.15.0"
 minijinja = "0.8.2"
-lddtree = "0.1.4"
+lddtree = "0.2.0"
+cc = "1.0.72"
 
 [dev-dependencies]
 indoc = "1.0.3"

Cargo.toml

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+* Add support for repairing cross compiled linux wheels in [#754](https://github.com/PyO3/maturin/pull/754)
+
 ## [0.12.5] - 2021-12-20
 
 * Fix docs for `new` and `init` commands in `maturin --help` in [#734](https://github.com/PyO3/maturin/pull/734)

Changelog.md

@@ -256,7 +256,6 @@ pub fn auditwheel_rs(
     if !target.is_linux() || platform_tag == Some(PlatformTag::Linux) {
         return Ok((Policy::default(), false));
     }
-    let cross_compiling = target.cross_compiling();
     let arch = target.target_arch().to_string();
     let mut file = File::open(path).map_err(AuditWheelError::IoError)?;
     let mut buffer = Vec::new();
@@ -300,15 +299,10 @@ pub fn auditwheel_rs(
                 should_repair = false;
                 break;
             }
-            Err(err @ AuditWheelError::LinksForbiddenLibrariesError(..)) => {
-                // TODO: support repair for cross compiled wheels
-                if !cross_compiling {
-                    highest_policy = Some(policy.clone());
-                    should_repair = true;
-                    break;
-                } else {
-                    return Err(err);
-                }
+            Err(AuditWheelError::LinksForbiddenLibrariesError(..)) => {
+                highest_policy = Some(policy.clone());
+                should_repair = true;
+                break;
             }
             Err(AuditWheelError::VersionedSymbolTooNewError(..))
             | Err(AuditWheelError::BlackListedSymbolsError(..))
@@ -340,14 +334,9 @@ pub fn auditwheel_rs(
                 should_repair = false;
                 Ok(policy)
             }
-            Err(err @ AuditWheelError::LinksForbiddenLibrariesError(..)) => {
-                // TODO: support repair for cross compiled wheels
-                if !cross_compiling {
-                    should_repair = true;
-                    Ok(policy)
-                } else {
-                    Err(err)
-                }
+            Err(AuditWheelError::LinksForbiddenLibrariesError(..)) => {
+                should_repair = true;
+                Ok(policy)
             }
             Err(err) => Err(err),
         }

src/auditwheel/audit.rs

@@ -8,4 +8,4 @@ mod repair;
 pub use audit::*;
 pub use platform_tag::PlatformTag;
 pub use policy::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES};
-pub use repair::{get_external_libs, hash_file};
+pub use repair::{find_external_libs, hash_file};

src/auditwheel/mod.rs

@@ -5,13 +5,14 @@ use fs_err as fs;
 use lddtree::DependencyAnalyzer;
 use sha2::{Digest, Sha256};
 use std::io;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
-pub fn get_external_libs(
+pub fn find_external_libs(
     artifact: impl AsRef<Path>,
     policy: &Policy,
+    sysroot: PathBuf,
 ) -> Result<Vec<lddtree::Library>, AuditWheelError> {
-    let dep_analyzer = DependencyAnalyzer::new();
+    let dep_analyzer = DependencyAnalyzer::new(sysroot);
     let deps = dep_analyzer.analyze(artifact).unwrap();
     let mut ext_libs = Vec::new();
     for (name, lib) in deps.libraries {

src/auditwheel/repair.rs

@@ -1,5 +1,5 @@
 use crate::auditwheel::{
-    auditwheel_rs, get_external_libs, hash_file, patchelf, PlatformTag, Policy,
+    auditwheel_rs, find_external_libs, hash_file, patchelf, PlatformTag, Policy,
 };
 use crate::compile::warn_missing_py_init;
 use crate::module_writer::{
@@ -15,6 +15,7 @@ use lddtree::Library;
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
 
 /// The way the rust code is used in the wheel
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -272,7 +273,8 @@ impl BuildContext {
                 }
             })?;
         let external_libs = if should_repair && !self.editable {
-            get_external_libs(&artifact, &policy).with_context(|| {
+            let sysroot = get_sysroot_path(&self.target)?;
+            find_external_libs(&artifact, &policy, sysroot).with_context(|| {
                 if let Some(platform_tag) = platform_tag {
                     format!("Error repairing wheel for {} compliance", platform_tag)
                 } else {
@@ -685,6 +687,50 @@ fn relpath(to: &Path, from: &Path) -> PathBuf {
     result
 }
 
+/// Get sysroot path from target C compiler
+///
+/// Currently only gcc is supported, clang doesn't have a `--print-sysroot` option
+/// TODO: allow specify sysroot from environment variable?
+fn get_sysroot_path(target: &Target) -> Result<PathBuf> {
+    use crate::target::get_host_target;
+
+    let host_triple = get_host_target()?;
+    let target_triple = target.target_triple();
+    if host_triple != target_triple {
+        let mut build = cc::Build::new();
+        build
+            // Suppress cargo metadata for example env vars printing
+            .cargo_metadata(false)
+            // opt_level, host and target are required
+            .opt_level(0)
+            .host(&host_triple)
+            .target(target_triple);
+        let compiler = build
+            .try_get_compiler()
+            .with_context(|| format!("Failed to get compiler for {}", target_triple))?;
+        let path = compiler.path();
+        let out = Command::new(path)
+            .arg("--print-sysroot")
+            .stdout(Stdio::piped())
+            .stderr(Stdio::null())
+            .output()
+            .with_context(|| format!("Failed to run `{} --print-sysroot`", path.display()))?;
+        if out.status.success() {
+            let sysroot = String::from_utf8(out.stdout)
+                .context("Failed to read the sysroot path")?
+                .trim()
+                .to_owned();
+            return Ok(PathBuf::from(sysroot));
+        } else {
+            bail!(
+                "Failed to get the sysroot path: {}",
+                String::from_utf8(out.stderr)?
+            );
+        }
+    }
+    Ok(PathBuf::from("/"))
+}
+
 #[cfg(test)]
 mod test {
     use super::relpath;