Auto merge of #13252 - connorworley:shallow_libgit2_fetches, r=weihanglo

Implementation of shallow libgit2 fetches behind an unstable flag

This is my first contribution, so guidance is appreciated.

Fixes #1171 by moving the `shallow-index` and `shallow-deps` aspects of `-Zgitoxide` to a new `-Zgit` unstable flag.

The only change in interaction with libgit2 happens in `src/cargo/sources/git/utils.rs`, where we set the depth fetch option if applicable.

Shallow fetch tests for gitoxide continue to pass, but libgit2 is harder to test as it silently ignores the depth option for local fetches. I would love any ideas on how to test it in a lightweight way or whether it's OK to wait for an upstream fix.

Author: bors

src/cargo/core/features.rs

@@ -770,6 +770,7 @@ unstable_cli_options!(
     dual_proc_macros: bool = ("Build proc-macros for both the host and the target"),
     features: Option<Vec<String>>,
     gc: bool = ("Track cache usage and \"garbage collect\" unused files"),
+    git: Option<GitFeatures> = ("Enable support for shallow git fetch operations"),
     gitoxide: Option<GitoxideFeatures> = ("Use gitoxide for the given git interactions, or all of them if no argument is given"),
     host_config: bool = ("Enable the `[host]` section in the .cargo/config.toml file"),
     lints: bool = ("Pass `[lints]` to the linting tools"),
@@ -875,13 +876,47 @@ where
 }
 
 #[derive(Debug, Copy, Clone, Default, Deserialize)]
-pub struct GitoxideFeatures {
-    /// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
-    pub fetch: bool,
+pub struct GitFeatures {
     /// When cloning the index, perform a shallow clone. Maintain shallowness upon subsequent fetches.
     pub shallow_index: bool,
     /// When cloning git dependencies, perform a shallow clone and maintain shallowness on subsequent fetches.
     pub shallow_deps: bool,
+}
+
+impl GitFeatures {
+    fn all() -> Self {
+        GitFeatures {
+            shallow_index: true,
+            shallow_deps: true,
+        }
+    }
+}
+
+fn parse_git(it: impl Iterator<Item = impl AsRef<str>>) -> CargoResult<Option<GitFeatures>> {
+    let mut out = GitFeatures::default();
+    let GitFeatures {
+        shallow_index,
+        shallow_deps,
+    } = &mut out;
+
+    for e in it {
+        match e.as_ref() {
+            "shallow-index" => *shallow_index = true,
+            "shallow-deps" => *shallow_deps = true,
+            _ => {
+                bail!(
+                    "unstable 'git' only takes 'shallow-index' and 'shallow-deps' as valid inputs"
+                )
+            }
+        }
+    }
+    Ok(Some(out))
+}
+
+#[derive(Debug, Copy, Clone, Default, Deserialize)]
+pub struct GitoxideFeatures {
+    /// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
+    pub fetch: bool,
     /// Checkout git dependencies using `gitoxide` (submodules are still handled by git2 ATM, and filters
     /// like linefeed conversions are unsupported).
     pub checkout: bool,
@@ -895,9 +930,7 @@ impl GitoxideFeatures {
     fn all() -> Self {
         GitoxideFeatures {
             fetch: true,
-            shallow_index: true,
             checkout: true,
-            shallow_deps: true,
             internal_use_git2: false,
         }
     }
@@ -907,9 +940,7 @@ impl GitoxideFeatures {
     fn safe() -> Self {
         GitoxideFeatures {
             fetch: true,
-            shallow_index: false,
             checkout: true,
-            shallow_deps: false,
             internal_use_git2: false,
         }
     }
@@ -921,21 +952,17 @@ fn parse_gitoxide(
     let mut out = GitoxideFeatures::default();
     let GitoxideFeatures {
         fetch,
-        shallow_index,
         checkout,
-        shallow_deps,
         internal_use_git2,
     } = &mut out;
 
     for e in it {
         match e.as_ref() {
             "fetch" => *fetch = true,
-            "shallow-index" => *shallow_index = true,
-            "shallow-deps" => *shallow_deps = true,
             "checkout" => *checkout = true,
             "internal-use-git2" => *internal_use_git2 = true,
             _ => {
-                bail!("unstable 'gitoxide' only takes `fetch`, 'shallow-index', 'shallow-deps' and 'checkout' as valid inputs")
+                bail!("unstable 'gitoxide' only takes `fetch` and 'checkout' as valid input, for shallow fetches see `-Zgit=shallow-index,shallow-deps`")
             }
         }
     }
@@ -1109,6 +1136,12 @@ impl CliUnstable {
             "doctest-xcompile" => self.doctest_xcompile = parse_empty(k, v)?,
             "dual-proc-macros" => self.dual_proc_macros = parse_empty(k, v)?,
             "gc" => self.gc = parse_empty(k, v)?,
+            "git" => {
+                self.git = v.map_or_else(
+                    || Ok(Some(GitFeatures::all())),
+                    |v| parse_git(v.split(',')),
+                )?
+            }
             "gitoxide" => {
                 self.gitoxide = v.map_or_else(
                     || Ok(Some(GitoxideFeatures::all())),

src/cargo/sources/git/mod.rs

@@ -16,7 +16,7 @@ mod utils;
 
 /// For `-Zgitoxide` integration.
 pub mod fetch {
-    use crate::core::features::GitoxideFeatures;
+    use crate::core::features::GitFeatures;
     use crate::Config;
 
     /// The kind remote repository to fetch.
@@ -37,18 +37,19 @@ pub mod fetch {
             repo_is_shallow: bool,
             config: &Config,
         ) -> gix::remote::fetch::Shallow {
-            let has_feature = |cb: &dyn Fn(GitoxideFeatures) -> bool| {
+            let has_feature = |cb: &dyn Fn(GitFeatures) -> bool| {
                 config
                     .cli_unstable()
-                    .gitoxide
+                    .git
                     .map_or(false, |features| cb(features))
             };
 
             // maintain shallow-ness and keep downloading single commits, or see if we can do shallow clones
             if !repo_is_shallow {
                 match self {
-                    RemoteKind::GitDependency if has_feature(&|git| git.shallow_deps) => {}
-                    RemoteKind::Registry if has_feature(&|git| git.shallow_index) => {}
+                    RemoteKind::GitDependency if has_feature(&|features| features.shallow_deps) => {
+                    }
+                    RemoteKind::Registry if has_feature(&|features| features.shallow_index) => {}
                     _ => return gix::remote::fetch::Shallow::NoChange,
                 }
             };

src/cargo/sources/git/oxide.rs

@@ -29,10 +29,9 @@ pub fn with_retry_and_progress(
 ) -> CargoResult<()> {
     std::thread::scope(|s| {
         let mut progress_bar = Progress::new("Fetch", config);
-        let is_shallow = config
-            .cli_unstable()
-            .gitoxide
-            .map_or(false, |gix| gix.shallow_deps || gix.shallow_index);
+        let is_shallow = config.cli_unstable().git.map_or(false, |features| {
+            features.shallow_deps || features.shallow_index
+        });
         network::retry::with_retry(config, || {
             let progress_root: Arc<gix::progress::tree::Root> =
                 gix::progress::tree::root::Options {

src/cargo/sources/git/source.rs

@@ -108,8 +108,8 @@ impl<'cfg> GitSource<'cfg> {
             &source_id,
             config
                 .cli_unstable()
-                .gitoxide
-                .map_or(false, |gix| gix.fetch && gix.shallow_deps),
+                .git
+                .map_or(false, |features| features.shallow_deps),
         );
 
         let source = GitSource {

src/cargo/sources/git/utils.rs

@@ -1117,6 +1117,9 @@ pub fn fetch(
             if tags {
                 opts.download_tags(git2::AutotagOption::All);
             }
+            if let gix::remote::fetch::Shallow::DepthAtRemote(depth) = shallow {
+                opts.depth(0i32.saturating_add_unsigned(depth.get()));
+            }
             // The `fetch` operation here may fail spuriously due to a corrupt
             // repository. It could also fail, however, for a whole slew of other
             // reasons (aka network related reasons). We want Cargo to automatically

src/cargo/sources/registry/mod.rs

@@ -476,8 +476,8 @@ impl<'cfg> RegistrySource<'cfg> {
             source_id,
             config
                 .cli_unstable()
-                .gitoxide
-                .map_or(false, |gix| gix.fetch && gix.shallow_index)
+                .git
+                .map_or(false, |features| features.shallow_index)
                 && !source_id.is_sparse(),
         );
         let ops = if source_id.is_sparse() {

src/doc/src/reference/unstable.md

@@ -1172,19 +1172,32 @@ to run with `gitoxide` with the `-Zgitoxide=operation[,operationN]` syntax.
 Valid operations are the following:
 
 * `fetch` - All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
+* `checkout` *(planned)* - checkout the worktree, with support for filters and submodules.
+
+## git
+
+* Tracking Issue: [#13285](https://github.com/rust-lang/cargo/issues/13285)
+
+With the 'git' unstable feature, both `gitoxide` and `git2` will perform shallow fetches of the crate
+index and git dependencies.
+
+While `-Zgit` enables all currently implemented features, one can individually select when to perform
+shallow fetches with the `-Zgit=operation[,operationN]` syntax.
+
+Valid operations are the following:
+
 * `shallow-index` - perform a shallow clone of the index.
 * `shallow-deps` - perform a shallow clone of git dependencies.
-* `checkout` *(planned)* - checkout the worktree, with support for filters and submodules.
 
 **Details on shallow clones**
 
-* To enable shallow clones, add `-Zgitoxide=fetch,shallow_deps` for fetching git dependencies or `-Zgitoxide=fetch,shallow_index` for fetching registry index.
+* To enable shallow clones, add `-Zgit=shallow-deps` for fetching git dependencies or `-Zgit=shallow-index` for fetching registry index.
 * Shallow-cloned and shallow-checked-out git repositories reside at their own `-shallow` suffixed directories, i.e,
   - `~/.cargo/registry/index/*-shallow`
   - `~/.cargo/git/db/*-shallow`
   - `~/.cargo/git/checkouts/*-shallow`
 * When the unstable feature is on, fetching/cloning a git repository is always a shallow fetch. This roughly equals to `git fetch --depth 1` everywhere.
-* Even with the presence of `Cargo.lock` or specifying a commit `{ rev = "…" }`, gitoxide is still smart enough to shallow fetch without unshallowing the existing repository.
+* Even with the presence of `Cargo.lock` or specifying a commit `{ rev = "…" }`, gitoxide and libgit2 are still smart enough to shallow fetch without unshallowing the existing repository.
 
 ## script
 

tests/testsuite/cargo/z_help/stdout.log

@@ -14,6 +14,7 @@ Available unstable (nightly-only) flags:
     -Z doctest-xcompile         Compile and run doctests for non-host target using runner config
     -Z dual-proc-macros         Build proc-macros for both the host and the target
     -Z gc                       Track cache usage and "garbage collect" unused files
+    -Z git                      Enable support for shallow git fetch operations
     -Z gitoxide                 Use gitoxide for the given git interactions, or all of them if no argument is given
     -Z host-config              Enable the `[host]` section in the .cargo/config.toml file
     -Z lints                    Pass `[lints]` to the linting tools