Wire credential helper to repository fetching.

After this change, credential helpers will be used to fetch credentials for
repository fetching (rctx.download and rctx.download_and_extract), which
take precedence over the `auth` parameter.

Also improve the documentation for credential helper related flags.

Fixes bazelbuild#15013.

Author: Yannic

src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java

@@ -148,11 +148,12 @@ public class AuthAndTLSOptions extends OptionsBase {
       documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
       effectTags = {OptionEffectTag.UNKNOWN},
       help =
-          "Configures Credential Helpers to use for retrieving credentials for the provided scope"
-              + " (domain).\n\n"
-              + "Credentials from Credential Helpers take precedence over credentials from"
-              + " <code>--google_default_credentials</code>, `--google_credentials`, or"
-              + " <code>.netrc</code>.\n\n"
+          "Configures a credential helper to use for retrieving authorization credentials for "
+              + " repository fetching, remote caching and execution, and the build event service.\n\n"
+              + "Credentials supplied by a helper take precedence over credentials supplied by"
+              + " --google_default_credentials, --google_credentials, a .netrc file, or the auth"
+              + " parameter to repository_ctx.download and repository_ctx.download_and_extract.\n\n"
+              + "May be specified multiple times to set up multiple helpers.\n\n"
               + "See https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md"
               + " for details.")
   public List<UnresolvedScopedCredentialHelper> credentialHelpers;
@@ -164,8 +165,8 @@ public class AuthAndTLSOptions extends OptionsBase {
       documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
       effectTags = {OptionEffectTag.UNKNOWN},
       help =
-          "Configures the timeout for the Credential Helper.\n\n"
-              + "Credential Helpers failing to respond within this timeout will fail the"
+          "Configures the timeout for a credential helper.\n\n"
+              + "Credential helpers failing to respond within this timeout will fail the"
               + " invocation.")
   public Duration credentialHelperTimeout;
 
@@ -176,31 +177,43 @@ public class AuthAndTLSOptions extends OptionsBase {
       documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
       effectTags = {OptionEffectTag.UNKNOWN},
       help =
-          "Configures the duration for which credentials from Credential Helpers are cached.\n\n"
+          "The duration for which credentials supplied by a credential helper are cached.\n\n"
               + "Invoking with a different value will adjust the lifetime of preexisting entries;"
               + " pass zero to clear the cache. A clean command always clears the cache, regardless"
               + " of this flag.")
   public Duration credentialHelperCacheTimeout;
 
-  /** One of the values of the `--credential_helper` flag. */
+  /**
+   * One of the values of the `--experimental_credential_helper` flag.
+   */
   @AutoValue
   public abstract static class UnresolvedScopedCredentialHelper {
-    /** Returns the scope of the credential helper (if any). */
+
+    /**
+     * Returns the scope of the credential helper (if any).
+     */
     public abstract Optional<String> getScope();
 
-    /** Returns the (unparsed) path of the credential helper. */
+    /**
+     * Returns the (unparsed) path of the credential helper.
+     */
     public abstract String getPath();
   }
 
-  /** A {@link Converter} for the `--credential_helper` flag. */
+  /**
+   * A {@link Converter} for the `--experimental_credential_helper` flag.
+   */
   public static final class UnresolvedScopedCredentialHelperConverter
       extends Converter.Contextless<UnresolvedScopedCredentialHelper> {
+
     public static final UnresolvedScopedCredentialHelperConverter INSTANCE =
         new UnresolvedScopedCredentialHelperConverter();
 
     @Override
     public String getTypeDescription() {
-      return "An (unresolved) path to a credential helper for a scope.";
+      return "Path to a credential helper, optionally prefixed by a scope (domain name) followed"
+          + " an '='. The path may be absolute or relative. If no scope is provided, the helper"
+          + " has universal scope.";
     }
 
     @Override

src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java

@@ -372,7 +372,6 @@ static Optional<Credentials> newCredentialsFromNetrc(
     }
   }
 
-  @VisibleForTesting
   public static CredentialHelperProvider newCredentialHelperProvider(
       CredentialHelperEnvironment environment,
       CommandLinePathFactory pathFactory,

src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD

@@ -20,6 +20,7 @@ java_library(
     deps = [
         "//src/main/java/com/google/devtools/build/lib:runtime",
         "//src/main/java/com/google/devtools/build/lib/authandtls",
+        "//src/main/java/com/google/devtools/common/options:options_internal",
         "//third_party:caffeine",
         "//third_party:guava",
     ],

src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialModule.java

@@ -21,22 +21,33 @@
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.runtime.BlazeModule;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.common.options.OptionsBase;
 import java.net.URI;
 import java.time.Duration;
 
-/** A module whose sole purpose is to hold the credential cache which is shared by other modules. */
+/**
+ * A module whose sole purpose is to hold the credential cache which is shared by other modules.
+ */
 public class CredentialModule extends BlazeModule {
+
   private final Cache<URI, ImmutableMap<String, ImmutableList<String>>> credentialCache =
       Caffeine.newBuilder()
           .expireAfterWrite(Duration.ZERO)
           .ticker(SystemMillisTicker.INSTANCE)
           .build();
 
-  /** Returns the credential cache. */
+  /**
+   * Returns the credential cache.
+   */
   public Cache<URI, ImmutableMap<String, ImmutableList<String>>> getCredentialCache() {
     return credentialCache;
   }
 
+  @Override
+  public Iterable<Class<? extends OptionsBase>> getCommonCommandOptions() {
+    return ImmutableList.of(AuthAndTLSOptions.class);
+  }
+
   @Override
   public void beforeCommand(CommandEnvironment env) {
     // Update the cache expiration policy according to the command options.

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -27,6 +27,9 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
         "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
+        "//src/main/java/com/google/devtools/build/lib/authandtls",
+        "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper",
+        "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection_impl",
@@ -55,6 +58,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository",
         "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
         "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
+        "//src/main/java/com/google/devtools/build/lib/util:exit_code",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/common/options",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -15,6 +15,7 @@
 
 package com.google.devtools.build.lib.bazel;
 
+import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
@@ -25,6 +26,13 @@
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
+import com.google.devtools.build.lib.authandtls.StaticCredentials;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
 import com.google.devtools.build.lib.bazel.bzlmod.AttributeValues;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
@@ -110,6 +118,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 
 /** Adds support for fetching external code. */
 public class BazelRepositoryModule extends BlazeModule {
@@ -146,6 +155,9 @@ public class BazelRepositoryModule extends BlazeModule {
   private List<String> allowedYankedVersions = ImmutableList.of();
   private SingleExtensionEvalFunction singleExtensionEvalFunction;
 
+  @Nullable
+  private CredentialModule credentialModule;
+
   public BazelRepositoryModule() {
     this.starlarkRepositoryFunction = new StarlarkRepositoryFunction(downloadManager);
     this.repositoryHandlers = repositoryRules();
@@ -263,6 +275,8 @@ SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace
         .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction)
         .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction());
     filesystem = runtime.getFileSystem();
+
+    credentialModule = Preconditions.checkNotNull(runtime.getBlazeModule(CredentialModule.class));
   }
 
   @Override
@@ -373,6 +387,41 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
                 Code.BAD_DOWNLOADER_CONFIG));
       }
 
+      try {
+        AuthAndTLSOptions authAndTlsOptions = env.getOptions().getOptions(AuthAndTLSOptions.class);
+        var credentialHelperEnvironment =
+            CredentialHelperEnvironment.newBuilder()
+                .setEventReporter(env.getReporter())
+                .setWorkspacePath(env.getWorkspace())
+                .setClientEnvironment(env.getClientEnv())
+                .setHelperExecutionTimeout(authAndTlsOptions.credentialHelperTimeout)
+                .build();
+        CredentialHelperProvider credentialHelperProvider =
+            GoogleAuthUtils.newCredentialHelperProvider(
+                credentialHelperEnvironment,
+                env.getCommandLinePathFactory(),
+                authAndTlsOptions.credentialHelpers);
+
+        downloadManager.setCredentialFactory(headers -> {
+          Preconditions.checkNotNull(headers);
+
+          return new CredentialHelperCredentials(
+              credentialHelperProvider,
+              credentialHelperEnvironment,
+              credentialModule.getCredentialCache(),
+              Optional.of(new StaticCredentials(headers)));
+        });
+      } catch (IOException e) {
+        env.getReporter().handle(Event.error(e.getMessage()));
+        env.getBlazeModuleEnvironment()
+            .exit(
+                new AbruptExitException(
+                    detailedExitCode(
+                        "Error initializing credential helper",
+                        Code.CREDENTIALS_INIT_FAILURE)));
+        return;
+      }
+
       if (repoOptions.experimentalDistdir != null) {
         downloadManager.setDistdir(
             repoOptions.experimentalDistdir.stream()

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java

@@ -20,6 +20,7 @@
 import com.google.auth.Credentials;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -61,6 +62,7 @@ public class DownloadManager {
   private int retries = 0;
   private boolean urlsAsDefaultCanonicalId;
   @Nullable private Credentials netrcCreds;
+  private CredentialFactory credentialFactory = new DefaultCredentialFactory();
 
   public DownloadManager(RepositoryCache repositoryCache, Downloader downloader) {
     this.repositoryCache = repositoryCache;
@@ -92,6 +94,10 @@ public void setNetrcCreds(Credentials netrcCreds) {
     this.netrcCreds = netrcCreds;
   }
 
+  public void setCredentialFactory(CredentialFactory credentialFactory) {
+    this.credentialFactory = credentialFactory;
+  }
+
   /**
    * Downloads file to disk and returns path.
    *
@@ -257,7 +263,7 @@ public Path download(
       try {
         downloader.download(
             rewrittenUrls,
-            new StaticCredentials(rewrittenAuthHeaders),
+            credentialFactory.create(rewrittenAuthHeaders),
             checksum,
             canonicalId,
             destination,
@@ -338,7 +344,7 @@ public byte[] downloadAndReadOneUrl(
     for (int attempt = 0; attempt <= retries; ++attempt) {
       try {
         return httpDownloader.downloadAndReadOneUrl(
-            rewrittenUrls.get(0), new StaticCredentials(authHeaders), eventHandler, clientEnv);
+            rewrittenUrls.get(0), credentialFactory.create(authHeaders), eventHandler, clientEnv);
       } catch (ContentLengthMismatchException e) {
         if (attempt == retries) {
           throw e;
@@ -427,4 +433,17 @@ public boolean isFinished() {
       return isFinished;
     }
   }
+
+  public interface CredentialFactory {
+    Credentials create(Map<URI, Map<String, List<String>>> authHeaders);
+  }
+
+  private static final class DefaultCredentialFactory implements CredentialFactory {
+    @Override
+    public Credentials create(Map<URI, Map<String, List<String>>> authHeaders) {
+      Preconditions.checkNotNull(authHeaders);
+
+      return new StaticCredentials(authHeaders);
+    }
+  }
 }

src/main/protobuf/failure_details.proto

@@ -249,6 +249,7 @@ message ExternalRepository {
     OVERRIDE_DISALLOWED_MANAGED_DIRECTORIES = 1 [(metadata) = { exit_code: 2 }];
     BAD_DOWNLOADER_CONFIG = 2 [(metadata) = { exit_code: 2 }];
     REPOSITORY_MAPPING_RESOLUTION_FAILED = 3 [(metadata) = { exit_code: 37 }];
+    CREDENTIALS_INIT_FAILURE = 4 [(metadata) = { exit_code: 2 }];
   }
   Code code = 1;
   // Additional data could include external repository names.

src/test/java/com/google/devtools/build/lib/buildtool/util/BUILD

@@ -54,6 +54,8 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:top_level_artifact_context",
         "//src/main/java/com/google/devtools/build/lib/analysis:transitive_info_collection",
         "//src/main/java/com/google/devtools/build/lib/analysis:workspace_status_action",
+        "//src/main/java/com/google/devtools/build/lib/authandtls",
+        "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
         "//src/main/java/com/google/devtools/build/lib/bazel:main",
         "//src/main/java/com/google/devtools/build/lib/bazel:repository_module",
         "//src/main/java/com/google/devtools/build/lib/bugreport",

src/test/java/com/google/devtools/build/lib/buildtool/util/BlazeRuntimeWrapper.java

@@ -32,6 +32,7 @@
 import com.google.devtools.build.lib.analysis.config.BuildOptions;
 import com.google.devtools.build.lib.analysis.config.BuildOptionsView;
 import com.google.devtools.build.lib.analysis.config.ExecutionTransitionFactory;
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.bugreport.Crash;
 import com.google.devtools.build.lib.bugreport.CrashContext;
 import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions;
@@ -285,6 +286,7 @@ private OptionsParser createOptionsParser(Command commandAnnotation) {
     Set<Class<? extends OptionsBase>> options =
         new HashSet<>(
             ImmutableList.of(
+                AuthAndTLSOptions.class,
                 BuildRequestOptions.class,
                 BuildEventProtocolOptions.class,
                 ExecutionOptions.class,

src/test/java/com/google/devtools/build/lib/buildtool/util/BuildIntegrationTestCase.java

@@ -53,6 +53,7 @@
 import com.google.devtools.build.lib.analysis.util.AnalysisMock;
 import com.google.devtools.build.lib.analysis.util.AnalysisTestUtil;
 import com.google.devtools.build.lib.analysis.util.AnalysisTestUtil.DummyWorkspaceStatusActionContext;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
 import com.google.devtools.build.lib.bazel.BazelRepositoryModule;
 import com.google.devtools.build.lib.bugreport.BugReport;
 import com.google.devtools.build.lib.bugreport.BugReporter;
@@ -571,7 +572,8 @@ protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
             .addBlazeModule(new OutputFilteringModule())
             .addBlazeModule(connectivityModule)
             .addBlazeModule(new SkymeldModule())
-            .addBlazeModule(getMockBazelRepositoryModule());
+            .addBlazeModule(getMockBazelRepositoryModule())
+            .addBlazeModule(new CredentialModule());
     getSpawnModules().forEach(builder::addBlazeModule);
     builder
         .addBlazeModule(getBuildInfoModule())

src/test/shell/bazel/remote_helpers.sh

@@ -38,8 +38,7 @@ function serve_file() {
   cd -
 }
 
-# Serves $1 as a file on localhost:$nc_port insisting on authentication (but
-# accepting any credentials.
+# Serves $1 as a file on localhost:$nc_port expecting authentication.
 #   * nc_port - the port nc is listening on.
 #   * nc_log - the path to nc's log.
 #   * nc_pid - the PID of nc.