Add netrc support to --bes_backend

Progress on bazelbuild#15856

Fixes bazelbuild#15928

Author: Yannic

src/main/java/com/google/devtools/build/lib/authandtls/BUILD

@@ -15,6 +15,8 @@ java_library(
     srcs = glob(["*.java"]),
     deps = [
         "//src/main/java/com/google/devtools/build/lib/concurrent",
+        "//src/main/java/com/google/devtools/build/lib/events",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/common/options",
         "//third_party:auth",
         "//third_party:auto_value",

src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java

@@ -19,6 +19,8 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
 import io.grpc.CallCredentials;
 import io.grpc.ClientInterceptor;
 import io.grpc.ManagedChannel;
@@ -41,6 +43,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
@@ -191,7 +195,20 @@ private static NettyChannelBuilder newNettyChannelBuilder(String targetUrl, Stri
    * @throws IOException in case the call credentials can't be constructed.
    */
   @Nullable
-  public static CallCredentials newCallCredentials(AuthAndTLSOptions options) throws IOException {
+  public static CallCredentials newCallCredentials(
+      Map<String, String> clientEnv,
+      FileSystem fileSystem,
+      AuthAndTLSOptions options) throws IOException {
+    Credentials creds = newCredentials(clientEnv, fileSystem, options);
+    if (creds != null) {
+      return MoreCallCredentials.from(creds);
+    }
+    return null;
+  }
+
+  @Nullable
+  @VisibleForTesting
+  public static CallCredentials newCallCredentialsForTesting(AuthAndTLSOptions options) throws IOException {
     Credentials creds = newCredentials(options);
     if (creds != null) {
       return MoreCallCredentials.from(creds);
@@ -217,12 +234,46 @@ public static CallCredentialsProvider newCallCredentialsProvider(@Nullable Crede
    */
   @Nullable
   public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) throws IOException {
+    Optional<Credentials> credentials = newGoogleCredentials(options);
+
+    return credentials.orElse(null);
+  }
+
+  /**
+   * Create a new {@link Credentials} with following order:
+   *
+   * <ol>
+   *   <li>If authentication enabled by flags, use it to create credentials
+   *   <li>Use .netrc to provide credentials if exists
+   *   <li>Otherwise, return {@code null}
+   * </ol>
+   *
+   * @throws IOException in case the credentials can't be constructed.
+   */
+  @VisibleForTesting
+  public static Credentials newCredentials(
+      Map<String, String> clientEnv,
+      FileSystem fileSystem,
+      AuthAndTLSOptions authAndTlsOptions)
+      throws IOException {
+    Optional<Credentials> credentials = newGoogleCredentials(authAndTlsOptions);
+
+    if (credentials.isEmpty()) {
+      // Fallback to .netrc if it exists.
+      credentials = newCredentialsFromNetrc(clientEnv, fileSystem);
+    }
+
+    return credentials.orElse(null);
+  }
+
+  private static Optional<Credentials> newGoogleCredentials(
+      @Nullable AuthAndTLSOptions options) throws IOException {
     if (options == null) {
-      return null;
+      return Optional.empty();
     } else if (options.googleCredentials != null) {
       // Credentials from file
       try (InputStream authFile = new FileInputStream(options.googleCredentials)) {
-        return newCredentials(authFile, options.googleAuthScopes);
+        return Optional.of(newGoogleCredentials(authFile, options.googleAuthScopes));
       } catch (FileNotFoundException e) {
         String message =
             String.format(
@@ -231,10 +282,10 @@ public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) th
         throw new IOException(message, e);
       }
     } else if (options.useGoogleDefaultCredentials) {
-      return newCredentials(
-          null /* Google Application Default Credentials */, options.googleAuthScopes);
+      return Optional.of(newGoogleCredentials(
+          null /* Google Application Default Credentials */, options.googleAuthScopes));
     }
-    return null;
+    return Optional.empty();
   }
 
   /**
@@ -243,7 +294,7 @@ public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) th
    * @throws IOException in case the credentials can't be constructed.
    */
   @VisibleForTesting
-  public static Credentials newCredentials(
+  public static Credentials newGoogleCredentials(
       @Nullable InputStream credentialsFile, List<String> authScopes) throws IOException {
     try {
       GoogleCredentials creds =
@@ -259,4 +310,43 @@ public static Credentials newCredentials(
       throw new IOException(message, e);
     }
   }
+
+  /**
+   * Create a new {@link Credentials} object by parsing the .netrc file with following order to
+   * search it:
+   *
+   * <ol>
+   *   <li>If environment variable $NETRC exists, use it as the path to the .netrc file
+   *   <li>Fallback to $HOME/.netrc
+   * </ol>
+   *
+   * @return the {@link Credentials} object or {@code null} if there is no .netrc file.
+   * @throws IOException in case the credentials can't be constructed.
+   */
+  @VisibleForTesting
+  static Optional<Credentials> newCredentialsFromNetrc(Map<String, String> clientEnv, FileSystem fileSystem)
+      throws IOException {
+    Optional<String> netrcFileString =
+        Optional.ofNullable(clientEnv.get("NETRC"))
+            .or(
+                () ->
+                    Optional.ofNullable(clientEnv.get("HOME"))
+                        .map(home -> home + "/.netrc"));
+    if (netrcFileString.isEmpty()) {
+      return Optional.empty();
+    }
+
+    Path netrcFile = fileSystem.getPath(netrcFileString.get());
+    if (!netrcFile.exists()) {
+      return Optional.empty();
+    }
+
+    try {
+      Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream());
+      return Optional.of(new NetrcCredentials(netrc));
+    } catch (IOException e) {
+      throw new IOException(
+          "Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), e);
+    }
+  }
 }

src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java

@@ -24,6 +24,7 @@
 import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
 import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceClient;
 import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceGrpcClient;
+import com.google.devtools.build.lib.runtime.CommandEnvironment;
 import io.grpc.ClientInterceptor;
 import io.grpc.ManagedChannel;
 import io.grpc.Metadata;
@@ -70,15 +71,16 @@ protected Class<BuildEventServiceOptions> optionsClass() {
 
   @Override
   protected BuildEventServiceClient getBesClient(
-      BuildEventServiceOptions besOptions, AuthAndTLSOptions authAndTLSOptions) throws IOException {
+      CommandEnvironment env, BuildEventServiceOptions besOptions, AuthAndTLSOptions authAndTLSOptions) throws IOException {
     BackendConfig newConfig = BackendConfig.create(besOptions, authAndTLSOptions);
     if (client == null || !Objects.equals(config, newConfig)) {
       clearBesClient();
       config = newConfig;
       client =
           new BuildEventServiceGrpcClient(
               newGrpcChannel(config),
-              GoogleAuthUtils.newCallCredentials(config.authAndTLSOptions()),
+              GoogleAuthUtils.newCallCredentials(
+                  env.getClientEnv(), env.getRuntime().getFileSystem(), config.authAndTLSOptions()),
               makeGrpcInterceptor(config));
     }
     return client;

src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java

@@ -699,7 +699,7 @@ private BuildEventServiceTransport createBesTransport(
 
     final BuildEventServiceClient besClient;
     try {
-      besClient = getBesClient(besOptions, authTlsOptions);
+      besClient = getBesClient(cmdEnv, besOptions, authTlsOptions);
     } catch (IOException | OptionsParsingException e) {
       reportError(
           reporter,
@@ -845,7 +845,7 @@ private static AbruptExitException createAbruptExitException(
   protected abstract Class<OptionsT> optionsClass();
 
   protected abstract BuildEventServiceClient getBesClient(
-      OptionsT besOptions, AuthAndTLSOptions authAndTLSOptions)
+      CommandEnvironment env, OptionsT besOptions, AuthAndTLSOptions authAndTLSOptions)
       throws IOException, OptionsParsingException;
 
   protected abstract void clearBesClient();

src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java

@@ -49,9 +49,6 @@
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions.UnresolvedScopedCredentialHelper;
 import com.google.devtools.build.lib.authandtls.CallCredentialsProvider;
 import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
-import com.google.devtools.build.lib.authandtls.Netrc;
-import com.google.devtools.build.lib.authandtls.NetrcCredentials;
-import com.google.devtools.build.lib.authandtls.NetrcParser;
 import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
 import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider;
 import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
@@ -1047,95 +1044,6 @@ RemoteActionContextProvider getActionContextProvider() {
     return actionContextProvider;
   }
 
-  /**
-   * Create a new {@link Credentials} object by parsing the .netrc file with following order to
-   * search it:
-   *
-   * <ol>
-   *   <li>If environment variable $NETRC exists, use it as the path to the .netrc file
-   *   <li>Fallback to $HOME/.netrc
-   * </ol>
-   *
-   * @return the {@link Credentials} object or {@code null} if there is no .netrc file.
-   * @throws IOException in case the credentials can't be constructed.
-   */
-  @Nullable
-  @VisibleForTesting
-  static Credentials newCredentialsFromNetrc(Map<String, String> clientEnv, FileSystem fileSystem)
-      throws IOException {
-    String netrcFileString =
-        Optional.ofNullable(clientEnv.get("NETRC"))
-            .orElseGet(
-                () ->
-                    Optional.ofNullable(clientEnv.get("HOME"))
-                        .map(home -> home + "/.netrc")
-                        .orElse(null));
-    if (netrcFileString == null) {
-      return null;
-    }
-
-    Path netrcFile = fileSystem.getPath(netrcFileString);
-    if (netrcFile.exists()) {
-      try {
-        Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream());
-        return new NetrcCredentials(netrc);
-      } catch (IOException e) {
-        throw new IOException(
-            "Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), e);
-      }
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * Create a new {@link Credentials} with following order:
-   *
-   * <ol>
-   *   <li>If authentication enabled by flags, use it to create credentials
-   *   <li>Use .netrc to provide credentials if exists
-   *   <li>Otherwise, return {@code null}
-   * </ol>
-   *
-   * @throws IOException in case the credentials can't be constructed.
-   */
-  @VisibleForTesting
-  static Credentials newCredentials(
-      Map<String, String> clientEnv,
-      FileSystem fileSystem,
-      Reporter reporter,
-      AuthAndTLSOptions authAndTlsOptions,
-      RemoteOptions remoteOptions)
-      throws IOException {
-    Credentials creds = GoogleAuthUtils.newCredentials(authAndTlsOptions);
-
-    // Fallback to .netrc if it exists
-    if (creds == null) {
-      try {
-        creds = newCredentialsFromNetrc(clientEnv, fileSystem);
-      } catch (IOException e) {
-        reporter.handle(Event.warn(e.getMessage()));
-      }
-
-      try {
-        if (creds != null
-            && remoteOptions.remoteCache != null
-            && Ascii.toLowerCase(remoteOptions.remoteCache).startsWith("http://")
-            && !creds.getRequestMetadata(new URI(remoteOptions.remoteCache)).isEmpty()) {
-          reporter.handle(
-              Event.warn(
-                  "Username and password from .netrc is transmitted in plaintext to "
-                      + remoteOptions.remoteCache
-                      + ". Please consider using an HTTPS endpoint."));
-        }
-      } catch (URISyntaxException e) {
-        throw new IOException(e.getMessage(), e);
-      }
-    }
-
-    return creds;
-  }
-
   @VisibleForTesting
   static CredentialHelperProvider newCredentialHelperProvider(
       CredentialHelperEnvironment environment,
@@ -1159,6 +1067,33 @@ static CredentialHelperProvider newCredentialHelperProvider(
     return builder.build();
   }
 
+  static Credentials newCredentials(
+      Map<String, String> clientEnv,
+      FileSystem fileSystem,
+      Reporter reporter,
+      AuthAndTLSOptions authAndTlsOptions,
+      RemoteOptions remoteOptions) throws IOException {
+    Credentials credentials = GoogleAuthUtils.newCredentials(clientEnv, fileSystem, authAndTlsOptions);
+
+    try {
+      if (credentials != null
+          && remoteOptions.remoteCache != null
+          && Ascii.toLowerCase(remoteOptions.remoteCache).startsWith("http://")
+          && !credentials.getRequestMetadata(new URI(remoteOptions.remoteCache)).isEmpty()) {
+        // TODO(yannic): Make this a error aborting the build.
+        reporter.handle(
+            Event.warn(
+                "Credentials are transmitted in plaintext to "
+                    + remoteOptions.remoteCache
+                    + ". Please consider using an HTTPS endpoint."));
+      }
+    } catch (URISyntaxException e) {
+      throw new IOException(e.getMessage(), e);
+    }
+
+    return credentials;
+  }
+
   @VisibleForTesting
   @AutoValue
   abstract static class ScopedCredentialHelper {

src/test/java/com/google/devtools/build/lib/authandtls/BUILD

@@ -25,7 +25,11 @@ java_library(
     ),
     deps = [
         "//src/main/java/com/google/devtools/build/lib/authandtls",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
         "//src/main/java/com/google/devtools/common/options",
+        "//src/test/java/com/google/devtools/build/lib/testutil",
+        "//third_party:auth_checked_in",
         "//third_party:guava",
         "//third_party:junit4",
         "//third_party:truth",