Merge branch 'master' into respect_rules_java

Author: meteorcloudy

.github/labeler.yml

@@ -37,6 +37,7 @@ team-Rules-Java:
 team-Android:
 - src/tools/android/java/com/google/devtools/build/android/**/*
 - src/test/java/com/google/devtools/build/android/**/*
+- src/test/shell/bazel/android/**/*
 
 team-Configurability:
 - src/main/java/com/google/devtools/build/lib/analysis/**/*

CODEOWNERS

@@ -27,6 +27,8 @@ android/ @ahumesky @ted-xie
 
 /src/test/java/com/google/devtools/build/android/dexer/ @ahumesky @ted-xie @kevin1e100
 
+/src/test/shell/bazel/android/ @ahumesky @ted-xie
+
 # Apple
 
 /src/main/java/com/google/devtools/build/lib/rules/apple @bazelbuild/rules_apple-team

WORKSPACE

@@ -425,11 +425,11 @@ maven_install(
         "com.google.code.findbugs:jsr305:3.0.2",
         "com.google.code.gson:gson:2.9.0",
         "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter:3.3.0",
-        "com.google.errorprone:error_prone_annotation:2.18.0",
-        "com.google.errorprone:error_prone_annotations:2.18.0",
-        "com.google.errorprone:error_prone_check_api:2.18.0",
-        "com.google.errorprone:error_prone_core:2.18.0",
-        "com.google.errorprone:error_prone_type_annotations:2.18.0",
+        "com.google.errorprone:error_prone_annotation:2.19.0",
+        "com.google.errorprone:error_prone_annotations:2.19.0",
+        "com.google.errorprone:error_prone_check_api:2.19.0",
+        "com.google.errorprone:error_prone_core:2.19.0",
+        "com.google.errorprone:error_prone_type_annotations:2.19.0",
         "com.google.flogger:flogger-system-backend:0.5.1",
         "com.google.flogger:flogger:0.5.1",
         "com.google.flogger:google-extensions:0.5.1",

maven_install.json

@@ -272,8 +272,12 @@ upper_tabs:
         path: /extending/repo
       - title: Configuring
         path: /extending/config
-      - title: Platforms and toolchains
+      - title: Platforms
         path: /extending/platforms
+      - title: Toolchains
+        path: /extending/toolchains
+      - title: Execution groups
+        path: /extending/exec-groups
     - name: Writing rules
       contents:
       - title: Creating a rule
@@ -309,12 +313,6 @@ upper_tabs:
       - title: Document rules with Stardoc
         path: https://github.com/bazelbuild/stardoc
         status: external
-    - name: Design docs
-      contents:
-      - title: Execution groups
-        path: /extending/exec-groups
-      - title: Toolchains
-        path: /extending/toolchains
     - name: APIs
       contents:
       - title: Build file API

site/en/_book.yaml

@@ -49,8 +49,8 @@
 import com.google.devtools.build.lib.graph.Node;
 import com.google.devtools.build.lib.packages.BazelStarlarkEnvironment;
 import com.google.devtools.build.lib.packages.NativeAspectClass;
+import com.google.devtools.build.lib.packages.PackageArgument;
 import com.google.devtools.build.lib.packages.PackageCallable;
-import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension;
 import com.google.devtools.build.lib.packages.RuleClass;
 import com.google.devtools.build.lib.packages.RuleClassProvider;
 import com.google.devtools.build.lib.packages.RuleFactory;
@@ -96,7 +96,7 @@
    * A coherent set of options, fragments, aspects and rules; each of these may declare a dependency
    * on other such sets.
    */
-  // TODO(b/280446865): Consider merging Bootstrap and EnvironmentExtension into this interface.
+  // TODO(b/280446865): Consider merging Bootstrap into this interface.
   public interface RuleSet {
     /** Add stuff to the configured rule class provider builder. */
     void init(ConfiguredRuleClassProvider.Builder builder);
@@ -157,7 +157,7 @@ public static final class Builder implements RuleDefinitionEnvironment {
     private final Map<String, RuleClass> ruleClassMap = new HashMap<>();
     private final Map<String, RuleDefinition> ruleDefinitionMap = new HashMap<>();
     private final Map<String, NativeAspectClass> nativeAspectClassMap = new HashMap<>();
-    private final List<EnvironmentExtension> environmentExtensions = new ArrayList<>();
+    private final List<PackageArgument<?>> packageArgs = new ArrayList<>();
     private final Map<Class<? extends RuleDefinition>, RuleClass> ruleMap = new HashMap<>();
     private final Digraph<Class<? extends RuleDefinition>> dependencyGraph = new Digraph<>();
     private final List<Class<? extends Fragment>> universalFragments = new ArrayList<>();
@@ -166,6 +166,7 @@ public static final class Builder implements RuleDefinitionEnvironment {
     private OptionsDiffPredicate shouldInvalidateCacheForOptionDiff =
         OptionsDiffPredicate.ALWAYS_INVALIDATE;
     private PrerequisiteValidator prerequisiteValidator;
+    private final ImmutableMap.Builder<String, Object> buildFileToplevels = ImmutableMap.builder();
     private final ImmutableList.Builder<Bootstrap> starlarkBootstraps = ImmutableList.builder();
     private final ImmutableMap.Builder<String, Object> starlarkAccessibleTopLevels =
         ImmutableMap.builder();
@@ -306,8 +307,8 @@ public Builder addNativeAspectClass(NativeAspectClass aspectFactoryClass) {
     }
 
     @CanIgnoreReturnValue
-    public Builder addEnvironmentExtension(EnvironmentExtension environmentExtension) {
-      environmentExtensions.add(environmentExtension);
+    public Builder addPackageArg(PackageArgument<?> arg) {
+      packageArgs.add(arg);
       return this;
     }
 
@@ -342,18 +343,39 @@ public Builder addUniversalConfigurationFragment(Class<? extends Fragment> fragm
       return this;
     }
 
+    /**
+     * Registers a new top-level symbol for BUILD files.
+     *
+     * <p>The symbol will also be available in BUILD-loaded .bzl files under the {@code native}
+     * module.
+     */
+    @CanIgnoreReturnValue
+    public Builder addBuildFileToplevel(String name, Object object) {
+      this.buildFileToplevels.put(name, object);
+      return this;
+    }
+
+    /**
+     * Registers all symbols contained in the {@code Bootstrap} as top-level symbols for .bzl files.
+     */
     @CanIgnoreReturnValue
     public Builder addStarlarkBootstrap(Bootstrap bootstrap) {
       this.starlarkBootstraps.add(bootstrap);
       return this;
     }
 
+    /** Registers a new top-level symbol for .bzl files. */
+    // TODO(b/280446865): rename to something like addBzlToplevel()
     @CanIgnoreReturnValue
     public Builder addStarlarkAccessibleTopLevels(String name, Object object) {
       this.starlarkAccessibleTopLevels.put(name, object);
       return this;
     }
 
+    /**
+     * Registers a new symbol for {@code @_builtins} .bzl files, to be made available under the
+     * {@code _builtins.internal} object.
+     */
     @CanIgnoreReturnValue
     public Builder addStarlarkBuiltinsInternal(String name, Object object) {
       this.starlarkBuiltinsInternals.put(name, object);
@@ -586,7 +608,7 @@ public ConfiguredRuleClassProvider build() {
           ImmutableMap.copyOf(ruleClassMap),
           ImmutableMap.copyOf(ruleDefinitionMap),
           ImmutableMap.copyOf(nativeAspectClassMap),
-          ImmutableList.copyOf(environmentExtensions),
+          ImmutableList.copyOf(packageArgs),
           FragmentRegistry.create(
               configurationFragmentClasses, universalFragments, configurationOptions),
           defaultWorkspaceFilePrefix.toString(),
@@ -596,6 +618,7 @@ public ConfiguredRuleClassProvider build() {
           toolchainTaggedTrimmingTransition,
           shouldInvalidateCacheForOptionDiff,
           prerequisiteValidator,
+          buildFileToplevels.buildOrThrow(),
           starlarkAccessibleTopLevels.buildOrThrow(),
           starlarkBuiltinsInternals.buildOrThrow(),
           starlarkBootstraps.build(),
@@ -705,7 +728,7 @@ private ConfiguredRuleClassProvider(
       ImmutableMap<String, RuleClass> ruleClassMap,
       ImmutableMap<String, RuleDefinition> ruleDefinitionMap,
       ImmutableMap<String, NativeAspectClass> nativeAspectClassMap,
-      ImmutableList<EnvironmentExtension> environmentExtensions,
+      ImmutableList<PackageArgument<?>> packageArgs,
       FragmentRegistry fragmentRegistry,
       String defaultWorkspaceFilePrefix,
       String defaultWorkspaceFileSuffix,
@@ -714,6 +737,7 @@ private ConfiguredRuleClassProvider(
       PatchTransition toolchainTaggedTrimmingTransition,
       OptionsDiffPredicate shouldInvalidateCacheForOptionDiff,
       PrerequisiteValidator prerequisiteValidator,
+      ImmutableMap<String, Object> buildFileToplevels,
       ImmutableMap<String, Object> starlarkAccessibleTopLevels,
       ImmutableMap<String, Object> starlarkBuiltinsInternals,
       ImmutableList<Bootstrap> starlarkBootstraps,
@@ -756,11 +780,11 @@ private ConfiguredRuleClassProvider(
     this.bazelStarlarkEnvironment =
         new BazelStarlarkEnvironment(
             /* ruleFunctions= */ RuleFactory.buildRuleFunctions(ruleClassMap),
-            /* environmentExtensions= */ environmentExtensions,
             // TODO(b/280446865): Instead of exposing the {@code package()} symbol separately, just
             // keep it part of the BUILD/native environments. Requires migrating {@code package()}
             // to be a @StarlarkMethod that gets registered as a BUILD toplevel.
-            /* packageCallable= */ PackageCallable.newPackageCallable(environmentExtensions),
+            /* packageCallable= */ PackageCallable.newPackageCallable(packageArgs),
+            buildFileToplevels,
             /* bzlToplevels= */ environment,
             /* nativeRuleSpecificBindings= */ nativeRuleSpecificBindings,
             /* workspaceBzlNativeBindings= */ WorkspaceFactory.createNativeModuleBindings(

src/main/java/com/google/devtools/build/lib/analysis/ConfiguredRuleClassProvider.java

@@ -88,6 +88,7 @@
 import com.google.devtools.build.lib.server.FailureDetails.Execution;
 import com.google.devtools.build.lib.server.FailureDetails.Execution.Code;
 import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
+import com.google.devtools.build.lib.skyframe.ActionExecutionInactivityWatchdog;
 import com.google.devtools.build.lib.skyframe.AspectKeyCreator.AspectKey;
 import com.google.devtools.build.lib.skyframe.BuildResultListener;
 import com.google.devtools.build.lib.skyframe.Builder;
@@ -339,8 +340,9 @@ public void prepareForExecution(Stopwatch executionTimer)
     }
 
     env.getEventBus()
-        .register(new ExecutionProgressReceiverSetup(skyframeExecutor, env, executionTimer));
-    // TODO(leba): Add watchdog support.
+        .register(
+            new ExecutionProgressReceiverSetup(
+                skyframeExecutor, env, executionTimer, buildRequestOptions.progressReportInterval));
     for (ExecutorLifecycleListener executorLifecycleListener : executorLifecycleListeners) {
       try (SilentCloseable c =
           Profiler.instance().profile(executorLifecycleListener + ".executionPhaseStarting")) {
@@ -1023,13 +1025,17 @@ private static class ExecutionProgressReceiverSetup {
     private final Stopwatch executionUnstartedTimer;
     private final AtomicBoolean activated = new AtomicBoolean(false);
 
+    private final int progressReportInterval;
+
     ExecutionProgressReceiverSetup(
         SkyframeExecutor skyframeExecutor,
         CommandEnvironment env,
-        Stopwatch executionUnstartedTimer) {
+        Stopwatch executionUnstartedTimer,
+        int progressReportInterval) {
       this.skyframeExecutor = skyframeExecutor;
       this.env = env;
       this.executionUnstartedTimer = executionUnstartedTimer;
+      this.progressReportInterval = progressReportInterval;
     }
 
     @Subscribe
@@ -1053,6 +1059,13 @@ public void setupExecutionProgressReceiver(SomeExecutionStartedEvent unused) {
         skyframeExecutor.setExecutionProgressReceiver(executionProgressReceiver);
         executionUnstartedTimer.start();
 
+        skyframeExecutor.setAndStartWatchdog(
+            new ActionExecutionInactivityWatchdog(
+                executionProgressReceiver.createInactivityMonitor(statusReporter),
+                executionProgressReceiver.createInactivityReporter(
+                    statusReporter, skyframeExecutor.getIsBuildingExclusiveArtifacts()),
+                progressReportInterval));
+
         env.getEventBus().unregister(this);
       }
     }

src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java

@@ -15,10 +15,8 @@
 package com.google.devtools.build.lib.packages;
 
 import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Sets;
-import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -83,10 +81,11 @@ public final class BazelStarlarkEnvironment {
    *
    * @param ruleFunctions a map from a rule class name (e.g. "java_library") to the (uninjected)
    *     Starlark callable that instantiates it
-   * @param environmentExtensions a list of {@link EnvironmentExtensions} to apply
    * @param packageCallable the symbol implementing the {@code package()} function in BUILD files
-   * @param bzlToplevels the map of top-level symbols available to .bzl files (excluding {@link
-   *     Starlark#UNIVERSE} symbols), prior to builtins injection
+   * @param buildFileToplevels the map of non-{@link Starlark#UNIVERSE} top-level symbols available
+   *     to BUILD files, prior to builtins injection
+   * @param bzlToplevels the map of non-universe top-level symbols available to .bzl files, prior to
+   *     builtins injection, excluding the {@code native} object
    * @param nativeRuleSpecificBindings a subset of {@code bzlToplevels} that excludes core symbols
    *     like {@code rule()} and {@code struct()}. This is intended to identify symbols that are
    *     eligible for builtins injection.
@@ -98,10 +97,8 @@ public final class BazelStarlarkEnvironment {
    */
   public BazelStarlarkEnvironment(
       ImmutableMap<String, ?> ruleFunctions,
-      // TODO(b/280446865): Eliminate the EnvironmentExtensions concept, replace with underlying
-      // PackageArguments.
-      ImmutableList<EnvironmentExtension> environmentExtensions,
       Object packageCallable,
+      ImmutableMap<String, Object> buildFileToplevels,
       ImmutableMap<String, Object> bzlToplevels,
       ImmutableMap<String, Object> nativeRuleSpecificBindings,
       ImmutableMap<String, Object> workspaceBzlNativeBindings,
@@ -113,8 +110,7 @@ public BazelStarlarkEnvironment(
     this.workspaceBzlNativeBindings = workspaceBzlNativeBindings;
 
     this.uninjectedBuildBzlNativeBindings =
-        createUninjectedBuildBzlNativeBindings(
-            ruleFunctions, environmentExtensions, packageCallable);
+        createUninjectedBuildBzlNativeBindings(ruleFunctions, packageCallable, buildFileToplevels);
     this.uninjectedBuildBzlEnv =
         createUninjectedBuildBzlEnv(bzlToplevels, uninjectedBuildBzlNativeBindings);
     this.uninjectedWorkspaceBzlEnv =
@@ -131,7 +127,7 @@ public BazelStarlarkEnvironment(
             uninjectedBuildBzlNativeBindings,
             uninjectedBuildBzlEnv);
     this.uninjectedBuildEnv =
-        createUninjectedBuildEnv(ruleFunctions, environmentExtensions, packageCallable);
+        createUninjectedBuildEnv(ruleFunctions, packageCallable, buildFileToplevels);
   }
 
   /**
@@ -191,18 +187,13 @@ public ImmutableMap<String, Object> getBzlmodBzlEnv() {
    */
   private static ImmutableMap<String, Object> createUninjectedBuildBzlNativeBindings(
       Map<String, ?> ruleFunctions,
-      List<PackageFactory.EnvironmentExtension> environmentExtensions,
-      Object packageCallable) {
+      Object packageCallable,
+      Map<String, Object> buildFileToplevels) {
     ImmutableMap.Builder<String, Object> env = new ImmutableMap.Builder<>();
     env.putAll(StarlarkNativeModule.BINDINGS_FOR_BUILD_FILES);
     env.putAll(ruleFunctions);
     env.put("package", packageCallable);
-    // TODO(b/280446865): Remove ability to register BUILD toplevels in this way. They should
-    // instead go directly on the ConfiguredRuleClassProvider.Builder, e.g. an addBuildToplevel()
-    // method.
-    for (PackageFactory.EnvironmentExtension ext : environmentExtensions) {
-      ext.update(env);
-    }
+    env.putAll(buildFileToplevels);
     return env.buildOrThrow();
   }
 
@@ -227,16 +218,14 @@ private static ImmutableMap<String, Object> createUninjectedBuildBzlEnv(
 
   private static ImmutableMap<String, Object> createUninjectedBuildEnv(
       Map<String, ?> ruleFunctions,
-      List<PackageFactory.EnvironmentExtension> environmentExtensions,
-      Object packageCallable) {
+      Object packageCallable,
+      Map<String, Object> buildFileToplevels) {
     ImmutableMap.Builder<String, Object> env = ImmutableMap.builder();
     env.putAll(StarlarkLibrary.BUILD); // e.g. select, depset
     env.putAll(StarlarkNativeModule.BINDINGS_FOR_BUILD_FILES);
     env.put("package", packageCallable);
     env.putAll(ruleFunctions);
-    for (PackageFactory.EnvironmentExtension ext : environmentExtensions) {
-      ext.update(env);
-    }
+    env.putAll(buildFileToplevels);
     return env.buildOrThrow();
   }