Skip to content

Commit d067669

Browse files
quvalcopybara-github
quval
authored and
copybara-github
committed
Support execution constraints per exec group
When computing exec properties from the execution platform for an action, take into account only the properties that are relevant to the action's exec groups. In particular, allow setting exec properties for arbitrary exec groups on platforms. Previously, any such properties were rejected. With this change, the following becomes possible: ``` cc_test( name = "my_test", ..., exec_properties = { "test.key": "value", }, ) ``` This will apply `{"key": "value"}` for the test-runner action only (i.e., compilation and linkage won't be affected). The following also becomes possible: ``` platform( name = "test_platform", constraint_values = [":test_constraint"], exec_properties = { "test.key": "value", }, ) cc_test( name = "my_test", ..., exec_compatible_with = [":test_constraint"], ) ``` This achieves the same in a more succinct way. For related discussion, see PR #12719 by @ulfjack. Closes #13110. PiperOrigin-RevId: 361167318
1 parent 1113ec0 commit d067669

File tree

3 files changed

+273
-17
lines changed

3 files changed

+273
-17
lines changed

‎src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java

Lines changed: 52 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -515,7 +515,8 @@ public ActionOwner getActionOwner(String execGroup) {
515515
aspectDescriptors,
516516
getConfiguration(),
517517
getExecProperties(execGroup, execProperties),
518-
getExecutionPlatform(execGroup));
518+
getExecutionPlatform(execGroup),
519+
ImmutableSet.of(execGroup));
519520
actionOwners.put(execGroup, actionOwner);
520521
return actionOwner;
521522
}
@@ -622,12 +623,31 @@ public ImmutableList<Artifact> getBuildInfo(BuildInfoKey key) throws Interrupted
622623
AnalysisUtils.isStampingEnabled(this, getConfiguration()), key, getConfiguration());
623624
}
624625

626+
/**
627+
* Computes a map of exec properties given the execution platform, taking only properties in exec
628+
* groups that are applicable to this action. Properties for specific exec groups take precedence
629+
* over properties that don't specify an exec group.
630+
*/
625631
private static ImmutableMap<String, String> computeExecProperties(
626-
Map<String, String> targetExecProperties, @Nullable PlatformInfo executionPlatform) {
632+
Map<String, String> targetExecProperties,
633+
@Nullable PlatformInfo executionPlatform,
634+
Set<String> execGroups) {
627635
Map<String, String> execProperties = new HashMap<>();
628636

629637
if (executionPlatform != null) {
630-
execProperties.putAll(executionPlatform.execProperties());
638+
Map<String, Map<String, String>> execPropertiesPerGroup =
639+
parseExecGroups(executionPlatform.execProperties());
640+
641+
if (execPropertiesPerGroup.containsKey(DEFAULT_EXEC_GROUP_NAME)) {
642+
execProperties.putAll(execPropertiesPerGroup.get(DEFAULT_EXEC_GROUP_NAME));
643+
execPropertiesPerGroup.remove(DEFAULT_EXEC_GROUP_NAME);
644+
}
645+
646+
for (Map.Entry<String, Map<String, String>> execGroup : execPropertiesPerGroup.entrySet()) {
647+
if (execGroups.contains(execGroup.getKey())) {
648+
execProperties.putAll(execGroup.getValue());
649+
}
650+
}
631651
}
632652

633653
// If the same key occurs both in the platform and in target-specific properties, the
@@ -643,7 +663,8 @@ public static ActionOwner createActionOwner(
643663
ImmutableList<AspectDescriptor> aspectDescriptors,
644664
BuildConfiguration configuration,
645665
Map<String, String> targetExecProperties,
646-
@Nullable PlatformInfo executionPlatform) {
666+
@Nullable PlatformInfo executionPlatform,
667+
Set<String> execGroups) {
647668
return ActionOwner.create(
648669
rule.getLabel(),
649670
aspectDescriptors,
@@ -653,7 +674,7 @@ public static ActionOwner createActionOwner(
653674
configuration.checksum(),
654675
configuration.toBuildEvent(),
655676
configuration.isHostConfiguration() ? HOST_CONFIGURATION_PROGRESS_TAG : null,
656-
computeExecProperties(targetExecProperties, executionPlatform),
677+
computeExecProperties(targetExecProperties, executionPlatform, execGroups),
657678
executionPlatform);
658679
}
659680

@@ -1389,20 +1410,18 @@ private ImmutableMap<String, ImmutableMap<String, String>> parseExecProperties(
13891410
return ImmutableMap.of(DEFAULT_EXEC_GROUP_NAME, ImmutableMap.of());
13901411
} else {
13911412
return parseExecProperties(
1392-
execProperties,
1393-
toolchainContexts == null ? ImmutableSet.of() : toolchainContexts.getExecGroups());
1413+
execProperties, toolchainContexts == null ? null : toolchainContexts.getExecGroups());
13941414
}
13951415
}
13961416

13971417
/**
13981418
* Parse raw exec properties attribute value into a map of exec group names to their properties.
13991419
* The raw map can have keys of two forms: (1) 'property' and (2) 'exec_group_name.property'. The
1400-
* former get parsed into the target's default exec group, the latter get parsed into their
1401-
* relevant exec groups.
1420+
* former get parsed into the default exec group, the latter get parsed into their relevant exec
1421+
* groups.
14021422
*/
1403-
private static ImmutableMap<String, ImmutableMap<String, String>> parseExecProperties(
1404-
Map<String, String> rawExecProperties, Set<String> execGroups)
1405-
throws InvalidExecGroupException {
1423+
private static Map<String, Map<String, String>> parseExecGroups(
1424+
Map<String, String> rawExecProperties) {
14061425
Map<String, Map<String, String>> consolidatedProperties = new HashMap<>();
14071426
consolidatedProperties.put(DEFAULT_EXEC_GROUP_NAME, new HashMap<>());
14081427
for (Map.Entry<String, String> execProperty : rawExecProperties.entrySet()) {
@@ -1415,14 +1434,30 @@ private static ImmutableMap<String, ImmutableMap<String, String>> parseExecPrope
14151434
} else {
14161435
String execGroup = rawProperty.substring(0, delimiterIndex);
14171436
String property = rawProperty.substring(delimiterIndex + 1);
1418-
if (!execGroups.contains(execGroup)) {
1437+
consolidatedProperties.putIfAbsent(execGroup, new HashMap<>());
1438+
consolidatedProperties.get(execGroup).put(property, execProperty.getValue());
1439+
}
1440+
}
1441+
return consolidatedProperties;
1442+
}
1443+
1444+
/**
1445+
* Parse raw exec properties attribute value into a map of exec group names to their properties.
1446+
* If given a set of exec groups, validates all the exec groups in the map are applicable to the
1447+
* action.
1448+
*/
1449+
private static ImmutableMap<String, ImmutableMap<String, String>> parseExecProperties(
1450+
Map<String, String> rawExecProperties, @Nullable Set<String> execGroups)
1451+
throws InvalidExecGroupException {
1452+
Map<String, Map<String, String>> consolidatedProperties = parseExecGroups(rawExecProperties);
1453+
if (execGroups != null) {
1454+
for (Map.Entry<String, Map<String, String>> execGroup : consolidatedProperties.entrySet()) {
1455+
String execGroupName = execGroup.getKey();
1456+
if (!execGroupName.equals(DEFAULT_EXEC_GROUP_NAME) && !execGroups.contains(execGroupName)) {
14191457
throw new InvalidExecGroupException(
14201458
String.format(
1421-
"Tried to set exec property '%s' for non-existent exec group '%s'.",
1422-
property, execGroup));
1459+
"Tried to set properties for non-existent exec group '%s'.", execGroupName));
14231460
}
1424-
consolidatedProperties.putIfAbsent(execGroup, new HashMap<>());
1425-
consolidatedProperties.get(execGroup).put(property, execProperty.getValue());
14261461
}
14271462
}
14281463

‎src/test/java/com/google/devtools/build/lib/analysis/StarlarkExecGroupTest.java

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,10 @@ private void createToolchainsAndPlatforms() throws Exception {
105105
"platform(",
106106
" name = 'platform_2',",
107107
" constraint_values = [':constraint_2'],",
108+
" exec_properties = {",
109+
" 'watermelon.ripeness': 'unripe',",
110+
" 'watermelon.color': 'red',",
111+
" },",
108112
")");
109113

110114
useConfiguration(
@@ -391,4 +395,54 @@ public void testInheritsRuleRequirements() throws Exception {
391395
ImmutableSet.of(Label.parseAbsoluteUnchecked("//rule:toolchain_type_1")),
392396
ImmutableSet.of(Label.parseAbsoluteUnchecked("//platform:constraint_1"))));
393397
}
398+
399+
@Test
400+
public void testInheritsPlatformExecGroupExecProperty() throws Exception {
401+
createToolchainsAndPlatforms();
402+
writeRuleWithActionsAndWatermelonExecGroup();
403+
404+
scratch.file(
405+
"test/BUILD",
406+
"load('//test:defs.bzl', 'with_actions')",
407+
"with_actions(",
408+
" name = 'papaya',",
409+
" output = 'out.txt',",
410+
" watermelon_output = 'watermelon_out.txt',",
411+
")");
412+
413+
ConfiguredTarget target = getConfiguredTarget("//test:papaya");
414+
415+
assertThat(
416+
getGeneratingAction(target, "test/watermelon_out.txt").getOwner().getExecProperties())
417+
.containsExactly("ripeness", "unripe", "color", "red");
418+
assertThat(getGeneratingAction(target, "test/out.txt").getOwner().getExecProperties())
419+
.containsExactly();
420+
}
421+
422+
@Test
423+
public void testOverridePlatformExecGroupExecProperty() throws Exception {
424+
createToolchainsAndPlatforms();
425+
writeRuleWithActionsAndWatermelonExecGroup();
426+
427+
scratch.file(
428+
"test/BUILD",
429+
"load('//test:defs.bzl', 'with_actions')",
430+
"with_actions(",
431+
" name = 'papaya',",
432+
" output = 'out.txt',",
433+
" watermelon_output = 'watermelon_out.txt',",
434+
" exec_properties = {",
435+
" 'watermelon.ripeness': 'ripe',",
436+
" 'ripeness': 'unknown',",
437+
" },",
438+
")");
439+
440+
ConfiguredTarget target = getConfiguredTarget("//test:papaya");
441+
442+
assertThat(
443+
getGeneratingAction(target, "test/watermelon_out.txt").getOwner().getExecProperties())
444+
.containsExactly("ripeness", "ripe", "color", "red");
445+
assertThat(getGeneratingAction(target, "test/out.txt").getOwner().getExecProperties())
446+
.containsExactly("ripeness", "unknown");
447+
}
394448
}

‎src/test/shell/bazel/platforms_test.sh

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,5 +299,172 @@ EOF
299299
grep "child_value" out.txt || fail "Did not find the overriding value"
300300
}
301301

302+
function test_platform_execgroup_properties_cc_test() {
303+
cat > a.cc <<'EOF'
304+
int main() {}
305+
EOF
306+
cat > BUILD <<'EOF'
307+
cc_test(
308+
name = "a",
309+
srcs = ["a.cc"],
310+
)
311+
312+
platform(
313+
name = "my_platform",
314+
parents = ["@local_config_platform//:host"],
315+
exec_properties = {
316+
"platform_key": "default_value",
317+
"test.platform_key": "test_value",
318+
}
319+
)
320+
EOF
321+
bazel build --extra_execution_platforms=":my_platform" :a --execution_log_json_file out.txt || fail "Build failed"
322+
grep "platform_key" out.txt || fail "Did not find the platform key"
323+
grep "default_value" out.txt || fail "Did not find the default value"
324+
grep "test_value" out.txt && fail "Used the test-action value when not testing"
325+
326+
bazel test --extra_execution_platforms=":my_platform" :a --execution_log_json_file out.txt || fail "Test failed"
327+
grep "platform_key" out.txt || fail "Did not find the platform key"
328+
grep "test_value" out.txt || fail "Did not find the test-action value"
329+
}
330+
331+
function test_platform_execgroup_properties_nongroup_override_cc_test() {
332+
cat > a.cc <<'EOF'
333+
int main() {}
334+
EOF
335+
cat > BUILD <<'EOF'
336+
cc_test(
337+
name = "a",
338+
srcs = ["a.cc"],
339+
exec_properties = {
340+
"platform_key": "override_value",
341+
},
342+
)
343+
344+
platform(
345+
name = "my_platform",
346+
parents = ["@local_config_platform//:host"],
347+
exec_properties = {
348+
"platform_key": "default_value",
349+
"test.platform_key": "test_value",
350+
}
351+
)
352+
EOF
353+
bazel build --extra_execution_platforms=":my_platform" :a --execution_log_json_file out.txt || fail "Build failed"
354+
grep "platform_key" out.txt || fail "Did not find the platform key"
355+
grep "override_value" out.txt || fail "Did not find the overriding value"
356+
grep "default_value" out.txt && fail "Used the default value"
357+
358+
bazel test --extra_execution_platforms=":my_platform" :a --execution_log_json_file out.txt || fail "Test failed"
359+
grep "platform_key" out.txt || fail "Did not find the platform key"
360+
grep "override_value" out.txt || fail "Did not find the overriding value"
361+
}
362+
363+
function test_platform_execgroup_properties_group_override_cc_test() {
364+
cat > a.cc <<'EOF'
365+
int main() {}
366+
EOF
367+
cat > BUILD <<'EOF'
368+
cc_test(
369+
name = "a",
370+
srcs = ["a.cc"],
371+
exec_properties = {
372+
"test.platform_key": "test_override",
373+
},
374+
)
375+
376+
platform(
377+
name = "my_platform",
378+
parents = ["@local_config_platform//:host"],
379+
exec_properties = {
380+
"platform_key": "default_value",
381+
"test.platform_key": "test_value",
382+
}
383+
)
384+
EOF
385+
bazel build --extra_execution_platforms=":my_platform" :a --execution_log_json_file out.txt || fail "Build failed"
386+
grep "platform_key" out.txt || fail "Did not find the platform key"
387+
grep "default_value" out.txt || fail "Used the default value"
388+
389+
bazel test --extra_execution_platforms=":my_platform" :a --execution_log_json_file out.txt || fail "Test failed"
390+
grep "platform_key" out.txt || fail "Did not find the platform key"
391+
grep "test_override" out.txt || fail "Did not find the overriding test-action value"
392+
}
393+
394+
function test_platform_execgroup_properties_override_group_and_default_cc_test() {
395+
cat > a.cc <<'EOF'
396+
int main() {}
397+
EOF
398+
cat > BUILD <<'EOF'
399+
cc_test(
400+
name = "a",
401+
srcs = ["a.cc"],
402+
exec_properties = {
403+
"platform_key": "override_value",
404+
"test.platform_key": "test_override",
405+
},
406+
)
407+
408+
platform(
409+
name = "my_platform",
410+
parents = ["@local_config_platform//:host"],
411+
exec_properties = {
412+
"platform_key": "default_value",
413+
"test.platform_key": "test_value",
414+
}
415+
)
416+
EOF
417+
bazel build --extra_execution_platforms=":my_platform" :a --execution_log_json_file out.txt || fail "Build failed"
418+
grep "platform_key" out.txt || fail "Did not find the platform key"
419+
grep "override_value" out.txt || fail "Did not find the overriding value"
420+
grep "default_value" out.txt && fail "Used the default value"
421+
422+
bazel test --extra_execution_platforms=":my_platform" :a --execution_log_json_file out.txt || fail "Test failed"
423+
grep "platform_key" out.txt || fail "Did not find the platform key"
424+
grep "test_override" out.txt || fail "Did not find the overriding test-action value"
425+
}
426+
427+
function test_platform_properties_only_applied_for_relevant_execgroups_cc_test() {
428+
cat > a.cc <<'EOF'
429+
int main() {}
430+
EOF
431+
cat > BUILD <<'EOF'
432+
cc_test(
433+
name = "a",
434+
srcs = ["a.cc"],
435+
)
436+
437+
platform(
438+
name = "my_platform",
439+
parents = ["@local_config_platform//:host"],
440+
exec_properties = {
441+
"platform_key": "default_value",
442+
"unknown.platform_key": "unknown_value",
443+
}
444+
)
445+
EOF
446+
bazel test --extra_execution_platforms=":my_platform" :a --execution_log_json_file out.txt || fail "Build failed"
447+
grep "platform_key" out.txt || fail "Did not find the platform key"
448+
grep "default_value" out.txt || fail "Did not find the default value"
449+
}
450+
451+
function test_cannot_set_properties_for_irrelevant_execgroup_on_target_cc_test() {
452+
cat > a.cc <<'EOF'
453+
int main() {}
454+
EOF
455+
cat > BUILD <<'EOF'
456+
cc_test(
457+
name = "a",
458+
srcs = ["a.cc"],
459+
exec_properties = {
460+
"platform_key": "default_value",
461+
"unknown.platform_key": "unknown_value",
462+
}
463+
)
464+
EOF
465+
bazel test :a &> $TEST_log && fail "Build passed when we expected an error"
466+
grep "Tried to set properties for non-existent exec group" $TEST_log || fail "Did not complain about unknown exec group"
467+
}
468+
302469
run_suite "platform mapping test"
303470

0 commit comments

Comments
 (0)