Skip to content

Commit 762c470

Browse files
WyveraldWyverald
authored
[6.3.0] Fix absolute file paths showing up in lockfiles (#18993)
* Redact absolute root module file path in `MODULE.bazel.lock` The absolute path of the root module file in Starlark `Location`s is replaced with a constant when writing to the lock file. Work towards #18936 Closes #18949. PiperOrigin-RevId: 549118781 Change-Id: Ie689f7b5edf92296772c605845d694d872074214 * Use a label as the location of the MODULE.bazel file of non-registry overrides Fixes #18936 PiperOrigin-RevId: 549310245 Change-Id: I852570fbc81c1592c2fc0b3848d4b58e1c9ffb7d
1 parent b13091a commit 762c470

File tree

6 files changed

+446
-29
lines changed

6 files changed

+446
-29
lines changed

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
package com.google.devtools.build.lib.bazel.bzlmod;
1717

18-
import static com.google.devtools.build.lib.bazel.bzlmod.GsonTypeAdapterUtil.LOCKFILE_GSON;
1918
import static java.nio.charset.StandardCharsets.UTF_8;
2019

2120
import com.google.common.collect.ImmutableList;
@@ -93,7 +92,13 @@ public static BazelLockFileValue getLockfileValue(RootedPath lockfilePath) throw
9392
BazelLockFileValue bazelLockFileValue;
9493
try {
9594
String json = FileSystemUtils.readContent(lockfilePath.asPath(), UTF_8);
96-
bazelLockFileValue = LOCKFILE_GSON.fromJson(json, BazelLockFileValue.class);
95+
bazelLockFileValue =
96+
GsonTypeAdapterUtil.createLockFileGson(
97+
lockfilePath
98+
.asPath()
99+
.getParentDirectory()
100+
.getRelative(LabelConstants.MODULE_DOT_BAZEL_FILE_NAME))
101+
.fromJson(json, BazelLockFileValue.class);
97102
} catch (FileNotFoundException e) {
98103
bazelLockFileValue = EMPTY_LOCKFILE;
99104
}

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414

1515
package com.google.devtools.build.lib.bazel.bzlmod;
1616

17-
import static com.google.devtools.build.lib.bazel.bzlmod.GsonTypeAdapterUtil.LOCKFILE_GSON;
1817
import static java.nio.charset.StandardCharsets.UTF_8;
1918

2019
import com.google.common.collect.ImmutableMap;
@@ -136,7 +135,14 @@ private ImmutableMap<ModuleExtensionId, LockFileModuleExtension> combineModuleEx
136135
public static void updateLockfile(RootedPath lockfilePath, BazelLockFileValue updatedLockfile) {
137136
try {
138137
FileSystemUtils.writeContent(
139-
lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(updatedLockfile));
138+
lockfilePath.asPath(),
139+
UTF_8,
140+
GsonTypeAdapterUtil.createLockFileGson(
141+
lockfilePath
142+
.asPath()
143+
.getParentDirectory()
144+
.getRelative(LabelConstants.MODULE_DOT_BAZEL_FILE_NAME))
145+
.toJson(updatedLockfile));
140146
} catch (IOException e) {
141147
logger.atSevere().withCause(e).log(
142148
"Error while updating MODULE.bazel.lock file: %s", e.getMessage());

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java

Lines changed: 99 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@
2020
import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_MAP;
2121
import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_SET;
2222

23+
import com.google.auto.value.AutoValue;
2324
import com.google.common.base.Preconditions;
2425
import com.google.common.base.Splitter;
2526
import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException;
2627
import com.google.devtools.build.lib.cmdline.Label;
2728
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
29+
import com.google.devtools.build.lib.vfs.Path;
2830
import com.google.gson.Gson;
2931
import com.google.gson.GsonBuilder;
3032
import com.google.gson.JsonParseException;
@@ -42,6 +44,7 @@
4244
import java.util.List;
4345
import java.util.Optional;
4446
import javax.annotation.Nullable;
47+
import net.starlark.java.syntax.Location;
4548

4649
/**
4750
* Utility class to hold type adapters and helper methods to get gson registered with type adapters
@@ -188,24 +191,102 @@ public Optional<T> read(JsonReader jsonReader) throws IOException {
188191
}
189192
}
190193

191-
public static final Gson LOCKFILE_GSON =
192-
new GsonBuilder()
193-
.setPrettyPrinting()
194-
.disableHtmlEscaping()
195-
.enableComplexMapKeySerialization()
196-
.registerTypeAdapterFactory(GenerateTypeAdapter.FACTORY)
197-
.registerTypeAdapterFactory(DICT)
198-
.registerTypeAdapterFactory(IMMUTABLE_MAP)
199-
.registerTypeAdapterFactory(IMMUTABLE_LIST)
200-
.registerTypeAdapterFactory(IMMUTABLE_BIMAP)
201-
.registerTypeAdapterFactory(IMMUTABLE_SET)
202-
.registerTypeAdapterFactory(OPTIONAL)
203-
.registerTypeAdapter(Version.class, VERSION_TYPE_ADAPTER)
204-
.registerTypeAdapter(ModuleKey.class, MODULE_KEY_TYPE_ADAPTER)
205-
.registerTypeAdapter(ModuleExtensionId.class, MODULE_EXTENSION_ID_TYPE_ADAPTER)
206-
.registerTypeAdapter(AttributeValues.class, new AttributeValuesAdapter())
207-
.registerTypeAdapter(byte[].class, BYTE_ARRAY_TYPE_ADAPTER)
208-
.create();
194+
/**
195+
* A variant of {@link Location} that converts the absolute path to the root module file to a
196+
* constant and back.
197+
*/
198+
// protected only for @AutoValue
199+
@GenerateTypeAdapter
200+
@AutoValue
201+
protected abstract static class RootModuleFileEscapingLocation {
202+
// This marker string is neither a valid absolute path nor a valid URL and thus cannot conflict
203+
// with any real module file location.
204+
private static final String ROOT_MODULE_FILE_LABEL = "@@//:MODULE.bazel";
205+
206+
public abstract String file();
207+
208+
public abstract int line();
209+
210+
public abstract int column();
211+
212+
public Location toLocation(String moduleFilePath) {
213+
String file;
214+
if (file().equals(ROOT_MODULE_FILE_LABEL)) {
215+
file = moduleFilePath;
216+
} else {
217+
file = file();
218+
}
219+
return Location.fromFileLineColumn(file, line(), column());
220+
}
221+
222+
public static RootModuleFileEscapingLocation fromLocation(
223+
Location location, String moduleFilePath) {
224+
String file;
225+
if (location.file().equals(moduleFilePath)) {
226+
file = ROOT_MODULE_FILE_LABEL;
227+
} else {
228+
file = location.file();
229+
}
230+
return new AutoValue_GsonTypeAdapterUtil_RootModuleFileEscapingLocation(
231+
file, location.line(), location.column());
232+
}
233+
}
234+
235+
private static final class LocationTypeAdapterFactory implements TypeAdapterFactory {
236+
237+
private final String moduleFilePath;
238+
239+
public LocationTypeAdapterFactory(Path moduleFilePath) {
240+
this.moduleFilePath = moduleFilePath.getPathString();
241+
}
242+
243+
@Nullable
244+
@Override
245+
@SuppressWarnings("unchecked")
246+
public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {
247+
if (typeToken.getRawType() != Location.class) {
248+
return null;
249+
}
250+
TypeAdapter<RootModuleFileEscapingLocation> relativizedLocationTypeAdapter =
251+
gson.getAdapter(RootModuleFileEscapingLocation.class);
252+
return (TypeAdapter<T>)
253+
new TypeAdapter<Location>() {
254+
255+
@Override
256+
public void write(JsonWriter jsonWriter, Location location) throws IOException {
257+
relativizedLocationTypeAdapter.write(
258+
jsonWriter,
259+
RootModuleFileEscapingLocation.fromLocation(location, moduleFilePath));
260+
}
261+
262+
@Override
263+
public Location read(JsonReader jsonReader) throws IOException {
264+
return relativizedLocationTypeAdapter.read(jsonReader).toLocation(moduleFilePath);
265+
}
266+
};
267+
}
268+
}
269+
270+
public static Gson createLockFileGson(Path moduleFilePath) {
271+
return new GsonBuilder()
272+
.setPrettyPrinting()
273+
.disableHtmlEscaping()
274+
.enableComplexMapKeySerialization()
275+
.registerTypeAdapterFactory(GenerateTypeAdapter.FACTORY)
276+
.registerTypeAdapterFactory(DICT)
277+
.registerTypeAdapterFactory(IMMUTABLE_MAP)
278+
.registerTypeAdapterFactory(IMMUTABLE_LIST)
279+
.registerTypeAdapterFactory(IMMUTABLE_BIMAP)
280+
.registerTypeAdapterFactory(IMMUTABLE_SET)
281+
.registerTypeAdapterFactory(OPTIONAL)
282+
.registerTypeAdapterFactory(new LocationTypeAdapterFactory(moduleFilePath))
283+
.registerTypeAdapter(Version.class, VERSION_TYPE_ADAPTER)
284+
.registerTypeAdapter(ModuleKey.class, MODULE_KEY_TYPE_ADAPTER)
285+
.registerTypeAdapter(ModuleExtensionId.class, MODULE_EXTENSION_ID_TYPE_ADAPTER)
286+
.registerTypeAdapter(AttributeValues.class, new AttributeValuesAdapter())
287+
.registerTypeAdapter(byte[].class, BYTE_ARRAY_TYPE_ADAPTER)
288+
.create();
289+
}
209290

210291
private GsonTypeAdapterUtil() {}
211292
}

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@
2424
import com.google.devtools.build.lib.actions.FileValue;
2525
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.NonRootModuleFileValue;
2626
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
27+
import com.google.devtools.build.lib.cmdline.Label;
2728
import com.google.devtools.build.lib.cmdline.LabelConstants;
29+
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
2830
import com.google.devtools.build.lib.cmdline.RepositoryName;
2931
import com.google.devtools.build.lib.events.Event;
3032
import com.google.devtools.build.lib.packages.StarlarkExportable;
@@ -37,6 +39,7 @@
3739
import com.google.devtools.build.lib.util.Fingerprint;
3840
import com.google.devtools.build.lib.vfs.FileSystemUtils;
3941
import com.google.devtools.build.lib.vfs.Path;
42+
import com.google.devtools.build.lib.vfs.PathFragment;
4043
import com.google.devtools.build.lib.vfs.Root;
4144
import com.google.devtools.build.lib.vfs.RootedPath;
4245
import com.google.devtools.build.skyframe.SkyFunction;
@@ -199,11 +202,11 @@ private SkyValue computeForRootModule(StarlarkSemantics starlarkSemantics, Envir
199202
if (env.getValue(FileValue.key(moduleFilePath)) == null) {
200203
return null;
201204
}
202-
ModuleFile moduleFile = readModuleFile(moduleFilePath.asPath());
203-
String moduleFileHash = new Fingerprint().addBytes(moduleFile.getContent()).hexDigestAndReset();
205+
byte[] moduleFileContents = readModuleFile(moduleFilePath.asPath());
206+
String moduleFileHash = new Fingerprint().addBytes(moduleFileContents).hexDigestAndReset();
204207
ModuleFileGlobals moduleFileGlobals =
205208
execModuleFile(
206-
moduleFile,
209+
ModuleFile.create(moduleFileContents, moduleFilePath.asPath().toString()),
207210
/* registry= */ null,
208211
ModuleKey.ROOT,
209212
/* ignoreDevDeps= */ Objects.requireNonNull(IGNORE_DEV_DEPS.get(env)),
@@ -335,8 +338,15 @@ private GetModuleFileResult getModuleFile(
335338
if (env.getValue(FileValue.key(moduleFilePath)) == null) {
336339
return null;
337340
}
341+
Label moduleFileLabel =
342+
Label.createUnvalidated(
343+
PackageIdentifier.create(key.getCanonicalRepoName(), PathFragment.EMPTY_FRAGMENT),
344+
LabelConstants.MODULE_DOT_BAZEL_FILE_NAME.getBaseName());
338345
GetModuleFileResult result = new GetModuleFileResult();
339-
result.moduleFile = readModuleFile(moduleFilePath.asPath());
346+
result.moduleFile =
347+
ModuleFile.create(
348+
readModuleFile(moduleFilePath.asPath()),
349+
moduleFileLabel.getUnambiguousCanonicalForm());
340350
return result;
341351
}
342352

@@ -392,10 +402,9 @@ private GetModuleFileResult getModuleFile(
392402
throw errorf(Code.MODULE_NOT_FOUND, "module not found in registries: %s", key);
393403
}
394404

395-
private static ModuleFile readModuleFile(Path path) throws ModuleFileFunctionException {
405+
private static byte[] readModuleFile(Path path) throws ModuleFileFunctionException {
396406
try {
397-
return ModuleFile.create(
398-
FileSystemUtils.readWithKnownFileSize(path, path.getFileSize()), path.getPathString());
407+
return FileSystemUtils.readWithKnownFileSize(path, path.getFileSize());
399408
} catch (IOException e) {
400409
throw errorf(Code.MODULE_NOT_FOUND, "MODULE.bazel expected but not found at %s", path);
401410
}

‎src/test/py/bazel/bzlmod/bazel_lockfile_test.py

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -505,6 +505,70 @@ def testRemoveModuleExtensionsNotUsed(self):
505505
lockfile = json.loads(f.read().strip())
506506
self.assertEqual(len(lockfile['moduleExtensions']), 0)
507507

508+
def testNoAbsoluteRootModuleFilePath(self):
509+
self.ScratchFile(
510+
'MODULE.bazel',
511+
[
512+
'ext = use_extension("extension.bzl", "ext")',
513+
'ext.dep(generate = True)',
514+
'use_repo(ext, ext_hello = "hello")',
515+
'other_ext = use_extension("extension.bzl", "other_ext")',
516+
'other_ext.dep(generate = False)',
517+
'use_repo(other_ext, other_ext_hello = "hello")',
518+
],
519+
)
520+
self.ScratchFile('BUILD.bazel')
521+
self.ScratchFile(
522+
'extension.bzl',
523+
[
524+
'def _repo_rule_impl(ctx):',
525+
' ctx.file("WORKSPACE")',
526+
' ctx.file("BUILD", "filegroup(name=\'lala\')")',
527+
'',
528+
'repo_rule = repository_rule(implementation=_repo_rule_impl)',
529+
'',
530+
'def _module_ext_impl(ctx):',
531+
' for mod in ctx.modules:',
532+
' for dep in mod.tags.dep:',
533+
' if dep.generate:',
534+
' repo_rule(name="hello")',
535+
'',
536+
'_dep = tag_class(attrs={"generate": attr.bool()})',
537+
'ext = module_extension(',
538+
' implementation=_module_ext_impl,',
539+
' tag_classes={"dep": _dep},',
540+
')',
541+
'other_ext = module_extension(',
542+
' implementation=_module_ext_impl,',
543+
' tag_classes={"dep": _dep},',
544+
')',
545+
],
546+
)
547+
548+
# Paths to module files in error message always use forward slashes as
549+
# separators, even on Windows.
550+
module_file_path = self.Path('MODULE.bazel').replace('\\', '/')
551+
552+
self.RunBazel(['build', '--nobuild', '@ext_hello//:all'])
553+
with open(self.Path('MODULE.bazel.lock'), 'r') as f:
554+
self.assertNotIn(module_file_path, f.read())
555+
556+
self.RunBazel(['shutdown'])
557+
exit_code, _, stderr = self.RunBazel(
558+
['build', '--nobuild', '@other_ext_hello//:all'], allow_failure=True
559+
)
560+
self.AssertNotExitCode(exit_code, 0, stderr)
561+
self.assertIn(
562+
(
563+
'ERROR: module extension "other_ext" from "//:extension.bzl" does '
564+
'not generate repository "hello", yet it is imported as '
565+
'"other_ext_hello" in the usage at '
566+
+ module_file_path
567+
+ ':4:26'
568+
),
569+
stderr,
570+
)
571+
508572

509573
if __name__ == '__main__':
510574
unittest.main()

0 commit comments

Comments
 (0)