Add option for ignoring whitespace

Sometimes, in the config generator, the files we are trying to compare to have been formatted to meet certain style guidelines. In these cases, ignoring whitespaces leads to more accurate results for the glob matching and core.move transform heuristics.

Adding this as a flag to allow for easy enable/disable behavior.

BUG=409065048
PiperOrigin-RevId: 744862693
Change-Id: I8493ed0e86e073ea1c602c4f3ff734db846b8da7

Author: chris-campos

java/com/google/copybara/configgen/ConfigGenHeuristics.java

@@ -67,6 +67,7 @@ public class ConfigGenHeuristics {
   private final ImmutableSet<Path> destinationOnlyPaths;
   private final int percentSimilar;
   private final boolean ignoreCarriageReturn;
+  private final boolean ignoreWhitespace;
 
   /**
    * Creates the Generator object
@@ -84,12 +85,14 @@ public ConfigGenHeuristics(
       Path destination,
       ImmutableSet<Path> destinationOnlyPaths,
       int percentSimilar,
-      boolean ignoreCarriageReturn) {
+      boolean ignoreCarriageReturn,
+      boolean ignoreWhitespace) {
     this.origin = checkNotNull(origin);
     this.destination = checkNotNull(destination);
     this.destinationOnlyPaths = checkNotNull(destinationOnlyPaths);
     this.percentSimilar = percentSimilar;
     this.ignoreCarriageReturn = ignoreCarriageReturn;
+    this.ignoreWhitespace = ignoreWhitespace;
   }
 
   /** Result of the config generation */
@@ -134,7 +137,12 @@ public ConfigGenHeuristics.Result run() throws IOException {
     ImmutableSet<Path> g3Files = listFiles(destination);
     SimilarityDetector similarityDetector =
         SimilarityDetector.create(
-            origin, gitFiles, destinationOnlyPaths, percentSimilar, ignoreCarriageReturn);
+            origin,
+            gitFiles,
+            destinationOnlyPaths,
+            percentSimilar,
+            ignoreCarriageReturn,
+            ignoreWhitespace);
     // Map of destination file paths to origin file paths with similarity score.
     Map<Path, PathAndScore> destinationToOriginMapping = new TreeMap<>();
 
@@ -499,9 +507,11 @@ private static SimilarityDetector create(
         ImmutableSet<Path> files,
         ImmutableSet<Path> destinationOnlyPaths,
         int percentSimilar,
-        boolean ignoreCarriageReturn)
+        boolean ignoreCarriageReturn,
+        boolean ignoreWhitespace)
         throws IOException {
-      RenameDetector<Path> similarLines = new RenameDetector<>(ignoreCarriageReturn);
+      RenameDetector<Path> similarLines =
+          new RenameDetector<>(ignoreCarriageReturn, ignoreWhitespace);
       HashMultimap<String, Path> hashes = HashMultimap.create(files.size(), 1);
       for (Path file : files) {
         byte[] bytes = Files.readAllBytes(parent.resolve(file));

java/com/google/copybara/onboard/ConfigHeuristicsInputProvider.java

@@ -59,18 +59,21 @@ public class ConfigHeuristicsInputProvider implements InputProvider {
 
   private final GitOptions gitOptions;
   private final GeneralOptions generalOptions;
+  private final GeneratorOptions generatorOptions;
   private final ImmutableSet<Path> destinationOnlyPaths;
   private final int percentSimilar;
   private final Console console;
 
   public ConfigHeuristicsInputProvider(
       GitOptions gitOptions,
       GeneralOptions generalOptions,
+      GeneratorOptions generatorOptions,
       ImmutableSet<Path> destinationOnlyPaths,
       int percentSimilar,
       Console console) {
     this.gitOptions = gitOptions;
     this.generalOptions = generalOptions;
+    this.generatorOptions = generatorOptions;
     this.destinationOnlyPaths = destinationOnlyPaths;
     this.percentSimilar = percentSimilar;
     this.console = console;
@@ -140,7 +143,8 @@ private Optional<Result> computeHeuristic(
               destination,
               destinationOnlyPaths,
               percentSimilar,
-              generalOptions.isTemporaryFeature("GENERATOR_IGNORE_CARRIAGE_RETURN", true));
+              generatorOptions.computeGlobIgnoreCarriageReturn,
+              generatorOptions.computeGlobIgnoreWhitespace);
 
       console.progressFmt("Computing globs");
       cached = Optional.of(heuristics.run());

java/com/google/copybara/onboard/GeneratorCmd.java

@@ -98,6 +98,7 @@ public ImmutableList<InputProvider> getInputProviders(CommandEnv commandEnv)
         new ConfigHeuristicsInputProvider(
             commandEnv.getOptions().get(GitOptions.class),
             commandEnv.getOptions().get(GeneralOptions.class),
+            commandEnv.getOptions().get(GeneratorOptions.class),
             ImmutableSet.of(),
             PERCENTAGE_SIMILAR,
             console),

java/com/google/copybara/onboard/GeneratorOptions.java

@@ -52,4 +52,19 @@ public class GeneratorOptions implements Option {
       names = "--new-package",
       description = "Whether or not files from this package exist in the destination.")
   public boolean newPackage;
+
+  @Parameter(
+      names = "--compute-glob-ignore-carriage-return",
+      description =
+          "Whether to ignore carriage return characters in file content comparisons during glob"
+              + " generation.",
+      arity = 1)
+  public boolean computeGlobIgnoreCarriageReturn = true;
+
+  @Parameter(
+      names = "--compute-glob-ignore-whitespace",
+      description =
+          "Whether to ignore whitespace in file content comparisons during glob generation.",
+      arity = 1)
+  public boolean computeGlobIgnoreWhitespace = true;
 }

java/com/google/copybara/util/RenameDetector.java

@@ -37,11 +37,13 @@
  */
 public final class RenameDetector<I> {
   private final boolean ignoreCarriageReturn;
+  private final boolean ignoreWhitespace;
 
   private final List<PriorFile<I>> priorFiles = new ArrayList<>();
 
-  public RenameDetector(boolean ignoreCarriageReturn) {
+  public RenameDetector(boolean ignoreCarriageReturn, boolean ignoreWhitespace) {
     this.ignoreCarriageReturn = ignoreCarriageReturn;
+    this.ignoreWhitespace = ignoreWhitespace;
   }
 
   private static final class PriorFile<I> {
@@ -63,10 +65,12 @@ private static final class HashingByteProcessor implements ByteProcessor<int[]>
 
     int hash;
     final boolean ignoreCarriageReturn;
+    final boolean ignoreWhitespace;
     final HashSet<Integer> hashes = new HashSet<>();
 
-    HashingByteProcessor(boolean ignoreCarriageReturn) {
+    HashingByteProcessor(boolean ignoreCarriageReturn, boolean ignoreWhitespace) {
       this.ignoreCarriageReturn = ignoreCarriageReturn;
+      this.ignoreWhitespace = ignoreWhitespace;
     }
 
     @Override
@@ -77,6 +81,9 @@ public boolean processBytes(byte[] buf, int off, int len) {
           // Skip carriage return in Windows-style line endings when hashing.
           continue;
         }
+        if (ignoreWhitespace && (b == ' ' || b == '\t')) {
+          continue;
+        }
 
         hash *= 31;
         hash += b;
@@ -102,7 +109,8 @@ public int[] getResult() {
    */
   private int[] hashes(InputStream input) throws IOException {
     try {
-      return ByteStreams.readBytes(input, new HashingByteProcessor(ignoreCarriageReturn));
+      return ByteStreams.readBytes(
+          input, new HashingByteProcessor(ignoreCarriageReturn, ignoreWhitespace));
     } finally {
       input.close();
     }

javatests/com/google/copybara/configgen/ConfigGenHeuristicsTest.java

@@ -73,7 +73,18 @@ public void testRename() throws IOException {
   public void testIgnoreCarriageReturn() throws IOException {
     writeFile(origin, "fileA", "aaa\n".repeat(20));
     writeFile(destination, "fileB", "aaa\r\n".repeat(20) + "CHANGED");
-    ConfigGenHeuristics.Result result = createHeuristics(true).run();
+    ConfigGenHeuristics.Result result =
+        createHeuristics(/* ignoreCarriageReturn= */ true, /* ignoreWhitespace= */ false).run();
+
+    globMatches(result.getOriginGlob(), "a/b/c/fileA");
+  }
+
+  @Test
+  public void testIgnoreWhitespace() throws IOException {
+    writeFile(origin, "fileA", "aaa                 \n".repeat(20));
+    writeFile(destination, "fileB", "aaa \n".repeat(20) + "CHANGED");
+    ConfigGenHeuristics.Result result =
+        createHeuristics(/* ignoreCarriageReturn= */ false, /* ignoreWhitespace= */ true).run();
 
     globMatches(result.getOriginGlob(), "a/b/c/fileA");
   }
@@ -251,11 +262,12 @@ private static void writeFile(Path basePath, String relativePath, String content
   }
 
   private ConfigGenHeuristics createHeuristics() {
-    return createHeuristics(false);
+    return createHeuristics(/* ignoreCarriageReturn= */ false, /* ignoreWhitespace= */ false);
   }
 
-  private ConfigGenHeuristics createHeuristics(boolean ignoreCarriageReturn) {
+  private ConfigGenHeuristics createHeuristics(
+      boolean ignoreCarriageReturn, boolean ignoreWhitespace) {
     return new ConfigGenHeuristics(
-        origin, destination, destinationOnlyPaths, 30, ignoreCarriageReturn);
+        origin, destination, destinationOnlyPaths, 30, ignoreCarriageReturn, ignoreWhitespace);
   }
 }

javatests/com/google/copybara/onboard/ConfigHeuristicsInputProviderTest.java

@@ -61,18 +61,19 @@ public class ConfigHeuristicsInputProviderTest {
   protected TestingConsole console;
   protected OptionsBuilder optionsBuilder;
   protected GeneralOptions generalOptions;
+  protected GeneratorOptions generatorOptions;
 
   @Before
   public void setup() throws Exception {
     console = new TestingConsole();
     repoGitDir = Files.createTempDirectory("GitDestinationTest-repoGitDir");
     workDir = Files.createTempDirectory("workdir");
     destination = Files.createTempDirectory("destination");
-    
     optionsBuilder = getOptionsBuilder(console);
     String unused = git("init", "--bare", repoGitDir.toString());
     generalOptions = optionsBuilder.general;
     gitOptions = optionsBuilder.git;
+    generatorOptions = optionsBuilder.generator;
     origin =
         GitRepository.newBareRepo(
             repoGitDir,
@@ -118,7 +119,7 @@ public <T> T resolve(Input<T> input) throws CannotProvideException {
 
     ConfigHeuristicsInputProvider inputProvider =
         new ConfigHeuristicsInputProvider(
-            gitOptions, generalOptions, ImmutableSet.of(), 30, console);
+            gitOptions, generalOptions, generatorOptions, ImmutableSet.of(), 30, console);
     Optional<Glob> glob = inputProvider.resolve(Inputs.ORIGIN_GLOB, resolver);
 
     // The result is an empty glob rather than glob(include = ["**"], exclude = ["**"])
@@ -155,7 +156,7 @@ public <T> T resolve(Input<T> input) throws CannotProvideException {
         };
     ConfigHeuristicsInputProvider inputProvider =
         new ConfigHeuristicsInputProvider(
-            gitOptions, generalOptions, ImmutableSet.of(), 30, console);
+            gitOptions, generalOptions, generatorOptions, ImmutableSet.of(), 30, console);
 
     DestinationExcludePaths paths =
         inputProvider.resolve(Inputs.DESTINATION_EXCLUDE_PATHS, resolver).get();
@@ -201,7 +202,7 @@ public <T> T resolve(Input<T> input) throws CannotProvideException {
 
     ConfigHeuristicsInputProvider inputProvider =
         new ConfigHeuristicsInputProvider(
-            gitOptions, generalOptions, ImmutableSet.of(), 30, console);
+            gitOptions, generalOptions, generatorOptions, ImmutableSet.of(), 30, console);
     Glob expectedGlob = Glob.createGlob(ImmutableList.of("**"), ImmutableList.of("bar.txt"));
     Optional<Glob> glob = inputProvider.resolve(Inputs.ORIGIN_GLOB, resolver);