Add support for .ar archives (and .deb files)

This implements #[15130](bazelbuild#15130).

As I was updating the docs for .ar and .deb formats, I also addressed some previous formats that had been added but not propagated through to all the documentation places.

Closes bazelbuild#15132.

PiperOrigin-RevId: 439569440

Author: kjlubick

src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java

@@ -0,0 +1,81 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.io.ByteStreams;
+import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
+import org.apache.commons.compress.archivers.ar.ArArchiveInputStream;
+
+/**
+ * Opens a .ar archive file. It ignores the prefix setting because these archives cannot contain
+ * directories.
+ */
+public class ArFunction implements Decompressor {
+
+  public static final Decompressor INSTANCE = new ArFunction();
+
+  // This is the same value as picked for .tar files, which appears to have worked well.
+  private static final int BUFFER_SIZE = 32 * 1024;
+
+  private InputStream getDecompressorStream(DecompressorDescriptor descriptor) throws IOException {
+    return new BufferedInputStream(
+        new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE);
+  }
+  ;
+
+  @Override
+  public Path decompress(DecompressorDescriptor descriptor)
+      throws InterruptedException, IOException {
+    if (Thread.interrupted()) {
+      throw new InterruptedException();
+    }
+
+    try (InputStream decompressorStream = getDecompressorStream(descriptor)) {
+      ArArchiveInputStream arStream = new ArArchiveInputStream(decompressorStream);
+      ArArchiveEntry entry;
+      while ((entry = arStream.getNextArEntry()) != null) {
+        Path filePath = descriptor.repositoryPath().getRelative(entry.getName());
+        filePath.getParentDirectory().createDirectoryAndParents();
+        if (entry.isDirectory()) {
+          // ar archives don't contain any directory information, so this should never
+          // happen
+          continue;
+        } else {
+          // We do not have to worry about symlinks in .ar files - it's not supported
+          // by the .ar file format.
+          try (OutputStream out = filePath.getOutputStream()) {
+            ByteStreams.copy(arStream, out);
+          }
+          filePath.chmod(entry.getMode());
+          // entry.getLastModified() appears to be in seconds, so we need to convert
+          // it into milliseconds for setLastModifiedTime
+          filePath.setLastModifiedTime(entry.getLastModified() * 1000L);
+        }
+        if (Thread.interrupted()) {
+          throw new InterruptedException();
+        }
+      }
+    }
+
+    return descriptor.repositoryPath();
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java

@@ -107,11 +107,13 @@ static Decompressor getDecompressor(Path archivePath)
       return TarZstFunction.INSTANCE;
     } else if (baseName.endsWith(".tar.bz2")) {
       return TarBz2Function.INSTANCE;
+    } else if (baseName.endsWith(".ar") || baseName.endsWith(".deb")) {
+      return ArFunction.INSTANCE;
     } else {
       throw new RepositoryFunctionException(
           Starlark.errorf(
               "Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz,"
-                  + " .tar.zst, .tzst, or .tar.bz2 suffix (got %s)",
+                  + " .tar.zst, .tzst, .tar.bz2, .ar or .deb suffix (got %s)",
               archivePath),
           Transience.PERSISTENT);
     }

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java

@@ -825,7 +825,8 @@ public void extract(Object archive, Object output, String stripPrefix, StarlarkT
                     + " By default, the archive type is determined from the file extension of"
                     + " the URL."
                     + " If the file has no extension, you can explicitly specify either \"zip\","
-                    + " \"jar\", \"war\", \"aar\", \"tar.gz\", \"tgz\", \"tar.bz2\", or \"tar.xz\""
+                    + " \"jar\", \"war\", \"aar\", \"tar\", \"tar.gz\", \"tgz\", \"tar.xz\","
+                    + " \"txz\", \".tar.zst\", \".tzst\", \"tar.bz2\", \".ar\", or \".deb\""
                     + " here."),
         @Param(
             name = "stripPrefix",

src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java

@@ -0,0 +1,88 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.devtools.build.lib.testutil.TestConstants;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.unix.UnixFileSystem;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.DigestHashFunction;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.runfiles.Runfiles;
+import java.io.File;
+import java.io.IOException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests decompressing archives. */
+@RunWith(JUnit4.class)
+public class ArFunctionTest {
+  /*
+   * .ar archive created with ar cr test_files.ar archived_first.txt archived_second.md
+   * The files contain short UTF-8 encoded strings.
+   */
+  private static final String ARCHIVE_NAME = "test_files.ar";
+  private static final String PATH_TO_TEST_ARCHIVE =
+      "/com/google/devtools/build/lib/bazel/repository/";
+  private static final String FIRST_FILE_NAME = "archived_first.txt";
+  private static final String SECOND_FILE_NAME = "archived_second.md";
+
+  @Test
+  public void testDecompress() throws Exception {
+    Path outputDir = decompress(createDescriptorBuilder());
+
+    assertThat(outputDir.exists()).isTrue();
+    Path firstFile = outputDir.getRelative(FIRST_FILE_NAME);
+    assertThat(firstFile.exists()).isTrue();
+    // There are 20 bytes in the content "this is test file 1"
+    assertThat(firstFile.getFileSize()).isEqualTo(20);
+    assertThat(firstFile.isSymbolicLink()).isFalse();
+
+    Path secondFile = outputDir.getRelative(SECOND_FILE_NAME);
+    assertThat(secondFile.exists()).isTrue();
+    // There are 20 bytes in the content "this is the second test file"
+    assertThat(secondFile.getFileSize()).isEqualTo(29);
+    assertThat(secondFile.isSymbolicLink()).isFalse();
+  }
+
+  private Path decompress(DecompressorDescriptor.Builder descriptorBuilder) throws Exception {
+    descriptorBuilder.setDecompressor(ArFunction.INSTANCE);
+    return new ArFunction().decompress(descriptorBuilder.build());
+  }
+
+  private DecompressorDescriptor.Builder createDescriptorBuilder() throws IOException {
+    // This was cribbed from TestArchiveDescriptor
+    FileSystem testFS =
+        OS.getCurrent() == OS.WINDOWS
+            ? new JavaIoFileSystem(DigestHashFunction.SHA256)
+            : new UnixFileSystem(DigestHashFunction.SHA256, /*hashAttributeName=*/ "");
+
+    // do not rely on TestConstants.JAVATESTS_ROOT end with slash, but ensure separators
+    // are not duplicated
+    String path =
+        (TestConstants.JAVATESTS_ROOT + PATH_TO_TEST_ARCHIVE + ARCHIVE_NAME).replace("//", "/");
+    Path tarballPath = testFS.getPath(Runfiles.create().rlocation(path));
+
+    Path workingDir = testFS.getPath(new File(TestUtils.tmpDir()).getCanonicalPath());
+    Path outDir = workingDir.getRelative("out");
+
+    return DecompressorDescriptor.builder().setRepositoryPath(outDir).setArchivePath(tarballPath);
+  }
+}

src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD

@@ -27,6 +27,7 @@ java_library(
     data = [
         "test_decompress_archive.tar.gz",
         "test_decompress_archive.zip",
+        "test_files.ar",
     ],
     deps = [
         "//src/main/java/com/google/devtools/build/lib/bazel/repository",

src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java

@@ -56,6 +56,10 @@ public void testKnownFileExtensionsDoNotThrow() throws Exception {
     DecompressorDescriptor.builder().setArchivePath(path).build();
     path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.bz2");
     DecompressorDescriptor.builder().setArchivePath(path).build();
+    path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.ar");
+    DecompressorDescriptor.builder().setArchivePath(path).build();
+    path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.deb");
+    DecompressorDescriptor.builder().setArchivePath(path).build();
   }
 
   @Test

src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar

@@ -0,0 +1,9 @@
+!<arch>
+//                                              40        `
+archived_first.txt/
+archived_second.md/
+/0              0           0     0     644     20        `
+this is test file 1
+/20             0           0     0     644     29        `
+this is the second test file
+

tools/build_defs/repo/http.bzl

@@ -269,7 +269,7 @@ match a directory in the archive, Bazel will return an error.""",
 By default, the archive type is determined from the file extension of the
 URL. If the file has no extension, you can explicitly specify one of the
 following: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, `"tar.gz"`, `"tgz"`,
-`"tar.xz"`, or `tar.bz2`.""",
+`"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`, or `"deb"`.""",
     ),
     "patches": attr.label_list(
         default = [],
@@ -357,8 +357,9 @@ http_archive = repository_rule(
         """Downloads a Bazel repository as a compressed archive file, decompresses it,
 and makes its targets available for binding.
 
-It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`,
-`"tar"`, `"tar.gz"`, `"tgz"`, `"tar.xz"`, and `tar.bz2`.
+It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`,
+`"tar.gz"`, `"tgz"`, `"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`,
+or `"deb"`.
 
 Examples:
   Suppose the current repository contains the source code for a chat program,