Basics: add support for .tar.gz archives

Package.swift

@@ -554,8 +554,10 @@ let package = Package(
             name: "BasicsTests",
             dependencies: ["Basics", "SPMTestSupport", "tsan_utils"],
             exclude: [
-                "Inputs/archive.zip",
-                "Inputs/invalid_archive.zip",
+                "Archiver/Inputs/archive.tar.gz",
+                "Archiver/Inputs/archive.zip",
+                "Archiver/Inputs/invalid_archive.tar.gz",
+                "Archiver/Inputs/invalid_archive.zip",
             ]
         ),
         .testTarget(

Sources/Basics/Archiver+Tar.swift

@@ -0,0 +1,147 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import class Dispatch.DispatchQueue
+import struct Dispatch.DispatchTime
+import struct TSCBasic.AbsolutePath
+import protocol TSCBasic.FileSystem
+import struct TSCBasic.FileSystemError
+import class TSCBasic.Process
+
+/// An `Archiver` that handles Tar archives using the command-line `tar` tool.
+public struct TarArchiver: Archiver {
+    public let supportedExtensions: Set<String> = ["tar", "tar.gz"]
+
+    /// The file-system implementation used for various file-system operations and checks.
+    private let fileSystem: FileSystem
+
+    /// Helper for cancelling in-flight requests
+    private let cancellator: Cancellator
+
+    /// The underlying command
+    private let tarCommand: String
+
+    /// Creates a `TarArchiver`.
+    ///
+    /// - Parameters:
+    ///   - fileSystem: The file system to used by the `TarArchiver`.
+    ///   - cancellator: Cancellation handler
+    public init(fileSystem: FileSystem, cancellator: Cancellator? = .none) {
+        self.fileSystem = fileSystem
+        self.cancellator = cancellator ?? Cancellator(observabilityScope: .none)
+
+        #if os(Windows)
+        self.tarCommand = "tar.exe"
+        #else
+        self.tarCommand = "tar"
+        #endif
+    }
+
+    public func extract(
+        from archivePath: AbsolutePath,
+        to destinationPath: AbsolutePath,
+        completion: @escaping (Result<Void, Error>) -> Void
+    ) {
+        do {
+            guard self.fileSystem.exists(archivePath) else {
+                throw FileSystemError(.noEntry, archivePath)
+            }
+
+            guard self.fileSystem.isDirectory(destinationPath) else {
+                throw FileSystemError(.notDirectory, destinationPath)
+            }
+
+            let process = TSCBasic.Process(
+                arguments: [self.tarCommand, "zxf", archivePath.pathString, "-C", destinationPath.pathString]
+            )
+
+            guard let registrationKey = self.cancellator.register(process) else {
+                throw CancellationError.failedToRegisterProcess(process)
+            }
+
+            DispatchQueue.sharedConcurrent.async {
+                defer { self.cancellator.deregister(registrationKey) }
+                completion(.init(catching: {
+                    try process.launch()
+                    let processResult = try process.waitUntilExit()
+                    guard processResult.exitStatus == .terminated(code: 0) else {
+                        throw try StringError(processResult.utf8stderrOutput())
+                    }
+                }))
+            }
+        } catch {
+            return completion(.failure(error))
+        }
+    }
+
+    public func compress(
+        directory: AbsolutePath,
+        to destinationPath: AbsolutePath,
+        completion: @escaping (Result<Void, Error>) -> Void
+    ) {
+        do {
+            guard self.fileSystem.isDirectory(directory) else {
+                throw FileSystemError(.notDirectory, directory)
+            }
+
+            let process = TSCBasic.Process(
+                arguments: [self.tarCommand, "acf", destinationPath.pathString, directory.basename],
+                workingDirectory: directory.parentDirectory
+            )
+
+            guard let registrationKey = self.cancellator.register(process) else {
+                throw CancellationError.failedToRegisterProcess(process)
+            }
+
+            DispatchQueue.sharedConcurrent.async {
+                defer { self.cancellator.deregister(registrationKey) }
+                completion(.init(catching: {
+                    try process.launch()
+                    let processResult = try process.waitUntilExit()
+                    guard processResult.exitStatus == .terminated(code: 0) else {
+                        throw try StringError(processResult.utf8stderrOutput())
+                    }
+                }))
+            }
+        } catch {
+            return completion(.failure(error))
+        }
+    }
+
+    public func validate(path: AbsolutePath, completion: @escaping (Result<Bool, Error>) -> Void) {
+        do {
+            guard self.fileSystem.exists(path) else {
+                throw FileSystemError(.noEntry, path)
+            }
+
+            let process = TSCBasic.Process(arguments: [self.tarCommand, "tf", path.pathString])
+            guard let registrationKey = self.cancellator.register(process) else {
+                throw CancellationError.failedToRegisterProcess(process)
+            }
+
+            DispatchQueue.sharedConcurrent.async {
+                defer { self.cancellator.deregister(registrationKey) }
+                completion(.init(catching: {
+                    try process.launch()
+                    let processResult = try process.waitUntilExit()
+                    return processResult.exitStatus == .terminated(code: 0)
+                }))
+            }
+        } catch {
+            return completion(.failure(error))
+        }
+    }
+
+    public func cancel(deadline: DispatchTime) throws {
+        try self.cancellator.cancel(deadline: deadline)
+    }
+}

Sources/Basics/Archiver+Zip.swift

@@ -53,7 +53,7 @@ public struct ZipArchiver: Archiver, Cancellable {
             let process = TSCBasic.Process(arguments: ["unzip", archivePath.pathString, "-d", destinationPath.pathString])
 #endif
             guard let registrationKey = self.cancellator.register(process) else {
-                throw StringError("cancellation")
+                throw CancellationError.failedToRegisterProcess(process)
             }
 
             DispatchQueue.sharedConcurrent.async {
@@ -95,7 +95,7 @@ public struct ZipArchiver: Archiver, Cancellable {
 #endif
 
             guard let registrationKey = self.cancellator.register(process) else {
-                throw StringError("Failed to register cancellation for Archiver")
+                throw CancellationError.failedToRegisterProcess(process)
             }
 
             DispatchQueue.sharedConcurrent.async {
@@ -125,7 +125,7 @@ public struct ZipArchiver: Archiver, Cancellable {
             let process = TSCBasic.Process(arguments: ["unzip", "-t", path.pathString])
 #endif
             guard let registrationKey = self.cancellator.register(process) else {
-                throw StringError("cancellation")
+                throw CancellationError.failedToRegisterProcess(process)
             }
 
             DispatchQueue.sharedConcurrent.async {

Sources/Basics/CMakeLists.txt

@@ -8,6 +8,7 @@
 
 add_library(Basics
   Archiver.swift
+  Archiver+Tar.swift
   Archiver+Zip.swift
   AuthorizationProvider.swift
   ByteString+Extensions.swift

Sources/Basics/Cancellator.swift

@@ -112,9 +112,24 @@ public protocol Cancellable {
 }
 
 public struct CancellationError: Error, CustomStringConvertible {
-    public let description = "Operation cancelled"
+    public let description: String
 
-    public init() {}
+    public init() {
+        self.init(description: "Operation cancelled")
+    }
+
+    private init(description: String) {
+        self.description = description
+    }
+
+    static func failedToRegisterProcess(_ process: TSCBasic.Process) -> Self {
+        Self(description: """
+            failed to register a cancellation handler for this process invocation `\(
+                process.arguments.joined(separator: " ")
+            )`
+            """
+        )
+    }
 }
 
 extension TSCBasic.Process {

Tests/BasicsTests/Inputs/invalid_archive.tar.gz

@@ -0,0 +1 @@
+not an archive

Tests/BasicsTests/TarArchiverTests.swift

@@ -0,0 +1,159 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Basics
+import TSCBasic
+import TSCclibc // for SPM_posix_spawn_file_actions_addchdir_np_supported
+import TSCTestSupport
+import XCTest
+
+final class TarArchiverTests: XCTestCase {
+    func testSuccess() throws {
+        try testWithTemporaryDirectory { tmpdir in
+            let archiver = TarArchiver(fileSystem: localFileSystem)
+            let inputArchivePath = AbsolutePath(path: #file).parentDirectory
+                .appending(components: "Inputs", "archive.tar.gz")
+            try archiver.extract(from: inputArchivePath, to: tmpdir)
+            let content = tmpdir.appending("file")
+            XCTAssert(localFileSystem.exists(content))
+            XCTAssertEqual((try? localFileSystem.readFileContents(content))?.cString, "Hello World!")
+        }
+    }
+
+    func testArchiveDoesntExist() {
+        let fileSystem = InMemoryFileSystem()
+        let archiver = TarArchiver(fileSystem: fileSystem)
+        let archive = AbsolutePath("/archive.tar.gz")
+        XCTAssertThrowsError(try archiver.extract(from: archive, to: "/")) { error in
+            XCTAssertEqual(error as? FileSystemError, FileSystemError(.noEntry, archive))
+        }
+    }
+
+    func testDestinationDoesntExist() throws {
+        let fileSystem = InMemoryFileSystem(emptyFiles: "/archive.tar.gz")
+        let archiver = TarArchiver(fileSystem: fileSystem)
+        let destination = AbsolutePath("/destination")
+        XCTAssertThrowsError(try archiver.extract(from: "/archive.tar.gz", to: destination)) { error in
+            XCTAssertEqual(error as? FileSystemError, FileSystemError(.notDirectory, destination))
+        }
+    }
+
+    func testDestinationIsFile() throws {
+        let fileSystem = InMemoryFileSystem(emptyFiles: "/archive.tar.gz", "/destination")
+        let archiver = TarArchiver(fileSystem: fileSystem)
+        let destination = AbsolutePath("/destination")
+        XCTAssertThrowsError(try archiver.extract(from: "/archive.tar.gz", to: destination)) { error in
+            XCTAssertEqual(error as? FileSystemError, FileSystemError(.notDirectory, destination))
+        }
+    }
+
+    func testInvalidArchive() throws {
+        try testWithTemporaryDirectory { tmpdir in
+            let archiver = TarArchiver(fileSystem: localFileSystem)
+            let inputArchivePath = AbsolutePath(path: #file).parentDirectory
+                .appending(components: "Inputs", "invalid_archive.tar.gz")
+            XCTAssertThrowsError(try archiver.extract(from: inputArchivePath, to: tmpdir)) { error in
+                #if os(Linux)
+                XCTAssertMatch((error as? StringError)?.description, .contains("not in gzip format"))
+                #else
+                XCTAssertMatch((error as? StringError)?.description, .contains("Unrecognized archive format"))
+                #endif
+            }
+        }
+    }
+
+    func testValidation() throws {
+        // valid
+        try testWithTemporaryDirectory { _ in
+            let archiver = TarArchiver(fileSystem: localFileSystem)
+            let path = AbsolutePath(path: #file).parentDirectory
+                .appending(components: "Inputs", "archive.tar.gz")
+            XCTAssertTrue(try archiver.validate(path: path))
+        }
+        // invalid
+        try testWithTemporaryDirectory { _ in
+            let archiver = TarArchiver(fileSystem: localFileSystem)
+            let path = AbsolutePath(path: #file).parentDirectory
+                .appending(components: "Inputs", "invalid_archive.tar.gz")
+            XCTAssertFalse(try archiver.validate(path: path))
+        }
+        // error
+        try testWithTemporaryDirectory { _ in
+            let archiver = TarArchiver(fileSystem: localFileSystem)
+            let path = AbsolutePath.root.appending("does_not_exist.tar.gz")
+            XCTAssertThrowsError(try archiver.validate(path: path)) { error in
+                XCTAssertEqual(error as? FileSystemError, FileSystemError(.noEntry, path))
+            }
+        }
+    }
+
+    func testCompress() throws {
+        #if os(Linux)
+        guard SPM_posix_spawn_file_actions_addchdir_np_supported() else {
+            throw XCTSkip("working directory not supported on this platform")
+        }
+        #endif
+
+        try testWithTemporaryDirectory { tmpdir in
+            let archiver = TarArchiver(fileSystem: localFileSystem)
+
+            let rootDir = tmpdir.appending(component: UUID().uuidString)
+            try localFileSystem.createDirectory(rootDir)
+            try localFileSystem.writeFileContents(rootDir.appending("file1.txt"), string: "Hello World!")
+
+            let dir1 = rootDir.appending("dir1")
+            try localFileSystem.createDirectory(dir1)
+            try localFileSystem.writeFileContents(dir1.appending("file2.txt"), string: "Hello World 2!")
+
+            let dir2 = dir1.appending("dir2")
+            try localFileSystem.createDirectory(dir2)
+            try localFileSystem.writeFileContents(dir2.appending("file3.txt"), string: "Hello World 3!")
+            try localFileSystem.writeFileContents(dir2.appending("file4.txt"), string: "Hello World 4!")
+
+            let archivePath = tmpdir.appending(component: UUID().uuidString + ".tar.gz")
+            try archiver.compress(directory: rootDir, to: archivePath)
+            XCTAssertFileExists(archivePath)
+
+            let extractRootDir = tmpdir.appending(component: UUID().uuidString)
+            try localFileSystem.createDirectory(extractRootDir)
+            try archiver.extract(from: archivePath, to: extractRootDir)
+            try localFileSystem.stripFirstLevel(of: extractRootDir)
+
+            XCTAssertFileExists(extractRootDir.appending("file1.txt"))
+            XCTAssertEqual(
+                try? localFileSystem.readFileContents(extractRootDir.appending("file1.txt")),
+                "Hello World!"
+            )
+
+            let extractedDir1 = extractRootDir.appending("dir1")
+            XCTAssertDirectoryExists(extractedDir1)
+            XCTAssertFileExists(extractedDir1.appending("file2.txt"))
+            XCTAssertEqual(
+                try? localFileSystem.readFileContents(extractedDir1.appending("file2.txt")),
+                "Hello World 2!"
+            )
+
+            let extractedDir2 = extractedDir1.appending("dir2")
+            XCTAssertDirectoryExists(extractedDir2)
+            XCTAssertFileExists(extractedDir2.appending("file3.txt"))
+            XCTAssertEqual(
+                try? localFileSystem.readFileContents(extractedDir2.appending("file3.txt")),
+                "Hello World 3!"
+            )
+            XCTAssertFileExists(extractedDir2.appending("file4.txt"))
+            XCTAssertEqual(
+                try? localFileSystem.readFileContents(extractedDir2.appending("file4.txt")),
+                "Hello World 4!"
+            )
+        }
+    }
+}