Stdlib serialization redux

Package.swift

@@ -55,6 +55,7 @@ let package = Package(
     .package(
       url: "https://github.com/apple/swift-format",
       from: "508.0.1"),
+    .package(url: "https://github.com/dabrahams/CBORCoding.git", branch: "main"),
     .package(
       url: "https://github.com/SwiftPackageIndex/SPIManifest.git",
       from: "0.12.0"),
@@ -129,6 +130,7 @@ let package = Package(
       name: "Utils",
       dependencies: [
         .product(name: "BigInt", package: "BigInt"),
+        .product(name: "CBORCoding", package: "CBORCoding"),
         .product(name: "Collections", package: "swift-collections"),
         .product(name: "Algorithms", package: "swift-algorithms"),
       ],
@@ -141,10 +143,12 @@ let package = Package(
 
     .target(
       name: "StandardLibrary",
-      dependencies: ["FrontEnd", "Utils"],
+      dependencies: ["FrontEnd", "Utils", .product(name: "CBORCoding", package: "CBORCoding")],
       path: "StandardLibrary",
       exclude: ["Sources"],
-      swiftSettings: allTargetsSwiftSettings),
+      swiftSettings: allTargetsSwiftSettings,
+      plugins: ["StandardLibraryBuilderPlugin"]
+    ),
 
     .plugin(
       name: "TestGeneratorPlugin", capability: .buildTool(),
@@ -159,6 +163,19 @@ let package = Package(
       ],
       swiftSettings: allTargetsSwiftSettings),
 
+    .plugin(
+      name: "StandardLibraryBuilderPlugin", capability: .buildTool(),
+      dependencies: osIsWindows ? [] : ["BuildStandardLibrary"]),
+
+    .executableTarget(
+      name: "BuildStandardLibrary",
+      dependencies: [
+        .product(name: "ArgumentParser", package: "swift-argument-parser"),
+        .product(name: "CBORCoding", package: "CBORCoding"),
+        "Core", "Utils", "FrontEnd",
+      ],
+      swiftSettings: allTargetsSwiftSettings),
+
     // Test targets.
     .testTarget(
       name: "UtilsTests",
@@ -180,6 +197,7 @@ let package = Package(
       dependencies: [
         "Core", "FrontEnd", "IR", "TestUtils", "StandardLibrary", "Utils",
         .product(name: "Algorithms", package: "swift-algorithms"),
+        .product(name: "CBORCoding", package: "CBORCoding"),
       ],
       exclude: ["TestCases"],
       swiftSettings: allTargetsSwiftSettings,
@@ -205,7 +223,7 @@ let package = Package(
       ? [
         .target(
           name: "BuildToolDependencies",
-          dependencies: ["GenerateHyloFileTests"],
+          dependencies: ["GenerateHyloFileTests", "BuildStandardLibrary"],
           swiftSettings: allTargetsSwiftSettings)
       ] : []) as [PackageDescription.Target]
 )

Plugins/StandardLibraryBuilderPlugin/SPMBuildToolSupport.swift

@@ -0,0 +1 @@
+../../SPMBuildToolSupport/SPMBuildToolSupport.swift

Plugins/StandardLibraryBuilderPlugin/StandardLibraryBuilderPlugin.swift

@@ -0,0 +1,41 @@
+import Foundation
+import PackagePlugin
+
+/// The Swift Package Manager plugin that builds the standard library.
+@main
+struct StandardLibraryBuilderPlugin: SPMBuildToolPlugin {
+
+  func buildCommands(
+    context: PackagePlugin.PluginContext, target: PackagePlugin.Target
+  ) throws -> [SPMBuildCommand] {
+    let sourceDirectory = URL(fileURLWithPath: #filePath)
+      .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
+      / "StandardLibrary" / "Sources"
+
+    let hostedSources = FileManager.default.enumerator(
+      at: sourceDirectory,
+      includingPropertiesForKeys: [.isRegularFileKey],
+      options: [.skipsHiddenFiles, .skipsPackageDescendants])!
+      .compactMap { $0 as? URL }
+      .filter { $0.pathExtension == "hylo" }
+
+    let freestandingRoot = (sourceDirectory / "Core").pathComponents
+    let freestandingSources = hostedSources.filter { $0.pathComponents.starts(with: freestandingRoot) }
+
+    func buildLibrary(name: String, sources: [URL]) -> SPMBuildCommand {
+      let output = context.pluginWorkDirectory.url / (name + ".cbor")
+      return .buildCommand(
+        displayName: "Building \(name) standard library module into \(output.platformString)",
+        executable:  .targetInThisPackage("BuildStandardLibrary"),
+        arguments: ["-o", output.platformString] + sources.map(\.platformString),
+        inputFiles: sources.map(\.spmPath),
+        outputFiles: [output.spmPath])
+    }
+
+    return [
+      buildLibrary(name: "hosted", sources: hostedSources),
+      buildLibrary(name: "freestanding", sources: freestandingSources)
+    ]
+  }
+
+}

Sources/BuildStandardLibrary/BuildStandardLibrary.swift

@@ -0,0 +1,46 @@
+import ArgumentParser
+import CBORCoding
+import Core
+import Foundation
+import FrontEnd
+import Utils
+
+/// A command-line tool that generates a Hylo standard library module as part of our build process.
+@main
+struct BuildStandardLibrary: ParsableCommand {
+
+  @Option(
+    name: [.customShort("o")],
+    help: ArgumentHelp("Write output to <file>.", valueName: "module-file"),
+    transform: URL.init(fileURLWithPath:))
+  var outputURL: URL
+
+  @Argument(
+    help: "The Hylo source files that comprise the library.",
+    transform: URL.init(fileURLWithPath:))
+  var hyloSourceFiles: [URL]
+
+  func run() throws {
+    try CBOREncoder().forAST
+      .encode(AST(sources: hyloSourceFiles, for: ConditionalCompilationFactors()))
+      .write(to: outputURL, options: .atomic)
+  }
+
+}
+
+extension AST {
+
+  /// Creates an instance that includes the Hylo library built from the given `sources`.
+  init(sources: [URL], for conditions: ConditionalCompilationFactors) throws {
+    self.init(conditions)
+    var diagnostics = DiagnosticSet()
+    coreLibrary = try makeModule(
+      "Hylo",
+      sourceCode: sources.map(SourceFile.init(contentsOf:)),
+      builtinModuleAccess: true,
+      diagnostics: &diagnostics)
+    assert(coreModuleIsLoaded)
+    self.coreTraits = .init(self)
+  }
+
+}

Sources/BuildStandardLibrary/FullPathInFatalErrors.swift

@@ -0,0 +1 @@
+../GenerateHyloFileTests/FullPathInFatalErrors.swift

Sources/Utils/StatefulCoder.swift

@@ -1,3 +1,4 @@
+import CBORCoding
 import Foundation
 
 /// A key used to access the coding state of encoders/decoders.
@@ -112,6 +113,8 @@ extension JSONEncoder: StatefulEncoder {}
 extension JSONDecoder: StatefulDecoder {}
 extension PropertyListEncoder: StatefulEncoder {}
 extension PropertyListDecoder: StatefulDecoder {}
+extension CBOREncoder: StatefulEncoder {}
+extension CBORDecoder: StatefulDecoder {}
 
 /// A (thread-unsafe) shared mutable wrapper for a `WrappedType` instance.
 ///

StandardLibrary/StandardLibrary.swift

@@ -1,7 +1,25 @@
+import CBORCoding
 import Core
 import Foundation
 import Utils
 
+/// An error thrown when a resource couldn't be found on the host.
+private struct ResourceNotFound: Error {
+
+  /// An identifier for the resource that wasn't found.
+  let resource: String
+
+}
+
+/// Returns the URL of the resource with given name and extension in the bundle associated with the
+/// current Swift module.
+private func resource(_ name: String, withExtension ext: String) throws -> URL {
+  guard let u = Bundle.module.url(forResource: name, withExtension: ext) else {
+    throw ResourceNotFound(resource: "\(name).\(ext)")
+  }
+  return u
+}
+
 // This path points into the source tree rather than to some copy so that when diagnostics are
 // issued for the standard library, they point to the original source files and edits to those
 // source files actually fix the problems (see https://github.com/hylo-lang/hylo/issues/932).  If we
@@ -20,14 +38,24 @@ private let freestandingLibrarySourceRoot = hostedLibrarySourceRoot.appendingPat
 
 extension Utils.Host {
 
+  /// Returns an AST representing the given standard library `variant` (either `"freestanding"` or
+  /// `"hosted"`), conditionally compiled for targeting the host platform.
+  private static func libraryAST(_ variant: String) -> Result<AST, Error> {
+    Result {
+      try CBORDecoder().forAST.decode(
+        AST.self,
+        from: Data(
+          contentsOf: resource(variant, withExtension: "cbor"),
+          options: .alwaysMapped))
+    }
+  }
+
   /// An AST representing the whole standard library, conditionally compiled for targeting the host
   /// platform.
-  public static let hostedLibraryAST
-    = Result { try AST(libraryRoot: hostedLibrarySourceRoot, ConditionalCompilationFactors(freestanding: false)) }
+  public static let hostedLibraryAST: Result<AST, Error> = libraryAST("hosted")
 
   /// An AST representing the freestanding core of standard library, conditionally compiled for
   /// targeting the host platform.
-  public static let freestandingLibraryAST
-    = Result { try AST(libraryRoot: freestandingLibrarySourceRoot, ConditionalCompilationFactors(freestanding: true)) }
+  public static let freestandingLibraryAST: Result<AST, Error> = libraryAST("freestanding")
 
 }

Tests/HyloTests/ASTTests.swift

@@ -1,3 +1,4 @@
+import CBORCoding
 import Core
 import FrontEnd
 import StandardLibrary