Skip to content

Package manifest checksum TOFU #6322

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 28, 2023
Merged

Package manifest checksum TOFU #6322

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a009844
Package manifest checksum TOFU
yim-lee Mar 21, 2023
4b1bc30
address review feedback
yim-lee Mar 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
360 changes: 256 additions & 104 deletions Sources/PackageFingerprint/FilePackageFingerprintStorage.swift
View file
Open in desktop

Large diffs are not rendered by default.

35 changes: 29 additions & 6 deletions Sources/PackageFingerprint/Model.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
//
// This source file is part of the Swift open source project
//
// Copyright (c) 2021 Apple Inc. and the Swift project authors
// Copyright (c) 2021-2023 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
Expand All @@ -12,25 +12,28 @@

import struct Foundation.URL

import PackageModel
import struct TSCUtility.Version

public struct Fingerprint: Equatable {
public let origin: Origin
public let value: String
public let contentType: ContentType

public init(origin: Origin, value: String) {
public init(origin: Origin, value: String, contentType: ContentType) {
self.origin = origin
self.value = value
self.contentType = contentType
}
}

public extension Fingerprint {
enum Kind: String, Hashable {
extension Fingerprint {
public enum Kind: String, Hashable {
case sourceControl
case registry
}

enum Origin: Equatable, CustomStringConvertible {
public enum Origin: Equatable, CustomStringConvertible {
case sourceControl(URL)
case registry(URL)

Expand Down Expand Up @@ -61,9 +64,29 @@ public extension Fingerprint {
}
}
}

/// Each package version has a dictionary of fingerprints identified by content type.
/// Fingerprints of content type `sourceCode` can come from registry (i.e., source archive checksum)
/// or git repo (commit hash). However, the current implementation only stores fingerprints for manifests
/// downloaded from registry. It doesn't not save fingerprints for manifests in git repo.
public enum ContentType: Hashable, CustomStringConvertible {
case sourceCode
case manifest(ToolsVersion?)

public var description: String {
switch self {
case .sourceCode:
return "sourceCode"
case .manifest(.none):
return Manifest.filename
case .manifest(.some(let toolsVersion)):
return "Package@swift-\(toolsVersion).swift"
}
}
}
}

public typealias PackageFingerprints = [Version: [Fingerprint.Kind: Fingerprint]]
public typealias PackageFingerprints = [Version: [Fingerprint.Kind: [Fingerprint.ContentType: Fingerprint]]]

public enum FingerprintCheckingMode: String {
case strict
Expand Down
23 changes: 14 additions & 9 deletions Sources/PackageFingerprint/PackageFingerprintStorage.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public protocol PackageFingerprintStorage {
version: Version,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
callback: @escaping (Result<[Fingerprint.Kind: Fingerprint], Error>) -> Void
callback: @escaping (Result<[Fingerprint.Kind: [Fingerprint.ContentType: Fingerprint]], Error>) -> Void
)

func put(
Expand All @@ -39,7 +39,7 @@ public protocol PackageFingerprintStorage {
version: Version,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
callback: @escaping (Result<[Fingerprint.Kind: Fingerprint], Error>) -> Void
callback: @escaping (Result<[Fingerprint.Kind: [Fingerprint.ContentType: Fingerprint]], Error>) -> Void
)

func put(
Expand All @@ -57,6 +57,7 @@ extension PackageFingerprintStorage {
package: PackageIdentity,
version: Version,
kind: Fingerprint.Kind,
contentType: Fingerprint.ContentType,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
callback: @escaping (Result<Fingerprint, Error>) -> Void
Expand All @@ -67,14 +68,15 @@ extension PackageFingerprintStorage {
observabilityScope: observabilityScope,
callbackQueue: callbackQueue
) { result in
self.get(kind: kind, result, callback: callback)
self.get(kind: kind, contentType: contentType, result, callback: callback)
}
}

public func get(
package: PackageReference,
version: Version,
kind: Fingerprint.Kind,
contentType: Fingerprint.ContentType,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
callback: @escaping (Result<Fingerprint, Error>) -> Void
Expand All @@ -85,17 +87,20 @@ extension PackageFingerprintStorage {
observabilityScope: observabilityScope,
callbackQueue: callbackQueue
) { result in
self.get(kind: kind, result, callback: callback)
self.get(kind: kind, contentType: contentType, result, callback: callback)
}
}

private func get(
kind: Fingerprint.Kind,
_ fingerprintsResult: Result<[Fingerprint.Kind: Fingerprint], Error>,
contentType: Fingerprint.ContentType,
_ fingerprintsByKindResult: Result<[Fingerprint.Kind: [Fingerprint.ContentType: Fingerprint]], Error>,
callback: @escaping (Result<Fingerprint, Error>) -> Void
) {
callback(fingerprintsResult.tryMap { fingerprints in
guard let fingerprint = fingerprints[kind] else {
callback(fingerprintsByKindResult.tryMap { fingerprintsByKind in
guard let fingerprintsByContentType = fingerprintsByKind[kind],
let fingerprint = fingerprintsByContentType[contentType]
else {
throw PackageFingerprintStorageError.notFound
}
return fingerprint
Expand All @@ -110,9 +115,9 @@ public enum PackageFingerprintStorageError: Error, Equatable, CustomStringConver
public var description: String {
switch self {
case .conflict(let given, let existing):
return "Fingerprint \(given) is different from previously recorded value \(existing)"
return "fingerprint \(given) is different from previously recorded value \(existing)"
case .notFound:
return "Not found"
return "not found"
}
}
}
79 changes: 74 additions & 5 deletions Sources/PackageRegistry/ChecksumTOFU.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ struct PackageVersionChecksumTOFU {
self.versionMetadataProvider = versionMetadataProvider
}

func validate(
// MARK: - source archive

func validateSourceArchive(
registry: Registry,
package: PackageIdentity.RegistryIdentity,
version: Version,
Expand All @@ -62,7 +64,7 @@ struct PackageVersionChecksumTOFU {
case .warn:
observabilityScope
.emit(
warning: "The checksum \(checksum) does not match previously recorded value \(expectedChecksum)"
warning: "the checksum \(checksum) for source archive of \(package) \(version) does not match previously recorded value \(expectedChecksum)"
)
}
}
Expand All @@ -84,6 +86,7 @@ struct PackageVersionChecksumTOFU {
self.readFromStorage(
package: package,
version: version,
contentType: .sourceCode,
observabilityScope: observabilityScope,
callbackQueue: callbackQueue
) { result in
Expand Down Expand Up @@ -113,6 +116,7 @@ struct PackageVersionChecksumTOFU {
package: package,
version: version,
checksum: checksum,
contentType: .sourceCode,
observabilityScope: observabilityScope,
callbackQueue: callbackQueue
) { writeResult in
Expand All @@ -130,9 +134,69 @@ struct PackageVersionChecksumTOFU {
}
}

// MARK: - manifests

func validateManifest(
registry: Registry,
package: PackageIdentity.RegistryIdentity,
version: Version,
toolsVersion: ToolsVersion?,
checksum: String,
timeout: DispatchTimeInterval?,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
completion: @escaping (Result<Void, Error>) -> Void
) {
let contentType = Fingerprint.ContentType.manifest(toolsVersion)

self.readFromStorage(
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unlike source archives, registry doesn't provide expected checksum for manifests, so here we do true TOFU I suppose, and that is if checksum not found in storage we save it (whereas for source archive we have registry as an other source).

package: package,
version: version,
contentType: .manifest(toolsVersion),
observabilityScope: observabilityScope,
callbackQueue: callbackQueue
) { result in
switch result {
case .success(.some(let expectedChecksum)):
// Previously recorded checksum
do {
if checksum != expectedChecksum {
switch self.fingerprintCheckingMode {
case .strict:
throw RegistryError.invalidChecksum(expected: expectedChecksum, actual: checksum)
case .warn:
observabilityScope
.emit(
warning: "the checksum \(checksum) for \(contentType) of \(package) \(version) does not match previously recorded value \(expectedChecksum)"
)
}
}
completion(.success(()))
} catch {
completion(.failure(error))
}
default:
self.writeToStorage(
registry: registry,
package: package,
version: version,
checksum: checksum,
contentType: .manifest(toolsVersion),
observabilityScope: observabilityScope,
callbackQueue: callbackQueue
) { writeResult in
completion(writeResult.tryMap { _ in () })
}
}
}
}

// MARK: - storage helpers

private func readFromStorage(
package: PackageIdentity.RegistryIdentity,
version: Version,
contentType: Fingerprint.ContentType,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
completion: @escaping (Result<String?, Error>) -> Void
Expand All @@ -145,6 +209,7 @@ struct PackageVersionChecksumTOFU {
package: package.underlying,
version: version,
kind: .registry,
contentType: contentType,
observabilityScope: observabilityScope,
callbackQueue: callbackQueue
) { result in
Expand All @@ -155,7 +220,9 @@ struct PackageVersionChecksumTOFU {
completion(.success(nil))
case .failure(let error):
observabilityScope
.emit(error: "Failed to get registry fingerprint for \(package) \(version) from storage: \(error)")
.emit(
error: "failed to get registry fingerprint for \(contentType) of \(package) \(version) from storage: \(error)"
)
completion(.failure(error))
}
}
Expand All @@ -166,6 +233,7 @@ struct PackageVersionChecksumTOFU {
package: PackageIdentity.RegistryIdentity,
version: Version,
checksum: String,
contentType: Fingerprint.ContentType,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
completion: @escaping (Result<Void, Error>) -> Void
Expand All @@ -174,10 +242,11 @@ struct PackageVersionChecksumTOFU {
return completion(.success(()))
}

let fingerprint = Fingerprint(origin: .registry(registry.url), value: checksum, contentType: contentType)
fingerprintStorage.put(
package: package.underlying,
version: version,
fingerprint: .init(origin: .registry(registry.url), value: checksum),
fingerprint: fingerprint,
observabilityScope: observabilityScope,
callbackQueue: callbackQueue
) { result in
Expand All @@ -191,7 +260,7 @@ struct PackageVersionChecksumTOFU {
case .warn:
observabilityScope
.emit(
warning: "The checksum \(checksum) from \(registry.url.absoluteString) does not match previously recorded value \(existing.value) from \(String(describing: existing.origin.url?.absoluteString))"
warning: "the checksum \(checksum) for \(contentType) of \(package) \(version) from \(registry.url.absoluteString) does not match previously recorded value \(existing.value) from \(String(describing: existing.origin.url?.absoluteString))"
)
completion(.success(()))
}
Expand Down
Loading