Skip to content

Commit 6c86ba0

Browse files
raboofraboof
authored
Compare local artifacts to those from maven central (#97)
* Compare local artifacts to those from maven central * Remove duplication by using Def.task * Cleanup, actually run diffoscope * Final fixes and reporting script * gitignore
1 parent 21d409d commit 6c86ba0

File tree

4 files changed

+177
-86
lines changed

4 files changed

+177
-86
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
target
2+
/report

‎report.sh

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/sh
2+
3+
set -e
4+
5+
rm -r report || true
6+
mkdir report
7+
8+
find . -name "reproducible-builds-report.md" | sort | xargs cat > report/index.md
9+
find . -name "reproducible-builds-diffoscope-output-*" | sort | while read line ; do cp -r $line report ; done
10+
11+
markdown report/index.md > report/index.html
12+

‎src/main/scala/ReproducibleBuildsPlugin.scala

+131-79
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import java.nio.charset.Charset
55
import java.nio.file.Files
66

77
import scala.concurrent.duration._
8-
import gigahorse.GigahorseSupport
8+
import gigahorse.{GigahorseSupport, StatusError}
99
import sbt.{io => _, _}
1010
import sbt.Keys._
1111
import sbt.Classpaths._
@@ -40,98 +40,151 @@ object ReproducibleBuildsPlugin extends AutoPlugin {
4040

4141
val reproducibleBuildsCertification = taskKey[File]("Create a Reproducible Builds certification")
4242
val reproducibleBuildsCheckCertification = taskKey[Unit]("Download and compare Reproducible Builds certifications")
43+
val reproducibleBuildsCheckMavenCentral = taskKey[File]("Compare Reproducible Build certifications against those published on Maven Central")
4344

4445
val bzztNetResolver = Resolver.url("repo.bzzt.net", url("http://repo.bzzt.net:8000"))(Patterns().withArtifactPatterns(Vector(
4546
// We default to a Maven-style pattern with host and timestamp to reduce naming collisions, and branch if populated
4647
"[organisation]/[module](_[scalaVersion])(_[sbtVersion])/([branch]/)[revision]/[artifact]-[revision](-[classifier])(-[host])(-[timestamp]).[ext]"
4748
)))
4849

50+
lazy val ourCertification = Def.task[Certification] {
51+
Certification(
52+
organization.value,
53+
reproducibleBuildsPackageName.value,
54+
version.value,
55+
scmInfo.value,
56+
(packagedArtifacts in Compile).value,
57+
(scalaVersion in artifactName).value,
58+
(scalaBinaryVersion in artifactName).value,
59+
sbtVersion.value
60+
)
61+
}
62+
63+
lazy val ourCertificationFile = Def.task[File] {
64+
val certification = ourCertification.value
65+
66+
val targetDirPath = crossTarget.value
67+
val targetFilePath = targetDirPath.toPath.resolve(targetFilename(certification.artifactId, certification.version, certification.classifier))
68+
69+
Files.write(targetFilePath, certification.asPropertyString.getBytes(Charset.forName("UTF-8")))
70+
71+
targetFilePath.toFile
72+
}
73+
74+
sealed trait ArtifactUrlTarget
75+
case object MavenCentral extends ArtifactUrlTarget
76+
case object PublishToPrefix extends ArtifactUrlTarget
77+
78+
def artifactUrl(target: ArtifactUrlTarget, ext: String) = Def.task {
79+
import scala.collection.JavaConverters._
80+
val extraModuleAttributes = {
81+
val scalaVer = Map("scalaVersion" -> scalaBinaryVersion.value)
82+
if (sbtPlugin.value) scalaVer + ("sbtVersion" -> sbtBinaryVersion.value)
83+
else scalaVer
84+
}.asJava
85+
86+
val pattern: String = target match {
87+
case MavenCentral =>
88+
"http://repo1.maven.org/maven2/[organisation]/[module](_[scalaVersion])/[revision]/[artifact](_[scalaVersion])-[revision](-[classifier]).[ext]"
89+
case PublishToPrefix =>
90+
val pattern = (publishTo in ReproducibleBuilds).value.getOrElse(bzztNetResolver).asInstanceOf[URLRepository].patterns.artifactPatterns.head
91+
pattern.substring(0, pattern.lastIndexOf("/") + 1)
92+
}
93+
94+
IvyPatternHelper.substitute(
95+
pattern,
96+
organization.value.replace('.', '/'),
97+
reproducibleBuildsPackageName.value,
98+
version.value,
99+
reproducibleBuildsPackageName.value,
100+
ext,
101+
ext,
102+
"compile",
103+
extraModuleAttributes,
104+
null
105+
)
106+
}
107+
49108
override lazy val projectSettings = Seq(
50109
publishCertification := true,
51110
hostname := InetAddress.getLocalHost.getHostName,
52111
resolvers += bzztNetResolver,
53112
packageBin in Compile := postProcessJar((packageBin in Compile).value),
54113
reproducibleBuildsPackageName := moduleName.value,
55-
reproducibleBuildsCertification := {
56-
val certification = Certification(
57-
organization.value,
58-
reproducibleBuildsPackageName.value,
59-
version.value,
60-
scmInfo.value,
61-
(packagedArtifacts in Compile).value,
62-
(scalaVersion in artifactName).value,
63-
(scalaBinaryVersion in artifactName).value,
64-
sbtVersion.value
65-
)
66-
67-
val targetDirPath = crossTarget.value
68-
val targetFilePath = targetDirPath.toPath.resolve(targetFilename(certification.artifactId, certification.version, certification.classifier))
69-
70-
Files.write(targetFilePath, certification.asPropertyString.getBytes(Charset.forName("UTF-8")))
71-
72-
targetFilePath.toFile
73-
},
114+
reproducibleBuildsCertification := ourCertificationFile.value,
74115
artifact in ReproducibleBuilds := Artifact(reproducibleBuildsPackageName.value, "buildinfo", "buildinfo"),
75116
packagedArtifacts ++= {
76117
val generatedArtifact = Map(
77-
(artifact in ReproducibleBuilds).value ->
78-
{
79-
val certification = Certification(
80-
organization.value,
81-
reproducibleBuildsPackageName.value,
82-
version.value,
83-
scmInfo.value,
84-
(packagedArtifacts in Compile).value,
85-
(scalaVersion in artifactName).value,
86-
(scalaBinaryVersion in artifactName).value,
87-
sbtVersion.value
88-
)
89-
90-
val targetDirPath = crossTarget.value
91-
val targetFilePath = targetDirPath.toPath.resolve(targetFilename(certification.artifactId, certification.version, certification.classifier))
92-
93-
Files.write(targetFilePath, certification.asPropertyString.getBytes(Charset.forName("UTF-8")))
94-
95-
targetFilePath.toFile
96-
}
118+
(artifact in ReproducibleBuilds).value -> ourCertificationFile.value
97119
)
98120

99121
if (publishCertification.value) generatedArtifact else Map.empty[Artifact, File]
100122
},
123+
reproducibleBuildsCheckMavenCentral := {
124+
val ourArtifacts = (packagedArtifacts in Compile).value
125+
val url = artifactUrl(MavenCentral,"buildinfo").value
126+
127+
val log = streams.value.log
128+
log.info(s"Downloading certification from [$url]")
129+
val targetDirPath = crossTarget.value
130+
131+
val report: Future[String] = checkVerification(ourCertification.value, uri(url))
132+
.flatMap(result => {
133+
showResult(log, result)
134+
if (result.ok) Future.successful(result.asMarkdown)
135+
else Future.sequence({
136+
result.verdicts
137+
.collect { case (filename: String, m: Mismatch) => {
138+
val ext = filename.substring(filename.lastIndexOf('.') + 1)
139+
val mavenArtifactUrl = artifactUrl(MavenCentral, "").value + ext
140+
141+
val artifactName = mavenArtifactUrl.substring(mavenArtifactUrl.lastIndexOf('/') + 1)
142+
143+
val ourArtifact = ourArtifacts.collect { case (art, file) if art.`type` == ext => file }.toList match {
144+
case List() => throw new IllegalStateException(s"Did not find local artifact for $artifactName ($ext)")
145+
case List(artifact) => artifact
146+
case multiple => throw new IllegalStateException(s"Found multiple artifacts for $ext")
147+
}
148+
149+
http.run(GigahorseSupport.url(mavenArtifactUrl)).map { entity =>
150+
val mavenCentralArtifactsPath = targetDirPath.toPath.resolve("mavenCentralArtifact")
151+
mavenCentralArtifactsPath.toFile.mkdirs()
152+
val mavenCentralArtifact = mavenCentralArtifactsPath.resolve(artifactName)
153+
Files.write(
154+
mavenCentralArtifact,
155+
entity.bodyAsByteBuffer.array()
156+
)
157+
val diffoscopeOutputDir = targetDirPath.toPath.resolve(s"reproducible-builds-diffoscope-output-$artifactName")
158+
val cmd = s"diffoscope --html-dir $diffoscopeOutputDir $ourArtifact $mavenCentralArtifact"
159+
new ProcessBuilder(
160+
"diffoscope",
161+
"--html-dir",
162+
diffoscopeOutputDir.toFile.getAbsolutePath,
163+
ourArtifact.getAbsolutePath,
164+
mavenCentralArtifact.toFile.getAbsolutePath
165+
).start().waitFor()
166+
log.info(s"Running '$cmd' for a detailed report on the differences")
167+
s"See the [diffoscope report](reproducible-builds-diffoscope-output-$artifactName/index.html) for a detailed explanation " +
168+
" of the differences between the freshly built artifact and the one published to Maven Central"
169+
}.recover {
170+
case s: StatusError if s.status == 404 =>
171+
s"Unfortunately no artifact was found at $mavenArtifactUrl to diff against."
172+
}
173+
}
174+
}
175+
}).map { verdicts => result.asMarkdown + "\n\n" + verdicts.mkString("", "\n\n", "\n\n") }
176+
})
177+
178+
val targetFilePath = targetDirPath.toPath.resolve("reproducible-builds-report.md")
179+
180+
Files.write(targetFilePath, Await.result(report, 30.seconds).getBytes(Charset.forName("UTF-8")))
181+
182+
targetFilePath.toFile
183+
},
101184
reproducibleBuildsCheckCertification := {
102-
val ours = Certification(
103-
organization.value,
104-
reproducibleBuildsPackageName.value,
105-
version.value,
106-
scmInfo.value,
107-
(packagedArtifacts in Compile).value,
108-
(scalaVersion in artifactName).value,
109-
(scalaBinaryVersion in artifactName).value,
110-
sbtVersion.value
111-
)
185+
val ours = ourCertification.value
112186
val groupId = organization.value
113-
// TODO also check against the 'official' published buildinfo
114-
val pattern = (publishTo in ReproducibleBuilds).value.getOrElse(bzztNetResolver).asInstanceOf[URLRepository].patterns.artifactPatterns.head
115-
val prefixPattern = pattern.substring(0, pattern.lastIndexOf("/") + 1)
116-
import scala.collection.JavaConverters._
117-
val extraModuleAttributes = {
118-
val scalaVer = Map("scalaVersion" -> scalaBinaryVersion.value)
119-
if (sbtPlugin.value) scalaVer + ("sbtVersion" -> sbtBinaryVersion.value)
120-
else scalaVer
121-
}.asJava
122-
123-
val prefix = IvyPatternHelper.substitute(
124-
prefixPattern,
125-
organization.value.replace('.', '/'),
126-
reproducibleBuildsPackageName.value,
127-
version.value,
128-
reproducibleBuildsPackageName.value,
129-
"buildinfo",
130-
"buildinfo",
131-
"compile",
132-
extraModuleAttributes,
133-
null
134-
)
187+
val prefix = artifactUrl(PublishToPrefix, "buildinfo").value
135188
val log = streams.value.log
136189
log.info(s"Discovering certifications at [$prefix]")
137190
// TODO add Accept header to request JSON-formatted
@@ -147,13 +200,7 @@ object ReproducibleBuildsPlugin extends AutoPlugin {
147200
Future.sequence(results)
148201
}.map { resultList =>
149202
log.info(s"Processed ${resultList.size} results. ${resultList.count(_.ok)} matching attestations, ${resultList.filterNot(_.ok).size} mismatches");
150-
resultList.foreach { result =>
151-
log.info(s"${result.uri}:")
152-
log.info("- " + (if (result.ok) "OK" else "NOT OK"))
153-
result.verdicts.foreach {
154-
case (filename, verdict) => log.info(s"- $filename: $verdict")
155-
}
156-
}
203+
resultList.foreach { result => showResult(log, result) }
157204
}
158205
Await.result(done, 30.seconds)
159206
},
@@ -218,6 +265,9 @@ object ReproducibleBuildsPlugin extends AutoPlugin {
218265
publishM2 := publishTask(publishM2Configuration).value
219266
))
220267

268+
private def showResult(log: Logger, result: VerificationResult): Unit =
269+
log.info(result.asMarkdown)
270+
221271
private def gpgPluginSettings =
222272
if (gpgPluginOnClasspath) GpgHelpers.settings
223273
else Seq.empty
@@ -242,6 +292,8 @@ object ReproducibleBuildsPlugin extends AutoPlugin {
242292
http.run(GigahorseSupport.url(uri.toASCIIString)).map { entity =>
243293
val theirs = Certification(entity.bodyAsString)
244294
VerificationResult(uri, ourSums, theirs.checksums)
295+
}.recover {
296+
case e: StatusError if e.status == 404 => VerificationResult(uri, ourSums, Seq.empty)
245297
}
246298
}
247299

‎src/main/scala/VerificationResult.scala

+33-7
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,49 @@ package net.bzzt.reproduciblebuilds
22

33
import java.net.URI
44

5+
sealed trait Verdict {
6+
val description: String
7+
}
8+
case object Match extends Verdict {
9+
val description = "Match"
10+
}
11+
case object MissingInTheirs extends Verdict {
12+
val description = "Missing in theirs but present in ours"
13+
}
14+
case object MissingInOurs extends Verdict {
15+
val description = "Missing in ours but present in theirs"
16+
}
17+
case class Mismatch(our: Checksum, their: Checksum) extends Verdict {
18+
val description = s"Mismatch: our ${our.hexChecksum} did not match their ${their.hexChecksum}"
19+
}
20+
521
case class VerificationResult(
622
uri: URI,
723
ourSums: Map[String, Checksum],
824
remoteSums: Map[String, Checksum],
925
) {
10-
def verdicts: Seq[(String, String)] =
26+
def asMarkdown = {
27+
val artifactName = uri.toASCIIString.substring(uri.toASCIIString.lastIndexOf('/') + 1)
28+
s"""# `$artifactName`: ${if (ok) "OK" else "NOT OK"}
29+
|
30+
|${verdicts.map { case (filename, verdict) => s"- $filename: ${verdict.description}" }.mkString("\n")}
31+
""".stripMargin
32+
}
33+
34+
/**
35+
* filename -> verdict
36+
*/
37+
def verdicts: Seq[(String, Verdict)] =
1138
ourSums.map {
1239
case (key, value) => (key, verdict(key, value))
1340
}.toSeq ++
14-
ourSums.keySet.diff(remoteSums.keySet).map { missingInTheirs => (missingInTheirs, "Missing in their checksums but present in ours") } ++
15-
remoteSums.keySet.diff(ourSums.keySet).map { missingInOurs => (missingInOurs, "Missing in our checksums but present in theirs") }
41+
remoteSums.keySet.diff(ourSums.keySet).map { missingInOurs => (missingInOurs, MissingInOurs) }
1642

17-
def verdict(filename: String, ourSum: Checksum): String = remoteSums.get(filename) match {
18-
case None => "Not found in remote attestation"
43+
def verdict(filename: String, ourSum: Checksum): Verdict = remoteSums.get(filename) match {
44+
case None => MissingInTheirs
1945
case Some(checksum) =>
20-
if (checksum == ourSum) "Match"
21-
else s"Mismatch: our ${ourSum.hexChecksum} did not match their ${checksum.hexChecksum}"
46+
if (checksum == ourSum) Match
47+
else Mismatch(ourSum, checksum)
2248
}
2349

2450
def ok = ourSums == remoteSums

0 commit comments

Comments
 (0)