opt-in caching for scalafix invocations

Author: github-brice-jaglin

src/main/scala-sbt-0.13/sbt/internal/sbtscalafix/Caching.scala

@@ -0,0 +1,42 @@
+package sbt.internal.sbtscalafix
+
+import java.io.ByteArrayOutputStream
+
+import sbinary._
+import sbt._
+import scalafix.internal.sbt.Arg
+
+import scala.util.DynamicVariable
+
+object Caching {
+
+  trait CacheKeysStamper
+      extends InputCache[Seq[Arg.CacheKey]]
+      with CacheImplicits {
+    private val output = new DynamicVariable[Output](null)
+
+    implicit val lastModifiedFormat = FileInfo.lastModified.format
+    val baFormat = implicitly[Format[Array[Byte]]]
+    val baEquiv = implicitly[Equiv[Array[Byte]]]
+
+    protected def stamp: Arg.CacheKey => Unit
+
+    final def write[T](t: T)(implicit format: Format[T]): Unit =
+      format.writes(output.value, t)
+
+    override final type Internal = Array[Byte]
+    override final def convert(keys: Seq[Arg.CacheKey]): Array[Byte] = {
+      val baos = new ByteArrayOutputStream()
+      output.withValue(new JavaOutput(baos)) {
+        keys.foreach(stamp)
+      }
+      Hash.apply(baos.toByteArray)
+    }
+    override final def read(from: Input): Array[Byte] =
+      baFormat.reads(from)
+    override final def write(to: Output, ba: Array[Byte]): Unit =
+      baFormat.writes(to, ba)
+    override final def equiv: Equiv[Array[Byte]] =
+      baEquiv
+  }
+}

src/main/scala-sbt-1.0/sbt/internal/sbtscalafix/Caching.scala

@@ -0,0 +1,35 @@
+package sbt.internal.sbtscalafix
+
+import scalafix.internal.sbt.Arg
+import sjsonnew._
+
+import scala.util.DynamicVariable
+
+object Caching {
+
+  trait CacheKeysStamper
+      extends JsonFormat[Seq[Arg.CacheKey]]
+      with BasicJsonProtocol {
+    private val builder = new DynamicVariable[Builder[_]](null)
+
+    protected def stamp: Arg.CacheKey => Unit
+
+    final def write[T](t: T)(implicit format: JsonFormat[T]): Unit =
+      format.write(t, builder.value)
+
+    override final def write[J](obj: Seq[Arg.CacheKey], b: Builder[J]): Unit = {
+      b.beginArray()
+      builder.withValue(b) {
+        obj.foreach(stamp)
+      }
+      b.endArray()
+    }
+
+    // we actually don't need to read anything back, see https://github.com/sbt/sbt/pull/5513
+    override final def read[J](
+        jsOpt: Option[J],
+        unbuilder: Unbuilder[J]
+    ): Seq[Arg.CacheKey] = ???
+  }
+
+}

src/main/scala/scalafix/internal/sbt/ScalafixCompletions.scala

@@ -145,10 +145,13 @@ class ScalafixCompletions(
         "--syntactic",
         "--verbose",
         "--version"
-      ).map(literal)
+      ).map(f => literal(f).map(ShellArgs.Extra(_))) ++ Seq(
+        "--no-cache".^^^(ShellArgs.NoCache)
+      )
       Parser.oneOf(flags) |
         hide(string) // catch-all for all args not known to sbt-scalafix
-    }.map(ShellArgs.Extra(_))
+          .map(ShellArgs.Extra(_))
+    }
 
     val rule: ArgP =
       fileRule |

src/main/scala/scalafix/internal/sbt/ScalafixInterface.scala

@@ -17,45 +17,54 @@ import scala.util.control.NonFatal
 sealed trait Arg extends (ScalafixArguments => ScalafixArguments)
 
 object Arg {
-  case class ToolClasspath(classLoader: URLClassLoader) extends Arg {
+  sealed trait CacheKey
+
+  case class ToolClasspath(classLoader: URLClassLoader)
+      extends Arg
+      with CacheKey {
     override def apply(sa: ScalafixArguments): ScalafixArguments =
       sa.withToolClasspath(classLoader)
   }
 
-  case class Rules(rules: Seq[String]) extends Arg {
+  case class Rules(rules: Seq[String]) extends Arg with CacheKey {
     override def apply(sa: ScalafixArguments): ScalafixArguments =
       sa.withRules(rules.asJava)
   }
 
-  case class Paths(paths: Seq[Path]) extends Arg {
+  case class Paths(paths: Seq[Path]) extends Arg { // this one is extracted/stamped directly
     override def apply(sa: ScalafixArguments): ScalafixArguments =
       sa.withPaths(paths.asJava)
   }
 
-  case class Config(file: Option[Path]) extends Arg {
+  case class Config(file: Option[Path]) extends Arg with CacheKey {
     override def apply(sa: ScalafixArguments): ScalafixArguments =
       sa.withConfig(jutil.Optional.ofNullable(file.orNull))
   }
 
-  case class ParsedArgs(args: Seq[String]) extends Arg {
+  case class ParsedArgs(args: Seq[String]) extends Arg with CacheKey {
     override def apply(sa: ScalafixArguments): ScalafixArguments =
       sa.withParsedArguments(args.asJava)
   }
 
-  case class ScalaVersion(version: String) extends Arg {
+  case class ScalaVersion(version: String) extends Arg { //FIXME: with CacheKey {
     override def apply(sa: ScalafixArguments): ScalafixArguments =
       sa.withScalaVersion(version)
   }
 
-  case class ScalacOptions(options: Seq[String]) extends Arg {
+  case class ScalacOptions(options: Seq[String]) extends Arg { //FIXME: with CacheKey {
     override def apply(sa: ScalafixArguments): ScalafixArguments =
       sa.withScalacOptions(options.asJava)
   }
 
-  case class Classpath(classpath: Seq[Path]) extends Arg {
+  case class Classpath(classpath: Seq[Path]) extends Arg { //FIXME: with CacheKey {
     override def apply(sa: ScalafixArguments): ScalafixArguments =
       sa.withClasspath(classpath.asJava)
   }
+
+  case object NoCache extends Arg with CacheKey {
+    override def apply(sa: ScalafixArguments): ScalafixArguments =
+      sa // caching is currently implemented in sbt-scalafix itself
+  }
 }
 
 class ScalafixInterface private (
@@ -89,7 +98,7 @@ class ScalafixInterface private (
   }
 
   def validate(): Option[ScalafixException] =
-    scalafixArguments.validate().asScala
+    Option(scalafixArguments.validate().orElse(null))
 
 }
 

src/main/scala/scalafix/internal/sbt/ShellArgs.scala

@@ -6,22 +6,26 @@ object ShellArgs {
   case class Rule(value: String) extends Arg
   case class File(value: String) extends Arg
   case class Extra(key: String, value: Option[String] = None) extends Arg
+  case object NoCache extends Arg
 
   def apply(args: Seq[Arg]): ShellArgs = {
     val rules = List.newBuilder[String]
     val files = List.newBuilder[String]
     val extra = List.newBuilder[String]
+    var noCache = false
     args.foreach {
       case x: Rule => rules += x.value
       case x: File => files += x.value
       case x: Extra => extra += x.value.foldLeft(x.key)((k, v) => s"$k=$v")
+      case NoCache => noCache = true
     }
-    ShellArgs(rules.result(), files.result(), extra.result())
+    ShellArgs(rules.result(), files.result(), extra.result(), noCache)
   }
 }
 
 case class ShellArgs(
     rules: List[String] = Nil,
     files: List[String] = Nil,
-    extra: List[String] = Nil
+    extra: List[String] = Nil,
+    noCache: Boolean = false
 )

src/main/scala/scalafix/sbt/ScalafixPlugin.scala

@@ -8,9 +8,13 @@ import sbt.Keys._
 import sbt.internal.sbtscalafix.{Compat, JLineAccess}
 import sbt.plugins.JvmPlugin
 import sbt._
+import sbt.internal.sbtscalafix.Caching._
 import scalafix.interfaces.ScalafixError
 import scalafix.internal.sbt._
 
+import scala.util.Try
+import scala.util.control.NoStackTrace
+
 object ScalafixPlugin extends AutoPlugin {
   override def trigger: PluginTrigger = allRequirements
   override def requires: Plugins = JvmPlugin
@@ -24,6 +28,10 @@ object ScalafixPlugin extends AutoPlugin {
           "For example: scalafix RemoveUnusedImports. " +
           "To run on test sources use test:scalafix."
       )
+    val scalafixCaching: SettingKey[Boolean] =
+      settingKey[Boolean](
+        "Cache scalafix invocations (off by default, still experimental)."
+      )
     val scalafixResolvers: SettingKey[Seq[Repository]] =
       settingKey[Seq[Repository]](
         "Optional list of Maven/Ivy repositories to use for fetching custom rules."
@@ -99,6 +107,7 @@ object ScalafixPlugin extends AutoPlugin {
       )
     },
     scalafixConfig := None, // let scalafix-cli try to infer $CWD/.scalafix.conf
+    scalafixCaching := false,
     scalafixResolvers := ScalafixCoursier.defaultResolvers,
     scalafixDependencies := Nil,
     commands += ScalafixEnable.command
@@ -158,11 +167,15 @@ object ScalafixPlugin extends AutoPlugin {
           scalafixDependencies.in(ThisBuild).value,
           scalafixResolvers.in(ThisBuild).value
         )
-        val mainInterface = mainInterface0.withArgs(
-          Arg.Config(scalafixConf),
-          Arg.Rules(shell.rules),
-          Arg.ParsedArgs(shell.extra)
-        )
+        val maybeNoCache =
+          if (shell.noCache || !scalafixCaching.value) Seq(Arg.NoCache) else Nil
+        val mainInterface = mainInterface0
+          .withArgs(maybeNoCache: _*)
+          .withArgs(
+            Arg.Config(scalafixConf),
+            Arg.Rules(shell.rules),
+            Arg.ParsedArgs(shell.extra)
+          )
         val rulesThatWillRun = mainInterface.rulesThatWillRun()
         val isSemantic = rulesThatWillRun.exists(_.kind().isSemantic)
         if (isSemantic) {
@@ -185,7 +198,7 @@ object ScalafixPlugin extends AutoPlugin {
       config: Configuration
   ): Def.Initialize[Task[Unit]] = Def.task {
     val files = filesToFix(shellArgs, config).value
-    runArgs(mainInterface.withArgs(Arg.Paths(files)), streams.value.log)
+    runArgs(mainInterface.withArgs(Arg.Paths(files)), streams.value)
   }
 
   private def scalafixSemantic(
@@ -209,7 +222,7 @@ object ScalafixPlugin extends AutoPlugin {
           Arg.Paths(files),
           Arg.Classpath(fullClasspath.value.map(_.data.toPath))
         )
-        runArgs(semanticInterface, streams.value.log)
+        runArgs(semanticInterface, streams.value)
       }
     } else {
       Def.task {
@@ -227,18 +240,78 @@ object ScalafixPlugin extends AutoPlugin {
 
   private def runArgs(
       interface: ScalafixInterface,
-      logger: Logger
+      streams: TaskStreams
   ): Unit = {
     val paths = interface.args.collect { case Arg.Paths(paths) => paths }.flatten
     if (paths.nonEmpty) {
       if (paths.lengthCompare(1) > 0) {
-        logger.info(s"Running scalafix on ${paths.size} Scala sources")
+        streams.log.info(s"Running scalafix on ${paths.size} Scala sources")
+      }
+
+      val cacheKeyArgs = interface.args.collect {
+        case cacheKey: Arg.CacheKey => cacheKey
+      }
+      val pathsFilesInfo = FilesInfo
+        .lastModified(paths.map(_.toFile).toSet)
+        .asInstanceOf[FilesInfo[ModifiedFileInfo]]
+
+      // used to signal that one of the argument cannot be reliably stamped
+      object StampingImpossible extends RuntimeException with NoStackTrace
+
+      implicit val stamper = new CacheKeysStamper {
+        override protected def stamp: Arg.CacheKey => Unit = {
+          case Arg.ToolClasspath(classLoader) =>
+            //FIXME: stamp content of files/directories, don't cache if remote
+            write(classLoader.getURLs)
+          case Arg.Rules(rules) =>
+            //FIXME: don't cache if remote
+            write(rules)
+          case Arg.Config(file) =>
+            //FIXME: stamp default conf file when not provided
+            write(file.map(p => FileInfo.lastModified(p.toFile)))
+          case Arg.ParsedArgs(args) =>
+            val cacheKeys = args.filter {
+              case "--check" | "--test" =>
+                // CHECK & IN_PLACE can share the same cache
+                false
+              case "--stdout" =>
+                // --stdout cannot be cached as we don't capture the output to replay it
+                throw StampingImpossible
+              case _ => true
+            }
+            write(cacheKeys)
+          case Arg.NoCache =>
+            throw StampingImpossible
+        }
+      }
+
+      def diffWithPreviousRun[T](f: Boolean => T): T = {
+        val tracker = Tracked.inputChanged(streams.cacheDirectory / "inputs") {
+          (inputChanged: Boolean, _: Seq[Arg.CacheKey]) =>
+            Tracked.outputChanged(streams.cacheDirectory / "outputs") {
+              (outputChanged: Boolean, _: FilesInfo[ModifiedFileInfo]) =>
+                f(inputChanged || outputChanged)
+            }
+        }
+        Try(tracker(cacheKeyArgs)(() => pathsFilesInfo)).recover {
+          // in sbt 1.x, this is not necessary as any exception thrown during stamping is already silently ignored,
+          // but having this here helps keeping code as common as possible
+          // https://github.com/sbt/util/blob/v1.0.0/util-tracking/src/main/scala/sbt/util/Tracked.scala#L180
+          case _ @StampingImpossible => f(true)
+        }.get
       }
 
-      val errors = interface.run()
-      if (errors.nonEmpty) {
-        throw new ScalafixFailed(errors.toList)
+      diffWithPreviousRun { changed =>
+        if (changed) {
+          //FIXME: only run on modified/new files, ignore deletions
+          val errors = interface.run()
+          if (errors.nonEmpty) throw new ScalafixFailed(errors.toList)
+        } else {
+          streams.log.debug(s"already ran on ${paths.length} files")
+        }
+        ()
       }
+
     } else {
       () // do nothing
     }

src/sbt-test/sbt-scalafix/caching/build.sbt

@@ -0,0 +1,11 @@
+import _root_.scalafix.sbt.{BuildInfo => Versions}
+
+inThisBuild(
+  List(
+    scalaVersion := Versions.scala212,
+    addCompilerPlugin(scalafixSemanticdb),
+    scalacOptions += "-Yrangepos",
+    scalacOptions += "-Ywarn-unused", // for RemoveUnused
+    scalafixCaching := true
+  )
+)

src/sbt-test/sbt-scalafix/caching/files/DisableSyntaxNull.scalafix.conf

@@ -0,0 +1,2 @@
+rules = [DisableSyntax]
+DisableSyntax.noNulls = true

src/sbt-test/sbt-scalafix/caching/files/DisableSyntaxVar.scalafix.conf

@@ -0,0 +1,2 @@
+rules = [DisableSyntax]
+DisableSyntax.noVars = true

src/sbt-test/sbt-scalafix/caching/files/Null.scala

@@ -0,0 +1,3 @@
+object Null {
+  println(null)
+}