tests

Author: nikhil-zlai

orchestration/src/main/resources/log4j2.properties

@@ -7,7 +7,7 @@ appender.console.type = Console
 appender.console.name = console
 appender.console.target = SYSTEM_OUT
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %yellow{%d{yyyy/MM/dd HH:mm:ss}} %highlight{%-5level} %green{%file:%line} - %message%n
+appender.console.layout.pattern = %cyan{%d{yyyy/MM/dd HH:mm:ss}} %highlight{%-5level} %magenta{%file:%line} - %message%n
 
 # Configure specific logger
 logger.chronon.name = ai.chronon

orchestration/src/main/scala/ai/chronon/orchestration/RepoIndex.scala

@@ -78,10 +78,10 @@ class RepoIndex[T](proc: ConfProcessor[T]) extends Logging {
 
     val newVersions = mutable.Map.empty[Name, Version]
 
-    fileHashes.foreach {
-      case (name, _) =>
-        val globalHash = computeGlobalHash(name)
+    fileHashes.foreach { case (name, _) => computeGlobalHash(name) }
 
+    globalHashes.foreach {
+      case (name, globalHash) =>
         val versionIndex = versionSequencer.potentialIndex(name, globalHash)
         newVersions.update(name, Version("v" + versionIndex.toString))
     }
@@ -98,6 +98,13 @@ class RepoIndex[T](proc: ConfProcessor[T]) extends Logging {
         case (name, (fileHash, content)) => update(fileHashToContent, name, fileHash, content)
       }
 
+      val newVersions = globalHashes.map {
+        case (name, globalHash) =>
+          val versionIndex = versionSequencer.insert(name, globalHash)
+          val version = Version("v" + versionIndex.toString)
+          name -> version
+      }
+
       branchToFileHash.update(branch, fileHashes)
       branchVersionIndex.update(branch, newVersions)
 

orchestration/src/test/scala/ai/chronon/orchestration/test/RepoIndexSpec.scala

@@ -53,7 +53,7 @@ class RepoIndexSpec extends AnyFlatSpec with Matchers {
 
   "RepoIndex" should "propagate updates" in {
     val confs = Seq(
-      TestConf("t1", "", "", Seq.empty),
+      TestConf("t1", "", "", Seq("sq1")),
       TestConf("t2", "", "", Seq.empty),
       TestConf("sq1", "v1", "4g", Seq.empty, Seq("t1")),
       TestConf("gb1", "v1", "4g", Seq("t1")),
@@ -106,13 +106,54 @@ class RepoIndexSpec extends AnyFlatSpec with Matchers {
     val branchConfs2 = Seq(
       TestConf("t1", "", "", Seq("sq1")),
       TestConf("t2", "", "", Seq.empty),
-      TestConf("sq1", "v3", "4g", Seq.empty, Seq("t1")), // updated
+      TestConf("sq1", "v2", "4g", Seq.empty, Seq("t1")),
       TestConf("gb1", "v1", "4g", Seq("t1")),
-      TestConf("gb2", "v1", "4g", Seq("t2")),
+      TestConf("gb2", "v1", "8g", Seq("t2")),  // non-semantic update
       TestConf("j1",  "v1", "4g", Seq("gb1", "gb2"), Seq("table_j1")),
       TestConf("m1", "v1", "4g", Seq("j1"), Seq("table_m1")),
     )
+
     updateIndex(branchConfs2, testBranch)
+
+    val branchConfs3 = Seq(
+      TestConf("t1", "", "", Seq("sq1")),
+      TestConf("t2", "", "", Seq.empty),
+      TestConf("sq1", "v1", "4g", Seq.empty, Seq("t1")), // reverted back
+      TestConf("gb1", "v1", "4g", Seq("t1")),
+      TestConf("gb2", "v1", "8g", Seq("t2")),
+      TestConf("j1",  "v1", "4g", Seq("gb1", "gb2"), Seq("table_j1")),
+      TestConf("m1", "v1", "4g", Seq("j1"), Seq("table_m1")),
+    )
+    updateIndex(branchConfs3, testBranch)
+
+    val branchConfs4 = Seq(
+      TestConf("t1", "", "", Seq("sq1")),
+      // TestConf("t2", "", "", Seq.empty), // deleted
+      TestConf("sq1", "v1", "4g", Seq.empty, Seq("t1")), // reverted back
+      TestConf("gb1", "v1", "4g", Seq("t1")),
+      // TestConf("gb2", "v1", "8g", Seq("t2")), // deleted
+      TestConf("j1",  "v1", "4g", Seq("gb1"), Seq("table_j1")), // parent deleted
+      TestConf("m1", "v1", "4g", Seq("j1"), Seq("table_m1")),
+    )
+
+    updateIndex(branchConfs4, testBranch)
+
+    updateIndex(branchConfs4, Branch.main)
+
+    val branchConfs5 = Seq(
+      TestConf("t1", "", "", Seq("sq1")),
+      TestConf("t3", "", "", Seq("sq3")), // new
+      TestConf("t2", "", "", Seq.empty),
+      TestConf("sq1", "v1", "4g", Seq.empty, Seq("t1")), // reverted back
+      TestConf("sq3", "v1", "4g", Seq.empty, Seq("t3")),  // new
+      TestConf("gb1", "v1", "4g", Seq("t1")),
+      TestConf("gb3", "v1", "4g", Seq("t3")),
+      TestConf("gb2", "v1", "8g", Seq("t2")),
+      TestConf("j1",  "v1", "4g", Seq("gb1", "gb2", "gb3"), Seq("table_j1")), // parent reverted + new
+      TestConf("m1", "v1", "4g", Seq("j1"), Seq("table_m1")),
+    )
+
+    updateIndex(branchConfs5, Branch.main)
   }