Skip to content

Commit b7b1836

Browse files
authored
Branch versioning logic (#163)
## Summary Writing down the cases & logic behind branching. Will continue writing about how these are internally represented. ## Cheour clientslist - [x] Added Unit Tests - [ ] Covered by existing CI - [ ] Integration tested - [x] Documentation update <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit ## Release Notes - **Bug Fixes** - Corrected a typo in `TableDependency` struct, renaming `forceComputae` to `forceCompute`. - **New Features** - Added support for branch-based workflows in data orchestration. - Introduced `RepoIndex` for managing repository data and versions. - Created `SequenceMap` utility for managing unique value sequences. - Added `TablePrinter` for formatted data output. - Introduced `VersionUpdate` class for traour clientsing version changes. - Added `StringExtensions` for MD5 hashing of strings. - **Refactoring** - Removed `LineageIndex` and `LogicalSet` classes. - Updated naming conventions for logical nodes. - Restructured repository parsing and version traour clientsing mechanisms. - **Documentation** - Added comprehensive README for branch workflow management. - Expanded documentation for global planning and batch workload processing. - **Chores** - Updated logging dependencies. - Modified build configuration. - Added new test specifications. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent 65a36f2 commit b7b1836

File tree

3 files changed

+14
-34
lines changed

3 files changed

+14
-34
lines changed

api/py/ai/chronon/utils.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,7 @@ def join_part_output_table_name(join, jp, full_name: bool = False):
262262
def partOutputTable(jp: JoinPart): String = (Seq(join.metaData.outputTable) ++ Option(jp.prefix) :+
263263
jp.groupBy.metaData.cleanName).mkString("_")
264264
"""
265+
print(join)
265266
if not join.metaData.name and isinstance(join, api.Join):
266267
__set_name(join, api.Join, "joins")
267268
return "_".join(

api/thrift/orchestration.thrift

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -82,18 +82,6 @@ struct NodeInfo {
8282
30: optional LogicalNode conf
8383
}
8484

85-
86-
/** First Pass
87-
* NodeInstance::(name, type, conf_hash) -> #[parent_nodes]
88-
* Node::(name, type) -> #[conf_hash]
89-
90-
* Second Pass
91-
* Node::(name, type, compute_hash) -> #[parent_nodes]
92-
93-
* different file_hashes but same lineage_hash should all go into the same orchestrator workflow
94-
* Node::(name, type, lineage_hash)
95-
**/
96-
9785
struct NodeConnections {
9886
1: optional list<NodeKey> parents
9987
2: optional list<NodeKey> children
@@ -272,7 +260,7 @@ struct TableDependency {
272260
* JoinParts could use data from batch baour clientsfills or upload tables when available
273261
* When not available they shouldn't force computation of the baour clientsfills and upload tables.
274262
**/
275-
21: optional bool forceComputae
263+
21: optional bool forceCompute
276264
}
277265

278266
union Dependency {

build.sbt

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,9 @@ lazy val api = project
136136
"org.scala-lang.modules" %% "scala-collection-compat" % "2.11.0",
137137
"com.novocode" % "junit-interface" % "0.11" % "test",
138138
"org.scalatest" %% "scalatest" % "3.2.19" % "test",
139-
"org.scalatestplus" %% "moour clientsito-3-4" % "3.2.10.0" % "test"
139+
"org.scalatestplus" %% "moour clientsito-3-4" % "3.2.10.0" % "test",
140+
// needed by thrift
141+
"org.slf4j" % "slf4j-api" % slf4jApiVersion,
140142
)
141143
)
142144

@@ -410,33 +412,22 @@ lazy val hub = (project in file("hub"))
410412
}
411413
)
412414

413-
val scala_test = "org.scalatest" %% "scalatest" % "3.2.19" % "test"
414-
val sl4j = "org.slf4j" % "slf4j-api" % slf4jApiVersion
415-
val logbaour clients = "ch.qos.logbaour clients" % "logbaour clients-classic" % logbaour clientsClassicVersion
416-
val commonDependencies = Seq(
417-
scala_test,
418-
sl4j,
419-
logbaour clients
420-
)
421415

422416
// orchestrator
423417
lazy val orchestration = project
424418
.dependsOn(online.%("compile->compile;test->test"))
425419
.settings(
426420
assembly / mainClass := Some("ai.chronon.orchestration.RepoParser"),
421+
427422
Compile / run / mainClass := Some("ai.chronon.orchestration.RepoParser"),
428-
assembly / assemblyMergeStrategy := {
429-
case "log4j2.properties" => MergeStrategy.first
430-
case "META-INF/log4j-provider.properties" => MergeStrategy.first
431-
case PathList("org", "apache", "logging", "log4j", "core", "config", "plugins", "Log4j2Plugins.dat") =>
432-
MergeStrategy.first
433-
case x => (assembly / assemblyMergeStrategy).value(x)
434-
},
435-
libraryDependencies ++= commonDependencies ++ Seq(
436-
"org.apache.logging.log4j" % "log4j-api" % log4j2_version,
437-
"org.apache.logging.log4j" % "log4j-core" % log4j2_version,
438-
"org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2_version
439-
)
423+
Compile / unmanagedResourceDirectories += baseDirectory.value / "src" / "main" / "resources",
424+
425+
libraryDependencies ++= Seq(
426+
"org.apache.logging.log4j" %% "log4j-api-scala" % "13.1.0",
427+
"org.apache.logging.log4j" % "log4j-core" % "2.20.0",
428+
// "org.slf4j" % "slf4j-api" % slf4jApiVersion,
429+
"org.scalatest" %% "scalatest" % "3.2.19" % "test",
430+
),
440431
)
441432

442433
ThisBuild / assemblyMergeStrategy := {

0 commit comments

Comments
 (0)