WIP

Author: ezvz

api/thrift/orchestration.thrift

@@ -39,7 +39,60 @@ enum LogicalType {
     TABULAR_DATA = 5
 }
 
+struct NodeKey {
+    1: optional string name
 
+    2: optional LogicalType logicalType
+    3: optional PhysicalNodeType physicalType
+
+    /**
+    * represents the computation of the node including the computation of all its parents
+    * direct and indirect changes that change output will affect lineage hash
+    **/
+    10: optional string lineageHash
+}
+
+struct NodeInfo {
+    /**
+    * represents the computation that a node does
+    * direct changes to conf that change output will affect semantic hash
+    * changing spark params etc shouldn't affect this
+    **/
+    11: optional string semanticHash
+
+    /**
+    * simple hash of the entire conf (that is TSimpleJsonProtocol serialized),
+    * computed by cli and used to check if new conf_json need to be pushed from user's machine
+    **/
+    12: optional string confHash
+
+    /**
+    * when new/updated conf's are pushed the branch is also set from the cli
+    * upon merging the branch will be unset
+    **/
+    20: optional string branch
+
+    /**
+    * will be set to the author of the last semantic change to node
+    * (non-semantic changes like code-mods or spark params don't affect this)
+    **/
+    21: optional string author
+
+    /**
+    * contents of the conf itself
+    **/
+    30: optional LogicalNode conf
+}
+
+struct NodeConnections {
+    1: optional list<NodeKey> parents
+    2: optional list<NodeKey> children
+}
+
+struct NodeGraph {
+    1: optional map<NodeKey, NodeConnections> connections
+    2: optional map<NodeKey, NodeInfo> infoMap
+}
 
 
 
@@ -96,12 +149,12 @@ union PhysicalNodeType {
 }
 
 struct PhysicalNode {
-    1: required string name,
-    2: required PhysicalNodeType nodeType,
-    3: required LogicalNode logicalNode,
-    4: required string confHash,
-    5: required list<common.TableDependency> tableDependencies,
-    6: required list<string> outputColumns,
+    1: required string name
+    2: required PhysicalNodeType nodeType
+    3: required LogicalNode logicalNode
+    4: required string confHash
+    5: required list<common.TableDependency> tableDependencies
+    6: required list<string> outputColumns
     7: required string output_table
 }