From 4d95af48f059d140634d96d1997ce062f4e3c77e Mon Sep 17 00:00:00 2001 From: ezvz Date: Tue, 15 Apr 2025 20:35:07 -0700 Subject: [PATCH 1/4] WIP --- api/python/test/canary/teams.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/python/test/canary/teams.py b/api/python/test/canary/teams.py index 27a45a99d5..8cdc41e77f 100644 --- a/api/python/test/canary/teams.py +++ b/api/python/test/canary/teams.py @@ -6,6 +6,11 @@ description="Default team", email="ml-infra@.com", # TODO: Infra team email outputNamespace="default", + conf=ConfigProperties( + common={ + "spark.chronon.partition.column": "ds", + } + ), env=EnvironmentVariables( common={ "VERSION": "latest", From f087fab710e387c51b480872df818304ac247bbb Mon Sep 17 00:00:00 2001 From: ezvz Date: Tue, 15 Apr 2025 20:39:20 -0700 Subject: [PATCH 2/4] WIP --- api/python/ai/chronon/airflow_helpers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/python/ai/chronon/airflow_helpers.py b/api/python/ai/chronon/airflow_helpers.py index f5b699f3b7..38f561be7e 100644 --- a/api/python/ai/chronon/airflow_helpers.py +++ b/api/python/ai/chronon/airflow_helpers.py @@ -73,8 +73,11 @@ def _get_airflow_deps_from_source(source, partition_column=None): def extract_default_partition_column(obj): - return obj.metaData.executionInfo.conf.common.get("spark.chronon.partition.column") - + try: + return obj.metaData.executionInfo.conf.common.get("spark.chronon.partition.column") + except Exception: + # Error handling occurs in `create_airflow_dependency` + return None def _set_join_deps(join): default_partition_col = extract_default_partition_column(join) From d435bf8edfa9d89a49d3e3534440a8a3c5fe7822 Mon Sep 17 00:00:00 2001 From: ezvz Date: Tue, 15 Apr 2025 21:37:09 -0700 Subject: [PATCH 3/4] WIP --- api/thrift/agent.thrift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/thrift/agent.thrift b/api/thrift/agent.thrift index 099795eaf3..d3ffa77bc6 100644 --- a/api/thrift/agent.thrift +++ b/api/thrift/agent.thrift @@ -33,7 +33,7 @@ struct YarnJob { 1: optional string appName 2: optional YarnJobType jobType - 10: optional list args + 10: optional list argsList 11: optional map env 12: optional map conf // creates local file with this name and contents - relative to cwd From 97d4a82d8eb11d4c83970ae4e82aea88fe8fc39b Mon Sep 17 00:00:00 2001 From: ezvz Date: Tue, 15 Apr 2025 21:39:06 -0700 Subject: [PATCH 4/4] WIP --- api/thrift/agent.thrift | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api/thrift/agent.thrift b/api/thrift/agent.thrift index d3ffa77bc6..2fcd37c209 100644 --- a/api/thrift/agent.thrift +++ b/api/thrift/agent.thrift @@ -1,4 +1,5 @@ namespace java ai.chronon.api +include "common.thrift" // TODO: Need to brainstorm and make necessary changes. just a starting point to unblock other work. struct YarnAutoScalingSpec { @@ -121,13 +122,9 @@ struct JobInfo { 10: optional YarnIncrementalJobStatus yarnIncrementalStatus } -struct DatePartitionRange { - 1: optional string start - 2: optional string end -} struct PartitionListingPutRequest { - 1: optional map> partitions + 1: optional map> partitions 2: optional map errors }