diff --git a/api/python/ai/chronon/airflow_helpers.py b/api/python/ai/chronon/airflow_helpers.py index f5b699f3b7..38f561be7e 100644 --- a/api/python/ai/chronon/airflow_helpers.py +++ b/api/python/ai/chronon/airflow_helpers.py @@ -73,8 +73,11 @@ def _get_airflow_deps_from_source(source, partition_column=None): def extract_default_partition_column(obj): - return obj.metaData.executionInfo.conf.common.get("spark.chronon.partition.column") - + try: + return obj.metaData.executionInfo.conf.common.get("spark.chronon.partition.column") + except Exception: + # Error handling occurs in `create_airflow_dependency` + return None def _set_join_deps(join): default_partition_col = extract_default_partition_column(join) diff --git a/api/python/test/canary/teams.py b/api/python/test/canary/teams.py index 27a45a99d5..8cdc41e77f 100644 --- a/api/python/test/canary/teams.py +++ b/api/python/test/canary/teams.py @@ -6,6 +6,11 @@ description="Default team", email="ml-infra@.com", # TODO: Infra team email outputNamespace="default", + conf=ConfigProperties( + common={ + "spark.chronon.partition.column": "ds", + } + ), env=EnvironmentVariables( common={ "VERSION": "latest", diff --git a/api/thrift/agent.thrift b/api/thrift/agent.thrift index 099795eaf3..2fcd37c209 100644 --- a/api/thrift/agent.thrift +++ b/api/thrift/agent.thrift @@ -1,4 +1,5 @@ namespace java ai.chronon.api +include "common.thrift" // TODO: Need to brainstorm and make necessary changes. just a starting point to unblock other work. struct YarnAutoScalingSpec { @@ -33,7 +34,7 @@ struct YarnJob { 1: optional string appName 2: optional YarnJobType jobType - 10: optional list args + 10: optional list argsList 11: optional map env 12: optional map conf // creates local file with this name and contents - relative to cwd @@ -121,13 +122,9 @@ struct JobInfo { 10: optional YarnIncrementalJobStatus yarnIncrementalStatus } -struct DatePartitionRange { - 1: optional string start - 2: optional string end -} struct PartitionListingPutRequest { - 1: optional map> partitions + 1: optional map> partitions 2: optional map errors }