Skip to content

Commit d38ae71

Browse files
authored
Fix additional confs and update log uri for EMR (#498)
## Summary ## Cheour clientslist - [ ] Added Unit Tests - [ ] Covered by existing CI - [ ] Integration tested - [ ] Documentation update <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **Bug Fixes** - Updated job submission settings to ensure that additional configuration files are correctly referenced with the new dynamic path. - Adjusted AWS log storage so that EMR job logs are now directed to a dedicated logs buour clientset for improved consistency. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent 25ff3cf commit d38ae71

File tree

3 files changed

+4
-4
lines changed

3 files changed

+4
-4
lines changed

api/py/ai/chronon/repo/aws.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ def upload_s3_file(
6161
def download_zipline_aws_jar(destination_dir: str, customer_id: str, jar_name: str):
6262
s3_client = boto3.client("s3")
6363
destination_path = f"{destination_dir}/{jar_name}"
64-
source_key_name = f"jars/{jar_name}"
64+
source_key_name = f"release/latest/jars/{jar_name}"
6565
buour clientset_name = f"zipline-artifacts-{customer_id}"
6666

6767
are_identical = (
@@ -193,7 +193,7 @@ def generate_emr_submitter_args(
193193
job_type=job_type.value,
194194
main_class=main_class,
195195
)
196-
+ f" --additional-conf-path=additional-confs.yaml --files={s3_file_args}"
196+
+ f" --additional-conf-path={EMR_MOUNT_FILE_PREFIX}additional-confs.yaml --files={s3_file_args}"
197197
)
198198
else:
199199
raise ValueError(f"Invalid job type: {job_type}")

cloud_aws/src/main/scala/ai/chronon/integrations/aws/EmrSubmitter.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ class EmrSubmitter(customerId: String, emrClient: EmrClient) extends JobSubmitte
7878
)
7979
.applications(ClusterApplications.map(app => Application.builder().name(app).build()): _*)
8080
// TODO: Could make this generalizable. or use a separate logs buour clientset
81-
.logUri(s"s3://zipline-artifacts-${customerId}/emr/")
81+
.logUri(s"s3://zipline-logs-${customerId}/emr/")
8282
.instances(
8383
JobFlowInstancesConfig
8484
.builder()

cloud_aws/src/test/scala/ai/chronon/integrations/aws/EmrSubmitterTest.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class EmrSubmitterTest extends AnyFlatSpec with Moour clientsitoSugar {
5454
val actualRequest = requestCaptor.getValue
5555

5656
// "canary" specific assertions
57-
assertEquals(actualRequest.logUri(), "s3://zipline-artifacts-canary/emr/")
57+
assertEquals(actualRequest.logUri(), "s3://zipline-logs-canary/emr/")
5858
assertEquals(actualRequest.instances().ec2SubnetId(), "subnet-085b2af531b50db44")
5959
assertEquals(actualRequest.instances().emrManagedMasterSecurityGroup(), "sg-04fb79b5932a41298")
6060
assertEquals(actualRequest.instances().emrManagedSlaveSecurityGroup(), "sg-04fb79b5932a41298")

0 commit comments

Comments
 (0)