Skip to content

Commit 8ec4a98

Browse files
heerenerheerener
committed
Infra bucket as env var
1 parent 1c30484 commit 8ec4a98

File tree

6 files changed

+25
-17
lines changed

6 files changed

+25
-17
lines changed

‎hpc_provisioner/src/hpc_provisioner/config-dev/_head_node.tpl.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,16 @@ Iam:
1010
AdditionalIamPolicies:
1111
- Policy: !config fsx_policy_arn
1212
S3Access:
13-
- BucketName: sboinfrastructureassets-sandbox
13+
- BucketName: !config infra_assets_bucket
1414
EnableWriteAccess: false
1515
- BucketName: sbosandbox-le3cn
1616
EnableWriteAccess: false
1717
CustomActions:
1818
OnNodeConfigured:
1919
Sequence:
20-
- Script: s3://sboinfrastructureassets-sandbox/scripts/create_users.py
20+
- Script: !config create_users_script
2121
Args: !config create_users_args
22-
- Script: s3://sboinfrastructureassets-sandbox/scripts/environment.sh
22+
- Script: !config environment_script
2323
Args: !config environment_args
2424
# - Script: s3://sboinfrastructureassets/scripts/all_or_nothing_allocation.sh
2525
# - Script: s3://sboinfrastructureassets/scripts/setup_lustre_directories.py

‎hpc_provisioner/src/hpc_provisioner/config-dev/_slurm_queues.tpl.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
MaxTime: 720
5555
Iam:
5656
S3Access:
57-
- BucketName: sboinfrastructureassets-sandbox
57+
- BucketName: !config infra_assets_bucket
5858
# CustomActions:
5959
# OnNodeConfigured:
6060
# Sequence:
@@ -96,7 +96,7 @@
9696
MaxTime: 720
9797
Iam:
9898
S3Access:
99-
- BucketName: sboinfrastructureassets-sandbox
99+
- BucketName: !config infra_assets_bucket
100100
# CustomActions:
101101
# OnNodeConfigured:
102102
# Sequence:
@@ -131,7 +131,7 @@
131131
MaxTime: 720
132132
Iam:
133133
S3Access:
134-
- BucketName: sboinfrastructureassets-sandbox
134+
- BucketName: !config infra_assets_bucket
135135

136136
# ==============================================================
137137
# prod-mpi-mem queue, for tightly-coupled, memory-intensive jobs
@@ -166,7 +166,7 @@
166166
MaxTime: 720
167167
Iam:
168168
S3Access:
169-
- BucketName: sboinfrastructureassets-sandbox
169+
- BucketName: !config infra_assets_bucket
170170

171171
# ====================================
172172
# prod-batch queue, for benchmarking
@@ -202,7 +202,7 @@
202202
MaxTime: 120
203203
Iam:
204204
S3Access:
205-
- BucketName: sboinfrastructureassets-sandbox
205+
- BucketName: !config infra_assets_bucket
206206
# CustomActions:
207207
# OnNodeConfigured:
208208
# Sequence:

‎hpc_provisioner/src/hpc_provisioner/config/_head_node.tpl.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,16 @@ Iam:
1010
AdditionalIamPolicies:
1111
- Policy: !config fsx_policy_arn
1212
S3Access:
13-
- BucketName: sboinfrastructureassets-sandbox
13+
- BucketName: !config infra_assets_bucket
1414
EnableWriteAccess: false
1515
- BucketName: sbosandbox-le3cn
1616
EnableWriteAccess: false
1717
CustomActions:
1818
OnNodeConfigured:
1919
Sequence:
20-
- Script: s3://sboinfrastructureassets-sandbox/scripts/create_users.py
20+
- Script: !config create_users_script
2121
Args: !config create_users_args
22-
- Script: s3://sboinfrastructureassets-sandbox/scripts/environment.sh
22+
- Script: !config environment_script
2323
Args: !config environment_args
2424
# - Script: s3://sboinfrastructureassets/scripts/all_or_nothing_allocation.sh
2525
# - Script: s3://sboinfrastructureassets/scripts/setup_lustre_directories.py

‎hpc_provisioner/src/hpc_provisioner/config/_slurm_queues.tpl.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
MaxTime: 720
5555
Iam:
5656
S3Access:
57-
- BucketName: sboinfrastructureassets-sandbox
57+
- BucketName: !config infra_assets_bucket
5858
# CustomActions:
5959
# OnNodeConfigured:
6060
# Sequence:
@@ -96,7 +96,7 @@
9696
MaxTime: 720
9797
Iam:
9898
S3Access:
99-
- BucketName: sboinfrastructureassets-sandbox
99+
- BucketName: !config infra_assets_bucket
100100
# CustomActions:
101101
# OnNodeConfigured:
102102
# Sequence:
@@ -131,7 +131,7 @@
131131
MaxTime: 720
132132
Iam:
133133
S3Access:
134-
- BucketName: sboinfrastructureassets-sandbox
134+
- BucketName: !config infra_assets_bucket
135135

136136
# ==============================================================
137137
# prod-mpi-mem queue, for tightly-coupled, memory-intensive jobs
@@ -166,7 +166,7 @@
166166
MaxTime: 720
167167
Iam:
168168
S3Access:
169-
- BucketName: sboinfrastructureassets-sandbox
169+
- BucketName: !config infra_assets_bucket
170170

171171
# ====================================
172172
# prod-batch queue, for benchmarking
@@ -202,7 +202,7 @@
202202
MaxTime: 120
203203
Iam:
204204
S3Access:
205-
- BucketName: sboinfrastructureassets-sandbox
205+
- BucketName: !config infra_assets_bucket
206206
# CustomActions:
207207
# OnNodeConfigured:
208208
# Sequence:

‎hpc_provisioner/src/hpc_provisioner/pcluster_manager.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
get_containers_bucket,
4242
get_efa_security_group_id,
4343
get_fsx_policy_arn,
44+
get_infra_bucket,
4445
get_sbonexusdata_bucket,
4546
get_scratch_bucket,
4647
)
@@ -94,6 +95,9 @@ def populate_config(
9495
CONFIG_VALUES["create_users_args"] = create_users_args
9596
CONFIG_VALUES["environment_args"] = [get_cluster_name(vlab_id, project_id)]
9697
CONFIG_VALUES["ami_id"] = get_ami_id()
98+
CONFIG_VALUES["infra_assets_bucket"] = get_infra_bucket()
99+
CONFIG_VALUES["create_users_script"] = f"s3://{get_infra_bucket()}/scripts/create_users.py"
100+
CONFIG_VALUES["environment_script"] = f"s3://{get_infra_bucket()}/scripts/environment.sh"
97101
logger.debug(f"Config values: {CONFIG_VALUES}")
98102

99103

@@ -213,7 +217,7 @@ def pcluster_create(
213217
if benchmark:
214218
pcluster_config["HeadNode"]["CustomActions"]["OnNodeConfigured"]["Sequence"].append(
215219
{
216-
"Script": "s3://sboinfrastructureassets-sandbox/scripts/80_cloudwatch_agent_config_prolog.sh",
220+
"Script": f"s3://{get_infra_bucket()}/scripts/80_cloudwatch_agent_config_prolog.sh",
217221
"Args": [cluster_name],
218222
}
219223
)

‎hpc_provisioner/src/hpc_provisioner/utils.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ def get_containers_bucket() -> str:
1818
return _get_env_var("CONTAINERS_BUCKET")
1919

2020

21+
def get_infra_bucket() -> str:
22+
return _get_env_var("INFRA_ASSETS_BUCKET")
23+
24+
2125
def get_scratch_bucket() -> str:
2226
return _get_env_var("SCRATCH_BUCKET")
2327

0 commit comments

Comments
 (0)