The cluster has no internet access, write the prolog script

Author: heerener

.github/workflows/release.yaml

@@ -67,6 +67,7 @@ jobs:
         run: |
           aws s3 cp provisioner_scripts/create_users.py s3://${{ secrets.S3_BUCKET }}/scripts/create_users.py
           aws s3 cp provisioner_scripts/environment.sh s3://${{ secrets.S3_BUCKET }}/scripts/environment.sh
+          aws s3 cp provisioner_scripts/80_cloudwatch_agent_config_prolog.sh s3://${{ secrets.S3_BUCKET }}/scripts/80_cloudwatch_agent_config_prolog.sh
       - name: Release
         uses: softprops/action-gh-release@v2
         if: github.ref == 'refs/heads/main'

hpc_provisioner/src/hpc_provisioner/config-dev/_head_node.tpl.yaml

@@ -21,6 +21,7 @@ CustomActions:
         Args: !config create_users_args
       - Script: s3://sboinfrastructureassets-sandbox/scripts/environment.sh
         Args: !config environment_args
+      - Script: s3://sboinfrastructureassets-sandbox/scripts/80_cloudwatch_agent_config_prolog.sh
 #       - Script: s3://sboinfrastructureassets/scripts/all_or_nothing_allocation.sh
 #       - Script: s3://sboinfrastructureassets/scripts/setup_lustre_directories.py
 #         Args:

provisioner_scripts/80_cloudwatch_agent_config_prolog.sh

@@ -0,0 +1,13 @@
+cat << _EOF_ >> /opt/slurm/etc/scripts/prolog.d/80_cloudwatch_agent_config_prolog.sh
+#!/bin/bash
+
+CWAGENT_CONFIG=/sbo/data/scratch/CWAgent_config_\$SLURM_CLUSTER_NAME.json
+
+if [ ! -f \$CWAGENT_CONFIG ]; then
+	echo "Create CWAGENT_CONFIG " \$CWAGENT_CONFIG
+	sed "s/\\$CLUSTER_NAME/\$SLURM_CLUSTER_NAME/g" /sbo/data/scratch/CWAgent_config_tpl.json > \$CWAGENT_CONFIG
+fi
+
+#sudo /opt/slurm/bin/srun --ntasks=\$NODES --ntasks-per-node=1 /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop
+sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/\$CWAGENT_CONFIG -s
+_EOF_

provisioner_scripts/environment.sh

@@ -3,5 +3,3 @@
 export CLUSTER_NAME=$1
 
 echo "CLUSTER_NAME=${CLUSTER_NAME}" >> /etc/environment
-
-curl --create-dirs --output /opt/slurm/etc/scripts/prolog.d/80_cloudwatch_agent_config_prolog.sh --create-file-mode 0755 https://raw.githubusercontent.com/openbraininstitute/hpc-resource-provisioner/refs/heads/slurm_prolog/provisioner_scripts/headnode/80_cloudwatch_agent_config_prolog.sh