ADLR/megatron-lm!3458 - ci: Benchmark release tests suite with TE2.2 on H100

Co-authored-by: Oliver Koenig <[email protected]>

Author: ko3n1g

tests/functional_tests/shell_test_utils/run_ci_test.sh

@@ -2,6 +2,12 @@
 
 set -exo pipefail
 
+# Increase soft limit for number of open files to match hard limit
+ulimit -Sn $(ulimit -Hn)
+
+# Increase soft limit for number of processes to match hard limit
+ulimit -Su $(ulimit -Hu)
+
 echo "------ARGUMENTS LIST --------"
 # Use eval to properly handle quoted arguments
 eval "set -- $@"
@@ -17,7 +23,7 @@ for ARGUMENT in "$@"; do
     VALUE="${VALUE#\'}"
 
     # Properly quote the value to preserve spaces and special characters
-    export "$KEY"="$VALUE"
+    export "$KEY"="$(eval echo $VALUE)"
     echo "$KEY=$VALUE"
 done
 echo "---------------------------------"
@@ -69,7 +75,7 @@ IS_NEMO_TEST=$([[ $(echo "$TRAINING_SCRIPT_PATH" | tr '[:upper:]' '[:lower:]') =
 export IS_NEMO_TEST
 
 # Adjust model_config for lightweight mode
-if [[ "$MODE" == "pretraining" ]]; then
+if [[ "$MODE" == "pretraining" && "$TEST_TYPE" != "release" ]]; then
     if [[ "$ENABLE_LIGHTWEIGHT_MODE" == "true" && "$IS_NEMO_TEST" == "true" ]]; then
         yq -i '.MODEL_ARGS."trainer.max_steps" = 2' $TRAINING_PARAMS_PATH
         TRAIN_ITERS=$(cat $TRAINING_PARAMS_PATH |
@@ -132,12 +138,9 @@ for i in $(seq 1 $N_REPEAT); do
 
     bash $ROOT_DIR/tests/functional_tests/shell_test_utils/_run_training.sh
 
-    IS_FROZEN_RESUME_BUT_NO_CHECKPOINT=$([[ "$TEST_TYPE" = "frozen-resume" && -z "$(ls -A "$_CHECKPOINT_LOAD_PATH" 2>/dev/null)" ]] && echo "true" || echo "false")
+    if [[ "$TEST_TYPE" = "frozen-resume" && -z "$(ls -A "$_CHECKPOINT_LOAD_PATH" 2>/dev/null)" ]]; then
+        echo "No frozen checkpoint found. Will skip second run."
 
-    if [[ "$IS_FROZEN_RESUME_BUT_NO_CHECKPOINT" == "true" && ${RECORD_CHECKPOINTS} != "true" ]]; then
-        echo "No frozen checkpoint found, but test type is frozen-resume. Will abort."
-        exit 1
-    elif [[ "$IS_FROZEN_RESUME_BUT_NO_CHECKPOINT" == "true" && ${RECORD_CHECKPOINTS} == "true" ]]; then
         export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH
         rm -rf "$CHECKPOINT_SAVE_PATH/iter_0000$TRAIN_ITERS"
         echo $((TRAIN_ITERS / 2)) >$CHECKPOINT_SAVE_PATH/latest_checkpointed_iteration.txt

tests/functional_tests/test_cases/bert/bert_release/golden_values_0.10.0_dgx_a100.json renamed to tests/functional_tests/test_cases/bert/bert_release/golden_values_0.10.0_dgx_a100_lts.json

@@ -21,7 +21,7 @@ MODEL_ARGS:
   --sequence-parallel: true
   --disable-bias-linear: true
   --micro-batch-size: 4
-  --rampup-batch-size: "384 384 97656250"
+  --rampup-batch-size: "[384 384 97656250]"
   --global-batch-size: 1152
   --train-samples: 19531250
   --manual-gc: true
@@ -81,7 +81,7 @@ MODEL_ARGS:
   --log-validation-ppl-to-tensorboard: true
   --log-throughput: true
   --log-interval: 100
-  --tensorboard-dir: ${OUTPUT_PATH}/tensorboard
+  --tensorboard-dir: ${TENSORBOARD_PATH}
   --wandb-project: megatron-core-release-runs
   --wandb-exp-name: ${WANDB_EXPERIMENT}
   # Add mixed precision args

tests/functional_tests/test_cases/bert/bert_release/golden_values_0.11.0_dgx_a100.json renamed to tests/functional_tests/test_cases/bert/bert_release/golden_values_0.11.0_dgx_a100_lts.json

@@ -68,8 +68,8 @@ MODEL_ARGS:
   --eval-iters: 32
   --eval-interval: 2000
   # Add checkpointing args
-  --load: ${OUTPUT_PATH}/checkpoints
-  --save: ${OUTPUT_PATH}/checkpoints
+  --save: ${CHECKPOINT_SAVE_PATH}
+  --load: ${CHECKPOINT_LOAD_PATH}
   --save-interval: 1000
   # Add initialization args
   --init-method-std: 0.0134
@@ -81,7 +81,7 @@ MODEL_ARGS:
   --log-validation-ppl-to-tensorboard: true
   --log-throughput: true
   --log-interval: 100
-  --tensorboard-dir: ${OUTPUT_PATH}/tensorboard
+  --tensorboard-dir: ${TENSORBOARD_PATH}
   --wandb-project: megatron-core-release-runs
   --wandb-exp-name: ${WANDB_EXPERIMENT}
   # Add mixed precision args