Gaudi3 tests (scheduled) #70
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Gaudi3 tests (scheduled) | |
| on: | |
| workflow_dispatch: | |
| schedule: # every day at 6 AM UTC | |
| - cron: "0 6 * * *" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| run-gaudi3-tests: | |
| runs-on: | |
| group: itac-bm-emr-gaudi3-dell-2gaudi | |
| container: | |
| image: docker://vault.habana.ai/gaudi-docker/1.21.1/ubuntu22.04/habanalabs/pytorch-installer-2.6.0:latest | |
| options: --runtime=habana --shm-size=64G --cap-add=sys_nice --env HABANA_VISIBLE_DEVICES | |
| env: | |
| OMPI_MCA_btl_vader_single_copy_mechanism: none | |
| PT_ENABLE_INT64_SUPPORT: 1 | |
| PT_HPU_LAZY_MODE: 0 | |
| RUN_SLOW: 1 | |
| steps: | |
| - name: HL-SMI (1) | |
| run: | | |
| hl-smi | |
| echo "HABANA_VISIBLE_DEVICES=${HABANA_VISIBLE_DEVICES}" | |
| echo "HABANA_VISIBLE_MODULES=${HABANA_VISIBLE_MODULES}" | |
| - name: Extract HPU visible modules | |
| id: add-modules | |
| run: | | |
| export HABANA_VISIBLE_MODULES=$(hl-smi -Q module_id -f csv,noheader | tr '\n' ',' | sed 's/,$//') | |
| echo "HABANA_VISIBLE_MODULES=${HABANA_VISIBLE_MODULES}" >> $GITHUB_ENV | |
| - name: HL-SMI (2) | |
| run: | | |
| hl-smi | |
| echo "HABANA_VISIBLE_DEVICES=${HABANA_VISIBLE_DEVICES}" | |
| echo "HABANA_VISIBLE_MODULES=${HABANA_VISIBLE_MODULES}" | |
| - name: Checkout to Accelerate | |
| uses: actions/checkout@v4 | |
| - name: Install Accelerate with Transformers & DeepSpeed | |
| run: | | |
| pip install -e .[testing] \ | |
| git+https://github.com/HabanaAI/[email protected] \ | |
| git+https://github.com/huggingface/transformers.git | |
| - name: Run CLI tests | |
| if: ${{ !cancelled() && (success() || failure()) }} | |
| run: | | |
| make test_cli | |
| - name: Run Core tests | |
| if: ${{ !cancelled() && (success() || failure()) }} | |
| run: | | |
| make test_core | |
| - name: Run Big Modeling tests | |
| if: ${{ !cancelled() && (success() || failure()) }} | |
| run: | | |
| make test_big_modeling | |
| - name: Run DeepSpeed integration tests | |
| if: ${{ !cancelled() && (success() || failure()) }} | |
| run: | | |
| make test_deepspeed | |
| - name: Run FSDP integration tests | |
| if: ${{ !cancelled() && (success() || failure()) }} | |
| run: | | |
| make test_fsdp | |
| - name: Run TP integration tests | |
| if: ${{ !cancelled() && (success() || failure()) }} | |
| run: | | |
| make test_tp | |
| - name: Run Examples tests | |
| if: ${{ !cancelled() && (success() || failure()) }} | |
| run: | | |
| make test_examples |