BioNeMo Sub-Package Workflow #170
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: BioNeMo Sub-Package Workflow | |
| on: | |
| # To test or publish sub-packages or adjustments to this workflow that are branched in PR's, manually dispatch this workflow on the PR's branch here: https://github.com/NVIDIA/bionemo-framework/actions/workflows/bionemo-subpackage-ci.yml. | |
| workflow_dispatch: | |
| inputs: | |
| subpackages: | |
| description: "BioNeMo sub-packages (comma-separated) to test or publish." | |
| required: true | |
| type: string | |
| test: | |
| description: "Test the sub-packages before publishing to PyPI. Strongly recommended for production releases to PyPI. Can be disabled when staging sub-packages on Test PyPI or publishing circular dependencies to PyPI." | |
| required: false | |
| type: boolean | |
| default: true | |
| publish: | |
| description: "Publish the built package to PyPI. If testing is specified, requires that all sub-package tests succeed based on dependencies published to Test PyPI or PyPI." | |
| required: false | |
| type: boolean | |
| default: false | |
| pypi: | |
| description: "Publish to PyPI instead of Test PyPI." | |
| required: false | |
| type: boolean | |
| default: false | |
| version_overwrite: | |
| description: "Overwrite the published version of the sub-package. (Sets skip-existing to False. Requires deleting existing wheels and other artifacts on PyPI.)" | |
| required: false | |
| type: boolean | |
| default: false | |
| build_framework: | |
| description: "Build framework to use for building and publishing." | |
| type: choice | |
| options: | |
| - "python" | |
| - "rust_pyo3_maturin" | |
| default: "python" | |
| required: true | |
| python_version: | |
| description: "Python version to use for testing and publishing." | |
| required: false | |
| type: string | |
| default: "3.12" | |
| gpu_runner: | |
| description: "Specify a GPU runner for testing on NVIDIA GitHub Actions. (For a list of available runners, refer to: https://docs.gha-runners.nvidia.com/runners/)" | |
| required: false | |
| type: string | |
| default: "linux-amd64-gpu-l4-latest-1" | |
| cuda_version: | |
| description: "NVIDIA CUDA container version to use for testing." | |
| required: false | |
| type: string | |
| default: "nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04" | |
| jobs: | |
| configure-workflow-packages: | |
| name: "[Configure Workflow Packages] Identify sub-packages for testing and publishing." | |
| runs-on: ubuntu-latest | |
| outputs: | |
| workflow_packages: ${{ steps.parse-dispatch-packages.outputs.dispatch_packages }} | |
| steps: | |
| - id: parse-dispatch-packages | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| name: Parse the sub-packages specified in the workflow dispatch. | |
| run: | | |
| # Send the parsed list of sub-packages to the next job. | |
| dispatch_packages=$(echo '${{ github.event.inputs.subpackages }}' | jq -R -c 'split(",")') | |
| echo "dispatch_packages=$dispatch_packages" >> "$GITHUB_OUTPUT" | |
| echo "[BioNeMo Sub-Package CI] Sub-packages to stage: $dispatch_packages" | |
| install-and-test: | |
| needs: configure-workflow-packages | |
| # Check if the previous job has any staged packages to test and publish. | |
| if: ${{ needs.configure-workflow-packages.outputs.workflow_packages != '[]' }} | |
| strategy: | |
| matrix: | |
| package: ${{ fromJson(needs.configure-workflow-packages.outputs.workflow_packages) }} | |
| fail-fast: false # Prevent all matrix jobs from failing if one fails. | |
| name: "[${{ matrix.package }}] Install and test sub-package." | |
| # Use GPU runner only when testing, otherwise use a standard runner. | |
| runs-on: ${{ github.event.inputs.test == 'true' && github.event.inputs.gpu_runner || 'ubuntu-latest' }} | |
| container: | |
| # GPU jobs must run in a container. Use a fresh CUDA base container for package installation and testing. | |
| # If testing is disabled, use a lightweight container to quickly skip this job. | |
| image: ${{ github.event.inputs.test == 'true' && github.event.inputs.cuda_version || 'ubuntu:latest' }} | |
| steps: | |
| # Silently skip all steps if testing is disabled, which does not block building or publishing. | |
| - name: Install git and system dependencies. | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| run: | | |
| apt-get update | |
| apt-get install -qyy git curl lsb-release build-essential | |
| - uses: actions/checkout@v4 | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| with: | |
| fetch-depth: 0 | |
| submodules: "recursive" | |
| - uses: actions/setup-python@v5 | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| with: | |
| python-version: ${{ github.event.inputs.python_version }} | |
| - id: install-rust | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| name: Install Rust. | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| . $HOME/.cargo/env | |
| rustc --version | |
| cargo --version | |
| rustup --version | |
| - id: install-subpackage-core | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| name: Install sub-package. | |
| run: | | |
| # Setup environment, i.e. add Rust to PATH and silence pip root user warnings. | |
| . $HOME/.cargo/env | |
| # Install sub-package and dependencies. | |
| pip install --upgrade pip setuptools uv maturin | |
| # Install required core & optional [test] dependencies. | |
| uv pip install --no-cache --no-build-isolation --system pytest sub-packages/${{ matrix.package }}[test] | |
| - id: install-subpackage-post | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| name: Install sub-package dependencies that need to be installed after the core dependencies. | |
| run: | | |
| # DEV: Post-install dependencies are configured in [project.optional-dependencies]. | |
| # `uv pip install --extra <optional-dependency> -r <pyproject.toml>` tracks | |
| # post-dependencies in the pyproject.toml and avoids installing core dependencies | |
| # redundantly, which causes errors with incompatible --config-setting. | |
| # TransformerEngine | |
| uv pip install --no-cache --no-build-isolation --system --extra te -r sub-packages/${{ matrix.package }}/pyproject.toml || echo "[BioNeMo Sub-Package CI] TE will not be installed." | |
| # # Apex | |
| # # NOTE: --cpp_ext and --cuda_ext are required for building fused Apex kernels. | |
| # uv pip install --no-cache --no-build-isolation --system --config-setting="--build-option=--cpp_ext" --config-setting="--build-option=--cuda_ext" --extra apex -r sub-packages/${{ matrix.package }}/pyproject.toml || echo "[BioNeMo Sub-Package CI] Apex will not be installed." | |
| - id: test-dispatch-subpackage | |
| if: ${{ github.event.inputs.test == 'true' }} | |
| name: Test sub-package. | |
| run: pytest -vv sub-packages/${{ matrix.package }} | |
| build-pypi: | |
| # Build distributions from either the workflow dispatch or PR. | |
| # Validate building before merging or publishing. | |
| needs: [configure-workflow-packages, install-and-test] | |
| if: ${{ needs.configure-workflow-packages.outputs.workflow_packages != '[]' && github.event.inputs.publish == 'true' }} | |
| outputs: | |
| staged_packages: ${{ needs.configure-workflow-packages.outputs.workflow_packages }} | |
| strategy: | |
| matrix: | |
| package: ${{ fromJson(needs.configure-workflow-packages.outputs.workflow_packages) }} | |
| fail-fast: false # Prevent all matrix jobs from failing if one fails. | |
| name: "[${{ matrix.package }}] Build the sub-package." | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ github.event.inputs.python_version }} | |
| - id: build-package | |
| name: Build a binary wheel and a source tarball for the sub-package. | |
| run: | | |
| if [[ "${{ github.event.inputs.test }}" != "true" && "${{ github.event.inputs.version_overwrite }}" != "true" ]]; then | |
| # For untested sub-packages, append '-dev' to the version for PyPI. | |
| sed -i 's/[[:space:]]*$//' sub-packages/${{ matrix.package }}/VERSION | |
| sed -i 's/$/-dev/' sub-packages/${{ matrix.package }}/VERSION | |
| fi | |
| # Build the sub-package. | |
| if [[ "${{ github.event.inputs.build_framework }}" == "python" ]]; then | |
| pip install build | |
| python -m build sub-packages/${{ matrix.package }} | |
| elif [[ "${{ github.event.inputs.build_framework }}" == "rust_pyo3_maturin" ]]; then | |
| # Install maturin[zig] to build the Rust sub-package with compatibility for manylinux_X_Y using zig. | |
| pip install maturin[zig] | |
| maturin build --release --zig -m sub-packages/${{ matrix.package }}/Cargo.toml | |
| fi | |
| - id: upload-distribution | |
| name: Upload distribution packages to the workflow. | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.package }}-build-artifacts | |
| path: ${{ github.event.inputs.build_framework == 'rust_pyo3_maturin' && format('sub-packages/{0}/target/wheels', matrix.package) || format('sub-packages/{0}/dist', matrix.package) }} | |
| publish-to-pypi: | |
| needs: [build-pypi, install-and-test] | |
| # Require staged sub-package builds for publishing to PyPI. | |
| if: ${{ needs.build-pypi.result == 'success' }} | |
| strategy: | |
| matrix: | |
| package: ${{ fromJson(needs.build-pypi.outputs.staged_packages) }} | |
| fail-fast: false # Prevent all matrix jobs from failing if one fails. | |
| name: Publish ${{ matrix.package }} to PyPI. | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: ${{ github.event.inputs.pypi && 'pypi' || 'testpypi' }} | |
| url: ${{ github.event.inputs.pypi && format('https://pypi.org/p/{0}', matrix.package) || format('https://test.pypi.org/p/{0}', matrix.package) }} | |
| permissions: | |
| id-token: write | |
| steps: | |
| - id: download-distribution | |
| name: Download the built distribution. | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ matrix.package }}-build-artifacts | |
| path: sub-packages/${{ matrix.package }}/dist | |
| - id: publish-to-testpypi | |
| name: Publish distribution 📦 to Test PyPI for PR. | |
| if: ${{ github.event.inputs.pypi == 'false' }} | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| verbose: true | |
| packages-dir: sub-packages/${{ matrix.package }}/dist | |
| repository-url: https://test.pypi.org/legacy/ | |
| skip-existing: ${{ github.event.inputs.version_overwrite }} | |
| - id: publish-to-pypi | |
| name: Publish distribution 📦 to PyPI for Workflow Dispatch. | |
| # To require testing before publishing to PyPI, add: ... && needs.install-and-test.result == 'success' | |
| # If testing is run but fails, the workflow will fail and not publish to PyPI (or Test PyPI). | |
| # We strongly recommend testing when publishing to production PyPI. | |
| if: ${{ github.event.inputs.pypi == 'true' }} | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| verbose: true | |
| packages-dir: sub-packages/${{ matrix.package }}/dist | |
| skip-existing: ${{ github.event.inputs.version_overwrite }} |