Skip to content

[Validation] Use linux_job for linux workers #1199

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 22, 2022
Merged

[Validation] Use linux_job for linux workers #1199

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
db71e37
Use linux_job for linux workers
atalman Nov 22, 2022
f86d58b
Fix if statement
atalman Nov 22, 2022
0b8c2f3
Fix typo
atalman Nov 22, 2022
8cae5af
Fix cuda version check
atalman Nov 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 45 additions & 94 deletions .github/workflows/validate-linux-binaries.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ on:
description: "Channel to use (nightly, test, release, all)"
required: true
type: string
ref:
description: 'Reference to checkout, defaults to empty'
default: ""
required: false
type: string
workflow_dispatch:
inputs:
channel:
Expand All @@ -18,108 +23,54 @@ on:
- nightly
- test
- all
ref:
description: 'Reference to checkout, defaults to empty'
default: ""
required: false
type: string

jobs:
generate-linux-conda-matrix:
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
with:
package-type: conda
os: linux
channel: ${{ inputs.channel }}
generate-linux-wheel-matrix:
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
with:
package-type: wheel
os: linux
channel: ${{ inputs.channel }}
with-py311: enable
generate-linux-libtorch-matrix:
generate-linux-matrix:
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
with:
package-type: libtorch
package-type: all
os: linux
channel: ${{ inputs.channel }}

linux-conda:
needs: generate-linux-conda-matrix
strategy:
matrix:
${{ fromJson(needs.generate-linux-conda-matrix.outputs.matrix) }}
fail-fast: false
runs-on: ${{ matrix.validation_runner }}
steps:
- name: Validate binary conda
uses: pytorch/builder/.github/actions/validate-binary@main
with:
gpu_arch_type: ${{ matrix.gpu_arch_type }}
gpu_arch_ver: ${{ matrix.gpu_arch_version }}
installation: ${{ matrix.installation }}
python_version: ${{ matrix.python_version }}
desired_cuda: ${{ matrix.desired_cuda }}
dev_toolset: ''
package_type: conda
target_os: linux

linux-wheel:
needs: generate-linux-wheel-matrix
linux:
needs: generate-linux-matrix
strategy:
matrix:
${{ fromJson(needs.generate-linux-wheel-matrix.outputs.matrix) }}
matrix: ${{ fromJson(needs.generate-linux-matrix.outputs.matrix) }}
fail-fast: false
runs-on: ${{ matrix.validation_runner }}
steps:
- name: Checkout PyTorch builder
uses: actions/checkout@v2
- name: Validate binary wheel
uses: ./.github/actions/validate-binary
with:
gpu_arch_type: ${{ matrix.gpu_arch_type }}
gpu_arch_ver: ${{ matrix.gpu_arch_version }}
installation: ${{ matrix.installation }}
installation_pypi: ${{ matrix.installation_pypi }}
python_version: ${{ matrix.python_version }}
desired_cuda: ${{ matrix.desired_cuda }}
dev_toolset: ''
package_type: wheel
target_os: linux
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
name: ${{ matrix.build_name }}
with:
runner: ${{ matrix.validation_runner }}
repository: "pytorch/builder"
ref: ${{ inputs.ref || github.ref }}
job-name: ${{ matrix.build_name }}
script: |
set -ex
export ENV_NAME="conda-env-${{ github.run_id }}"
export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
export INSTALLATION="${{ matrix.installation }}"
export CUDA_VER="${{ matrix.desired_cuda }}"
export DESIRED_PYTHON="${{ matrix.python_version }}"
export DESIRED_CUDA="${{ matrix.desired_cuda }}"
export DESIRED_DEVTOOLSET="${{ matrix.devtoolset }}"
export PACKAGE_TYPE="${{ matrix.package_type }}"
export TARGET_OS="linux"
conda create -y -n ${ENV_NAME} python=${{ matrix.python_version }} numpy pillow
conda activate ${ENV_NAME}
export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib"
export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH

linux-libt:
needs: generate-linux-libtorch-matrix
strategy:
matrix:
${{ fromJson(needs.generate-linux-libtorch-matrix.outputs.matrix) }}
fail-fast: false
runs-on: ${{ matrix.validation_runner }}
env:
PYTHON_VERSION: ${{ matrix.python_version }}
steps:
- name: Checkout PyTorch builder
uses: actions/checkout@v2
- name: Install Conda
uses: conda-incubator/setup-miniconda@v2
with:
python-version: ${{ matrix.python_version }}
auto-update-conda: true
miniconda-version: "latest"
activate-environment: testenv
- name: Install pytorch and smoke test
shell: bash
env:
INSTALLATION: ${{ matrix.installation }}
ENV_NAME: conda-env-${{ github.run_id }}
DESIRED_PYTHON: ${{ matrix.python_version }}
DESIRED_CUDA: ${{ matrix.desired_cuda }}
DESIRED_DEVTOOLSET: ${{ matrix.devtoolset }}
PACKAGE_TYPE: libtorch
run: |
sudo apt-get install unzip -y
set -ex
curl ${INSTALLATION} -o libtorch.zip
if [[ ${{ matrix.package_type }} == "libtorch" ]]; then
curl ${{ matrix.installation }} -o libtorch.zip
unzip libtorch.zip
conda create -yp ${ENV_NAME} python=${{ matrix.python_version }} numpy
export LD_LIBRARY_PATH="$(dirname $(which python))/lib"
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib
if [[ ${{ matrix.libtorch_variant }} == "static-with-deps" ]]; then
conda run -p ${ENV_NAME} --cwd libtorch env LD_LIBRARY_PATH=${LD_LIBRARY_PATH} bash ./../check_binary.sh
fi
conda env remove -p ${ENV_NAME}
else
eval $INSTALLATION
python ./test/smoke_test/smoke_test.py
${PWD}/check_binary.sh
fi
57 changes: 18 additions & 39 deletions test/smoke_test/smoke_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,26 +14,6 @@
SCRIPT_DIR = Path(__file__).parent
NIGHTLY_ALLOWED_DELTA = 3

# helper function to return the conda installed packages
# and return package we are insterseted in
def get_anaconda_output_for_package(pkg_name_str):
import subprocess as sp

# If we are installing using conda just list package name
if installation_str.find("conda install") != -1:
cmd = "conda list --explicit"
output = sp.getoutput(cmd)
for item in output.split("\n"):
if pkg_name_str in item:
return item
return f"{pkg_name_str} can't be found"
else:
cmd = "conda list -f " + pkg_name_str
output = sp.getoutput(cmd)
# Get the last line only
return output.strip().split('\n')[-1]


def check_nightly_binaries_date(package: str) -> None:
from datetime import datetime, timedelta
format_dt = '%Y%m%d'
Expand Down Expand Up @@ -62,6 +42,18 @@ def check_nightly_binaries_date(package: str) -> None:
f"Expected torchaudio, torchvision to be less then {NIGHTLY_ALLOWED_DELTA} days. But they are from {date_ta_str}, {date_tv_str} respectively"
)

def check_cuda_version(version: str, dlibary: str):
version = torch.ops.torchaudio.cuda_version()
if version is not None and torch.version.cuda is not None:
version_str = str(version)
ta_version = f"{version_str[:-3]}.{version_str[-2]}"
t_version = torch.version.cuda.split(".")
t_version = f"{t_version[0]}.{t_version[1]}"
if ta_version != t_version:
raise RuntimeError(
"Detected that PyTorch and {dlibary} were compiled with different CUDA versions. "
f"PyTorch has CUDA version {t_version} whereas {dlibary} has CUDA version {ta_version}. "
)

def smoke_test_cuda(package: str) -> None:
if not torch.cuda.is_available() and is_cuda_system:
Expand All @@ -76,27 +68,14 @@ def smoke_test_cuda(package: str) -> None:
print(f"torch cudnn: {torch.backends.cudnn.version()}")
print(f"cuDNN enabled? {torch.backends.cudnn.enabled}")

if(package == 'all'):
if(package == 'all' and is_cuda_system):
import torchaudio
import torchvision
# There is an issue with current windows runners calling conda from python
# https://github.com/pytorch/test-infra/issues/1054
if installation_str.find("nightly") != -1 or platform.system() == "Windows" :
# just print out cuda version, as version check were already performed during import
print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}")
print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}")
else:
# torchaudio runtime added the cuda verison check on 09/23/2022 via
# https://github.com/pytorch/audio/pull/2707
# so relying on anaconda output for pytorch-test and pytorch channel
torchaudio_allstr = get_anaconda_output_for_package(torchaudio.__name__)
if (
is_cuda_system
and "cu" + str(gpu_arch_ver).replace(".", "") not in torchaudio_allstr
):
raise RuntimeError(
f"CUDA version issue. Loaded: {torchaudio_allstr} Expected: {gpu_arch_ver}"
)
print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}")
print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}")
check_cuda_version(torch.ops.torchvision._cuda_version(), "TorchVision")
check_cuda_version(torch.ops.torchaudio.cuda_version(), "TorchAudio")


def smoke_test_conv2d() -> None:
import torch.nn as nn
Expand Down