Skip to content

Commit 3f66d9a

Browse files
atalmanatalman
committed
Validate linux binaries change
Add options Import torchvision Adding python 3.11 install pass package to check nightly binaries date Test test Add python 3.11 code testing Adding python 3.11 test Add python 3.11 validation Adding zlib develop install Install zlib etc.. Adding zlib1g as well testing testing Adding validate windows binary Trying to workaround testing Refacor smoke test
1 parent f3b676a commit 3f66d9a

File tree

3 files changed

+43
-32
lines changed

3 files changed

+43
-32
lines changed

‎.github/actions/validate-binary/action.yml

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ runs:
4444
run: |
4545
nvidia-smi
4646
- name: Install Conda Linux
47-
if: ${{ inputs.target_os == 'linux' }}
47+
if: ${{ inputs.target_os == 'linux' && inputs.python_version != '3.11' }}
4848
uses: conda-incubator/setup-miniconda@v2
4949
with:
5050
python-version: ${{ inputs.python_version }}
@@ -66,16 +66,29 @@ runs:
6666
DESIRED_DEVTOOLSET: ${{ inputs.dev_toolset }}
6767
PACKAGE_TYPE: ${{ inputs.package_type }}
6868
run: |
69-
set -ex
70-
conda create -yp ${ENV_NAME} python=${{ inputs.python_version }} numpy
71-
conda run -p ${ENV_NAME} $INSTALLATION
72-
if [ $DESIRED_PYTHON == "3.11" ]; then
73-
conda run -p ${ENV_NAME} python3 --package torchonly ./test/smoke_test/smoke_test.py
69+
70+
if [ $DESIRED_PYTHON == '3.11' ]; then
71+
set -ex
72+
export CPYTHON_VERSIONS=3.11.0
73+
sudo apt-get install build-essential gdb lcov libbz2-dev libffi-dev \
74+
libgdbm-dev liblzma-dev libncurses5-dev libreadline6-dev \
75+
libsqlite3-dev libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g zlib1g-dev -y
76+
77+
export PYTHON_PATH="/opt/_internal/cpython-3.11.0rc2/bin/"
78+
export PIP_PATH="${PYTHON_PATH}/pip"
79+
export PIP_INSTALLATION="${INSTALLATION/pip3/"$PIP_PATH"}"
80+
./common/install_cpython.sh
81+
82+
eval ${PYTHON_PATH}/python --version
83+
eval ${PIP_INSTALLATION}
84+
eval ${PYTHON_PATH}/python ./test/smoke_test/smoke_test.py --package torchonly
7485
else
86+
set -ex
87+
conda create -yp ${ENV_NAME} python=${{ inputs.python_version }} numpy
88+
conda run -p ${ENV_NAME} $INSTALLATION
7589
conda run -p ${ENV_NAME} python3 ./test/smoke_test/smoke_test.py
90+
export LD_LIBRARY_PATH="$(dirname $(which python))/lib"
91+
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib
92+
conda run -p ${ENV_NAME} env LD_LIBRARY_PATH=$LD_LIBRARY_PATH bash ${PWD}/check_binary.sh
93+
conda env remove -p ${ENV_NAME}
7694
fi
77-
78-
export LD_LIBRARY_PATH="$(dirname $(which python))/lib"
79-
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib
80-
conda run -p ${ENV_NAME} env LD_LIBRARY_PATH=$LD_LIBRARY_PATH bash ${PWD}/check_binary.sh
81-
conda env remove -p ${ENV_NAME}

‎.github/workflows/validate-linux-binaries.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ jobs:
3232
package-type: wheel
3333
os: linux
3434
channel: ${{ inputs.channel }}
35+
with-py311: enable
3536
generate-linux-libtorch-matrix:
3637
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
3738
with:
@@ -70,7 +71,7 @@ jobs:
7071
- name: Checkout PyTorch builder
7172
uses: actions/checkout@v2
7273
- name: Validate binary wheel
73-
uses: .github/actions/validate-binary
74+
uses: ./.github/actions/validate-binary
7475
with:
7576
gpu_arch_type: ${{ matrix.gpu_arch_type }}
7677
gpu_arch_ver: ${{ matrix.gpu_arch_version }}

‎test/smoke_test/smoke_test.py

Lines changed: 17 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,13 @@
55
import argparse
66
import torch
77

8-
9-
108
gpu_arch_ver = os.getenv("GPU_ARCH_VER")
119
gpu_arch_type = os.getenv("GPU_ARCH_TYPE")
1210
# use installation env variable to tell if it is nightly channel
1311
installation_str = os.getenv("INSTALLATION")
1412
is_cuda_system = gpu_arch_type == "cuda"
1513
SCRIPT_DIR = Path(__file__).parent
14+
NIGHTLY_ALLOWED_DELTA = 3
1615

1716
# helper function to return the conda installed packages
1817
# and return package we are insterseted in
@@ -35,32 +34,31 @@ def get_anaconda_output_for_package(pkg_name_str):
3534

3635

3736
def check_nightly_binaries_date(package: str) -> None:
37+
from datetime import datetime, timedelta
38+
format_dt = '%Y%m%d'
39+
3840
torch_str = torch.__version__
3941
date_t_str = re.findall("dev\d+", torch.__version__)
42+
date_t_delta = datetime.now() - datetime.datetime.strptime(date_t_str.lstrip("dev"), format_dt)
43+
if date_t_delta.days >= NIGHTLY_ALLOWED_DELTA:
44+
raise RuntimeError(
45+
f"the binaries are from {date_t_str} and are more than {NIGHTLY_ALLOWED_DELTA} days old!"
46+
)
4047

4148
if(package == "all"):
4249
ta_str = torchaudio.__version__
4350
tv_str = torchvision.__version__
4451
date_ta_str = re.findall("dev\d+", torchaudio.__version__)
4552
date_tv_str = re.findall("dev\d+", torchvision.__version__)
53+
date_ta_delta = datetime.now() - datetime.datetime.strptime(date_ta_str.lstrip("dev"), format_dt)
54+
date_tv_delta = datetime.now() - datetime.datetime.strptime(date_tv_str.lstrip("dev"), format_dt)
4655

4756
# check that the above three lists are equal and none of them is empty
48-
if not date_t_str or not date_t_str == date_ta_str == date_tv_str:
57+
if date_ta_delta.days > NIGHTLY_ALLOWED_DELTA or date_tv_delta.days > NIGHTLY_ALLOWED_DELTA:
4958
raise RuntimeError(
50-
f"Expected torch, torchaudio, torchvision to be the same date. But they are from {date_t_str}, {date_ta_str}, {date_tv_str} respectively"
59+
f"Expected torchaudio, torchvision to be less then {NIGHTLY_ALLOWED_DELTA} days. But they are from {date_ta_str}, {date_tv_str} respectively"
5160
)
5261

53-
# check that the date is recent, at this point, date_torch_str is not empty
54-
binary_date_str = date_t_str[0][3:]
55-
from datetime import datetime
56-
57-
binary_date_obj = datetime.strptime(binary_date_str, "%Y%m%d").date()
58-
today_obj = datetime.today().date()
59-
delta = today_obj - binary_date_obj
60-
if delta.days >= 2:
61-
raise RuntimeError(
62-
f"the binaries are from {binary_date_obj} and are more than 2 days old!"
63-
)
6462

6563
def smoke_test_cuda(package: str) -> None:
6664
if not torch.cuda.is_available() and is_cuda_system:
@@ -165,6 +163,7 @@ def smoke_test_torchvision_resnet50_classify(device: str = "cpu") -> None:
165163

166164

167165
def smoke_test_torchaudio() -> None:
166+
import torchaudio
168167
import torchaudio.compliance.kaldi # noqa: F401
169168
import torchaudio.datasets # noqa: F401
170169
import torchaudio.functional # noqa: F401
@@ -184,20 +183,18 @@ def main() -> None:
184183
choices=["all", "torchonly"],
185184
default="all",
186185
)
187-
186+
options = parser.parse_args()
188187
print(f"torch: {torch.__version__}")
188+
189189
smoke_test_cuda(options.package)
190190
smoke_test_conv2d()
191191

192192
# only makes sense to check nightly package where dates are known
193193
if installation_str.find("nightly") != -1:
194-
check_nightly_binaries_date()
194+
check_nightly_binaries_date(options.package)
195195

196196
if options.package == "all":
197197
import torchaudio
198-
# the following import would invoke
199-
# _check_cuda_version()
200-
# via torchvision.extension._check_cuda_version()
201198
import torchvision
202199
print(f"torchvision: {torchvision.__version__}")
203200
print(f"torchaudio: {torchaudio.__version__}")

0 commit comments

Comments
 (0)