From 2203c8fa9d89df8cddc9ef6334a335dc8670695a Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Wed, 25 Jan 2023 19:43:40 +0000 Subject: [PATCH 01/40] created scripts for actions workflow --- aarch64_ci_build.sh | 22 +++ aarch64_wheel_ci_build.py | 365 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 aarch64_ci_build.sh create mode 100755 aarch64_wheel_ci_build.py diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh new file mode 100644 index 000000000..de3d3da25 --- /dev/null +++ b/aarch64_ci_build.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -eux -o pipefail + +PYTHON_VERSION=3.10 + +############################################################################### +# Install conda +############################################################################### +echo 'Installing conda-forge' +curl -L -o ~/mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh +chmod +x ~/mambaforge.sh +~/mambaforge.sh -b -p /opt/conda +rm ~/mambaforge.sh +/opt/conda/bin/conda install -c conda-forge python=${PYTHON_VERSION} numpy pyyaml setuptools +/opt/conda/bin/conda init bash +source ~/.bashrc + +############################################################################### +# Run aarch64 builder python +############################################################################### +cd / +python /builder/aarch64_wheel_ci_build.py --python-version ${PYTHON_VERSION} --enable-mkldnn True diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py new file mode 100755 index 000000000..51a994926 --- /dev/null +++ b/aarch64_wheel_ci_build.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 + +import os +import subprocess +from typing import Dict, List, Optional, Tuple, Union + + +@staticmethod +def _split_cmd(args: Union[str, List[str]]) -> List[str]: + return args.split() if isinstance(args, str) else args + + +def run_cmd(args: Union[str, List[str]]) -> None: + p = subprocess.Popen(stdin=subprocess.PIPE) + p.communicate(input=_split_cmd(args)).encode("utf-8") + rc = p.wait() + if rc != 0: + raise subprocess.CalledProcessError(rc) + + +def list_dir(path: str) -> List[str]: + return run_cmd(["ls", "-1", path]).split("\n") + + +def build_OpenBLAS(git_clone_flags: str = "") -> None: + print('Building OpenBLAS') + run_cmd(f"git clone https://github.com/xianyi/OpenBLAS -b v0.3.19 {git_clone_flags}") + make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8" + run_cmd(f"pushd OpenBLAS; make {make_flags} -j8; sudo make {make_flags} install; popd; rm -rf OpenBLAS") + + +def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: + print('Building Arm Compute Library') + run_cmd("mkdir /acl") + run_cmd(f"git clone https://github.com/ARM-software/ComputeLibrary.git -b v22.05 {git_clone_flags}") + run_cmd(f"pushd ComputeLibrary; export acl_install_dir=/acl; " \ + f"scons Werror=1 -j8 debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native build_dir=$acl_install_dir/build; " \ + f"cp -r arm_compute $acl_install_dir; " \ + f"cp -r include $acl_install_dir; " \ + f"cp -r utils $acl_install_dir; " \ + f"cp -r support $acl_install_dir; popd") + + +def embed_libgomp(use_conda, wheel_name) -> None: + run_cmd("pip3 install auditwheel") + run_cmd("conda install -y patchelf") + from tempfile import NamedTemporaryFile + with NamedTemporaryFile() as tmp: + tmp.write(embed_library_script.encode('utf-8')) + tmp.flush() + run_cmd(f"mv {tmp.name} ./embed_library.py") + + print('Embedding libgomp into wheel') + run_cmd(f"python3 embed_library.py {wheel_name} --update-tag") + + +def checkout_repo(branch: str = "master", + url: str = "", + git_clone_flags: str = "", + mapping: Dict[str, Tuple[str, str]] = []) -> Optional[str]: + for prefix in mapping: + if not branch.startswith(prefix): + continue + tag = f"v{mapping[prefix][0]}-{mapping[prefix][1]}" + run_cmd(f"git clone {url} -b {tag} {git_clone_flags}") + return mapping[prefix][0] + + run_cmd(f"git clone {url} {git_clone_flags}") + return None + + +def build_torchvision(branch: str = "main", + use_conda: bool = True, + git_clone_flags: str = "") -> str: + print('Checking out TorchVision repo') + build_version = checkout_repo(branch=branch, + url="https://github.com/pytorch/vision", + git_clone_flags=git_clone_flags, + mapping={ + "v1.7.1": ("0.8.2", "rc2"), + "v1.8.0": ("0.9.0", "rc3"), + "v1.8.1": ("0.9.1", "rc1"), + "v1.9.0": ("0.10.0", "rc1"), + "v1.10.0": ("0.11.1", "rc1"), + "v1.10.1": ("0.11.2", "rc1"), + "v1.10.2": ("0.11.3", "rc1"), + "v1.11.0": ("0.12.0", "rc1"), + "v1.12.0": ("0.13.0", "rc4"), + "v1.12.1": ("0.13.1", "rc6"), + "v1.13.0": ("0.14.0", "rc4"), + "v1.13.1": ("0.14.1", "rc2"), + }) + print('Building TorchVision wheel') + build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " + if branch == 'nightly': + version = run_cmd(["if [ -f vision/version.txt ]; then cat vision/version.txt; fi"]).strip() + if len(version) == 0: + # In older revisions, version was embedded in setup.py + version = run_cmd(["grep", "\"version = '\"", "vision/setup.py"]).strip().split("'")[1][:-2] + build_date = run_cmd("cd /pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + build_vars += f"BUILD_VERSION={version}.dev{build_date}" + elif build_version is not None: + build_vars += f"BUILD_VERSION={build_version}" + + run_cmd(f"cd /vision; {build_vars} python3 setup.py bdist_wheel") + wheel_name = run_cmd("ls /vision/dist")[0] + embed_libgomp(use_conda, os.path.join('vision', 'dist', wheel_name)) + + print('Move TorchVision wheel to artfacts') + run_cmd(f"mv /vision/dist/{wheel_name} /artifacts/") + return wheel_name + + +def build_torchtext(branch: str = "main", + use_conda: bool = True, + git_clone_flags: str = "") -> str: + print('Checking out TorchText repo') + run_cmd(f"cd /") + git_clone_flags += " --recurse-submodules" + build_version = checkout_repo(branch=branch, + url="https://github.com/pytorch/text", + git_clone_flags=git_clone_flags, + mapping={ + "v1.9.0": ("0.10.0", "rc1"), + "v1.10.0": ("0.11.0", "rc2"), + "v1.10.1": ("0.11.1", "rc1"), + "v1.10.2": ("0.11.2", "rc1"), + "v1.11.0": ("0.12.0", "rc1"), + "v1.12.0": ("0.13.0", "rc2"), + "v1.12.1": ("0.13.1", "rc5"), + "v1.13.0": ("0.14.0", "rc3"), + "v1.13.1": ("0.14.1", "rc1"), + }) + print('Building TorchText wheel') + build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " + if branch == 'nightly': + version = run_cmd(["if [ -f text/version.txt ]; then cat text/version.txt; fi"]).strip() + build_date = run_cmd("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + build_vars += f"BUILD_VERSION={version}.dev{build_date}" + elif build_version is not None: + build_vars += f"BUILD_VERSION={build_version}" + + run_cmd(f"cd text; {build_vars} python3 setup.py bdist_wheel") + wheel_name = run_cmd("ls /text/dist")[0] + embed_libgomp(use_conda, os.path.join('text', 'dist', wheel_name)) + + print('Move TorchText wheel to artfacts') + run_cmd(f"mv /text/dist/{wheel_name} /artifacts/") + return wheel_name + + +def build_torchaudio(branch: str = "main", + use_conda: bool = True, + git_clone_flags: str = "") -> str: + print('Checking out TorchAudio repo') + git_clone_flags += " --recurse-submodules" + build_version = checkout_repo(branch=branch, + url="https://github.com/pytorch/audio", + git_clone_flags=git_clone_flags, + mapping={ + "v1.9.0": ("0.9.0", "rc2"), + "v1.10.0": ("0.10.0", "rc5"), + "v1.10.1": ("0.10.1", "rc1"), + "v1.10.2": ("0.10.2", "rc1"), + "v1.11.0": ("0.11.0", "rc1"), + "v1.12.0": ("0.12.0", "rc3"), + "v1.12.1": ("0.12.1", "rc5"), + "v1.13.0": ("0.13.0", "rc4"), + "v1.13.1": ("0.13.1", "rc2"), + }) + print('Building TorchAudio wheel') + build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " + if branch == 'nightly': + version = run_cmd(["grep", "\"version = '\"", "audio/setup.py"]).strip().split("'")[1][:-2] + build_date = run_cmd("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + build_vars += f"BUILD_VERSION={version}.dev{build_date}" + elif build_version is not None: + build_vars += f"BUILD_VERSION={build_version}" + + run_cmd(f"cd /audio; {build_vars} python3 setup.py bdist_wheel") + wheel_name = run_cmd("ls /audio/dist")[0] + embed_libgomp(use_conda, os.path.join('audio', 'dist', wheel_name)) + + print('Move TorchAudio wheel to artfacts') + run_cmd(f"mv /audio/dist/{wheel_name} /artifacts/") + return wheel_name + + +def build_torchdata(branch: str = "main", + use_conda: bool = True, + git_clone_flags: str = "") -> str: + print('Checking out TorchData repo') + git_clone_flags += " --recurse-submodules" + build_version = checkout_repo(branch=branch, + url="https://github.com/pytorch/data", + git_clone_flags=git_clone_flags, + mapping={ + "v1.11.0": ("0.3.0", "rc1"), + "v1.12.0": ("0.4.0", "rc3"), + "v1.12.1": ("0.4.1", "rc5"), + "v1.13.1": ("0.5.1", "rc2"), + }) + print('Building TorchData wheel') + build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " + if branch == 'nightly': + version = run_cmd(["grep", "\"version = '\"", "audio/setup.py"]).strip().split("'")[1][:-2] + build_date = run_cmd("cd /pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + build_vars += f"BUILD_VERSION={version}.dev{build_date}" + elif build_version is not None: + build_vars += f"BUILD_VERSION={build_version}" + + run_cmd(f"cd /data; {build_vars} python3 setup.py bdist_wheel") + wheel_name = run_cmd("ls /data/dist")[0] + embed_libgomp(use_conda, os.path.join('data', 'dist', wheel_name)) + + print('Move TorchAudio wheel to artfacts') + run_cmd(f"mv /data/dist/{wheel_name} /artifacts/") + return wheel_name + + +def start_build(branch="master", + compiler="gcc-8", + use_conda=True, + python_version="3.8", + shallow_clone=True, + enable_mkldnn=False) -> Tuple[str, str]: + git_clone_flags = " --depth 1 --shallow-submodules" if shallow_clone else "" + run_cmd("yum install -y sudo") + run_cmd("conda install -y ninja scons") + + build_OpenBLAS(git_clone_flags) + + print('Building PyTorch wheel') + # Breakpad build fails on aarch64 + build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " + run_cmd(f"cd /pytorch") + if branch == 'nightly': + build_date = run_cmd("git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + version = run_cmd("cat pytorch/version.txt").strip()[:-2] + build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={version}.dev{build_date} PYTORCH_BUILD_NUMBER=1" + if branch.startswith("v1."): + build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={branch[1:branch.find('-')]} PYTORCH_BUILD_NUMBER=1" + if enable_mkldnn: + build_ArmComputeLibrary(git_clone_flags) + print("build pytorch with mkldnn+acl backend") + build_vars += " USE_MKLDNN=ON USE_MKLDNN_ACL=ON" + run_cmd(f"cd /pytorch ; export ACL_ROOT_DIR=$HOME/acl; {build_vars} python3 setup.py bdist_wheel") + print('Repair the wheel') + pytorch_wheel_name = list_dir("pytorch/dist")[0] + run_cmd(f"export LD_LIBRARY_PATH=/acl/build:/pytorch/build/lib; auditwheel repair $HOME/pytorch/dist/{pytorch_wheel_name}") + print('replace the original wheel with the repaired one') + pytorch_repaired_wheel_name = list_dir("wheelhouse")[0] + run_cmd(f"cp /wheelhouse/{pytorch_repaired_wheel_name} /pytorch/dist/{pytorch_wheel_name}") + else: + print("build pytorch without mkldnn backend") + run_cmd(f"cd pytorch ; {build_vars} python3 setup.py bdist_wheel") + + print("Deleting build folder") + run_cmd("cd /pytorch; rm -rf build") + pytorch_wheel_name = run_cmd("ls /pytorch/dist")[0] + embed_libgomp(use_conda, os.path.join('pytorch', 'dist', pytorch_wheel_name)) + print('Move PyTorch wheel to artfacts') + run_cmd(f"mv /pytorch/dist/{pytorch_wheel_name} /artifacts/") + + vision_wheel_name = build_torchvision(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) + audio_wheel_name = build_torchaudio(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) + text_wheel_name = build_torchtext(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) + data_wheel_name = build_torchdata(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) + return [pytorch_wheel_name, vision_wheel_name, audio_wheel_name, text_wheel_name, data_wheel_name] + + +embed_library_script = """ +#!/usr/bin/env python3 + +from auditwheel.patcher import Patchelf +from auditwheel.wheeltools import InWheelCtx +from auditwheel.elfutils import elf_file_filter +from auditwheel.repair import copylib +from auditwheel.lddtree import lddtree +from subprocess import check_call +import os +import shutil +import sys +from tempfile import TemporaryDirectory + + +def replace_tag(filename): + with open(filename, 'r') as f: + lines = f.read().split("\\n") + for i,line in enumerate(lines): + if not line.startswith("Tag: "): + continue + lines[i] = line.replace("-linux_", "-manylinux2014_") + print(f'Updated tag from {line} to {lines[i]}') + + with open(filename, 'w') as f: + f.write("\\n".join(lines)) + + +class AlignedPatchelf(Patchelf): + def set_soname(self, file_name: str, new_soname: str) -> None: + check_call(['patchelf', '--page-size', '65536', '--set-soname', new_soname, file_name]) + + def replace_needed(self, file_name: str, soname: str, new_soname: str) -> None: + check_call(['patchelf', '--page-size', '65536', '--replace-needed', soname, new_soname, file_name]) + + +def embed_library(whl_path, lib_soname, update_tag=False): + patcher = AlignedPatchelf() + out_dir = TemporaryDirectory() + whl_name = os.path.basename(whl_path) + tmp_whl_name = os.path.join(out_dir.name, whl_name) + with InWheelCtx(whl_path) as ctx: + torchlib_path = os.path.join(ctx._tmpdir.name, 'torch', 'lib') + ctx.out_wheel=tmp_whl_name + new_lib_path, new_lib_soname = None, None + for filename, elf in elf_file_filter(ctx.iter_files()): + if not filename.startswith('torch/lib'): + continue + libtree = lddtree(filename) + if lib_soname not in libtree['needed']: + continue + lib_path = libtree['libs'][lib_soname]['path'] + if lib_path is None: + print(f"Can't embed {lib_soname} as it could not be found") + break + if lib_path.startswith(torchlib_path): + continue + + if new_lib_path is None: + new_lib_soname, new_lib_path = copylib(lib_path, torchlib_path, patcher) + patcher.replace_needed(filename, lib_soname, new_lib_soname) + print(f'Replacing {lib_soname} with {new_lib_soname} for {filename}') + if update_tag: + # Add manylinux2014 tag + for filename in ctx.iter_files(): + if os.path.basename(filename) != 'WHEEL': + continue + replace_tag(filename) + shutil.move(tmp_whl_name, whl_path) + + +if __name__ == '__main__': + embed_library(sys.argv[1], 'libgomp.so.1', len(sys.argv) > 2 and sys.argv[2] == '--update-tag') +""" + +def parse_arguments(): + from argparse import ArgumentParser + parser = ArgumentParser("AARCH64 wheels python CD") + parser.add_argument("--debug", action="store_true") + parser.add_argument("--build-only", action="store_true") + parser.add_argument("--test-only", type=str) + parser.add_argument("--python-version", type=str, choices=['3.6', '3.7', '3.8', '3.9', '3.10'], default=None) + parser.add_argument("--branch", type=str, default="master") + parser.add_argument("--compiler", type=str, choices=['gcc-7', 'gcc-8', 'gcc-9', 'clang'], default="gcc-8") + parser.add_argument("--enable-mkldnn", action="store_true") + return parser.parse_args() + +if __name__ == '__main__': + args = parse_arguments() + + start_build(branch=args.branch, + compiler=args.compiler, + python_version=args.python_version, + enable_mkldnn=args.enable_mkldnn) From bf17ca6442e3842425d5fd06f40814ac05897dcb Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Wed, 25 Jan 2023 22:49:21 +0000 Subject: [PATCH 02/40] change OS calls and add version checks --- aarch64_ci_build.sh | 11 ++-- aarch64_wheel_ci_build.py | 103 +++++++++++++++++--------------------- 2 files changed, 52 insertions(+), 62 deletions(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index de3d3da25..010c2fff0 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -11,12 +11,15 @@ curl -L -o ~/mambaforge.sh https://github.com/conda-forge/miniforge/releases/lat chmod +x ~/mambaforge.sh ~/mambaforge.sh -b -p /opt/conda rm ~/mambaforge.sh -/opt/conda/bin/conda install -c conda-forge python=${PYTHON_VERSION} numpy pyyaml setuptools -/opt/conda/bin/conda init bash -source ~/.bashrc +/opt/conda/bin/conda install -y -c conda-forge python=${PYTHON_VERSION} numpy pyyaml setuptools +export CONDA_PYTHON_EXE='/opt/conda/bin/python' +export CONDA_EXE='/opt/conda/bin/conda' +export PATH='/opt/conda/bin:$PATH' +python --version +conda --version ############################################################################### # Run aarch64 builder python ############################################################################### cd / -python /builder/aarch64_wheel_ci_build.py --python-version ${PYTHON_VERSION} --enable-mkldnn True +python /builder/aarch64_wheel_ci_build.py --python-version ${PYTHON_VERSION} --enable-mkldnn diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 51a994926..2be77b05d 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -5,35 +5,23 @@ from typing import Dict, List, Optional, Tuple, Union -@staticmethod -def _split_cmd(args: Union[str, List[str]]) -> List[str]: - return args.split() if isinstance(args, str) else args - - -def run_cmd(args: Union[str, List[str]]) -> None: - p = subprocess.Popen(stdin=subprocess.PIPE) - p.communicate(input=_split_cmd(args)).encode("utf-8") - rc = p.wait() - if rc != 0: - raise subprocess.CalledProcessError(rc) - def list_dir(path: str) -> List[str]: - return run_cmd(["ls", "-1", path]).split("\n") + return os.system(["ls", "-1", path]).split("\n") def build_OpenBLAS(git_clone_flags: str = "") -> None: print('Building OpenBLAS') - run_cmd(f"git clone https://github.com/xianyi/OpenBLAS -b v0.3.19 {git_clone_flags}") + os.system(f"git clone https://github.com/xianyi/OpenBLAS -b v0.3.19 {git_clone_flags}") make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8" - run_cmd(f"pushd OpenBLAS; make {make_flags} -j8; sudo make {make_flags} install; popd; rm -rf OpenBLAS") + os.system(f"pushd OpenBLAS; make {make_flags} -j8; make {make_flags} install; popd; rm -rf OpenBLAS") def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: print('Building Arm Compute Library') - run_cmd("mkdir /acl") - run_cmd(f"git clone https://github.com/ARM-software/ComputeLibrary.git -b v22.05 {git_clone_flags}") - run_cmd(f"pushd ComputeLibrary; export acl_install_dir=/acl; " \ + os.system("mkdir /acl") + os.system(f"git clone https://github.com/ARM-software/ComputeLibrary.git -b v22.05 {git_clone_flags}") + os.system(f"pushd ComputeLibrary; export acl_install_dir=/acl; " \ f"scons Werror=1 -j8 debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native build_dir=$acl_install_dir/build; " \ f"cp -r arm_compute $acl_install_dir; " \ f"cp -r include $acl_install_dir; " \ @@ -42,16 +30,16 @@ def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: def embed_libgomp(use_conda, wheel_name) -> None: - run_cmd("pip3 install auditwheel") - run_cmd("conda install -y patchelf") + os.system("pip3 install auditwheel") + os.system("conda install -y patchelf") from tempfile import NamedTemporaryFile with NamedTemporaryFile() as tmp: tmp.write(embed_library_script.encode('utf-8')) tmp.flush() - run_cmd(f"mv {tmp.name} ./embed_library.py") + os.system(f"mv {tmp.name} ./embed_library.py") print('Embedding libgomp into wheel') - run_cmd(f"python3 embed_library.py {wheel_name} --update-tag") + os.system(f"python3 embed_library.py {wheel_name} --update-tag") def checkout_repo(branch: str = "master", @@ -62,10 +50,10 @@ def checkout_repo(branch: str = "master", if not branch.startswith(prefix): continue tag = f"v{mapping[prefix][0]}-{mapping[prefix][1]}" - run_cmd(f"git clone {url} -b {tag} {git_clone_flags}") + os.system(f"git clone {url} -b {tag} {git_clone_flags}") return mapping[prefix][0] - run_cmd(f"git clone {url} {git_clone_flags}") + os.system(f"git clone {url} {git_clone_flags}") return None @@ -93,21 +81,21 @@ def build_torchvision(branch: str = "main", print('Building TorchVision wheel') build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " if branch == 'nightly': - version = run_cmd(["if [ -f vision/version.txt ]; then cat vision/version.txt; fi"]).strip() + version = os.system(["if [ -f vision/version.txt ]; then cat vision/version.txt; fi"]).strip() if len(version) == 0: # In older revisions, version was embedded in setup.py - version = run_cmd(["grep", "\"version = '\"", "vision/setup.py"]).strip().split("'")[1][:-2] - build_date = run_cmd("cd /pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + version = os.system(["grep", "\"version = '\"", "vision/setup.py"]).strip().split("'")[1][:-2] + build_date = os.system("cd /pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") build_vars += f"BUILD_VERSION={version}.dev{build_date}" elif build_version is not None: build_vars += f"BUILD_VERSION={build_version}" - run_cmd(f"cd /vision; {build_vars} python3 setup.py bdist_wheel") - wheel_name = run_cmd("ls /vision/dist")[0] + os.system(f"cd /vision; {build_vars} python3 setup.py bdist_wheel") + wheel_name = os.system("ls /vision/dist")[0] embed_libgomp(use_conda, os.path.join('vision', 'dist', wheel_name)) print('Move TorchVision wheel to artfacts') - run_cmd(f"mv /vision/dist/{wheel_name} /artifacts/") + os.system(f"mv /vision/dist/{wheel_name} /artifacts/") return wheel_name @@ -115,7 +103,7 @@ def build_torchtext(branch: str = "main", use_conda: bool = True, git_clone_flags: str = "") -> str: print('Checking out TorchText repo') - run_cmd(f"cd /") + os.system(f"cd /") git_clone_flags += " --recurse-submodules" build_version = checkout_repo(branch=branch, url="https://github.com/pytorch/text", @@ -134,18 +122,18 @@ def build_torchtext(branch: str = "main", print('Building TorchText wheel') build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " if branch == 'nightly': - version = run_cmd(["if [ -f text/version.txt ]; then cat text/version.txt; fi"]).strip() - build_date = run_cmd("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + version = os.system(["if [ -f text/version.txt ]; then cat text/version.txt; fi"]).strip() + build_date = os.system("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") build_vars += f"BUILD_VERSION={version}.dev{build_date}" elif build_version is not None: build_vars += f"BUILD_VERSION={build_version}" - run_cmd(f"cd text; {build_vars} python3 setup.py bdist_wheel") - wheel_name = run_cmd("ls /text/dist")[0] + os.system(f"cd text; {build_vars} python3 setup.py bdist_wheel") + wheel_name = os.system("ls /text/dist")[0] embed_libgomp(use_conda, os.path.join('text', 'dist', wheel_name)) print('Move TorchText wheel to artfacts') - run_cmd(f"mv /text/dist/{wheel_name} /artifacts/") + os.system(f"mv /text/dist/{wheel_name} /artifacts/") return wheel_name @@ -171,18 +159,18 @@ def build_torchaudio(branch: str = "main", print('Building TorchAudio wheel') build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " if branch == 'nightly': - version = run_cmd(["grep", "\"version = '\"", "audio/setup.py"]).strip().split("'")[1][:-2] - build_date = run_cmd("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + version = os.system(["grep", "\"version = '\"", "audio/setup.py"]).strip().split("'")[1][:-2] + build_date = os.system("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") build_vars += f"BUILD_VERSION={version}.dev{build_date}" elif build_version is not None: build_vars += f"BUILD_VERSION={build_version}" - run_cmd(f"cd /audio; {build_vars} python3 setup.py bdist_wheel") - wheel_name = run_cmd("ls /audio/dist")[0] + os.system(f"cd /audio; {build_vars} python3 setup.py bdist_wheel") + wheel_name = os.system("ls /audio/dist")[0] embed_libgomp(use_conda, os.path.join('audio', 'dist', wheel_name)) print('Move TorchAudio wheel to artfacts') - run_cmd(f"mv /audio/dist/{wheel_name} /artifacts/") + os.system(f"mv /audio/dist/{wheel_name} /artifacts/") return wheel_name @@ -203,18 +191,18 @@ def build_torchdata(branch: str = "main", print('Building TorchData wheel') build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " if branch == 'nightly': - version = run_cmd(["grep", "\"version = '\"", "audio/setup.py"]).strip().split("'")[1][:-2] - build_date = run_cmd("cd /pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + version = os.system(["grep", "\"version = '\"", "audio/setup.py"]).strip().split("'")[1][:-2] + build_date = os.system("cd /pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") build_vars += f"BUILD_VERSION={version}.dev{build_date}" elif build_version is not None: build_vars += f"BUILD_VERSION={build_version}" - run_cmd(f"cd /data; {build_vars} python3 setup.py bdist_wheel") - wheel_name = run_cmd("ls /data/dist")[0] + os.system(f"cd /data; {build_vars} python3 setup.py bdist_wheel") + wheel_name = os.system("ls /data/dist")[0] embed_libgomp(use_conda, os.path.join('data', 'dist', wheel_name)) print('Move TorchAudio wheel to artfacts') - run_cmd(f"mv /data/dist/{wheel_name} /artifacts/") + os.system(f"mv /data/dist/{wheel_name} /artifacts/") return wheel_name @@ -225,18 +213,17 @@ def start_build(branch="master", shallow_clone=True, enable_mkldnn=False) -> Tuple[str, str]: git_clone_flags = " --depth 1 --shallow-submodules" if shallow_clone else "" - run_cmd("yum install -y sudo") - run_cmd("conda install -y ninja scons") + os.system(f"conda install -y ninja scons") build_OpenBLAS(git_clone_flags) print('Building PyTorch wheel') # Breakpad build fails on aarch64 build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " - run_cmd(f"cd /pytorch") + os.system(f"cd /pytorch") if branch == 'nightly': - build_date = run_cmd("git log --pretty=format:%s -1").strip().split()[0].replace("-", "") - version = run_cmd("cat pytorch/version.txt").strip()[:-2] + build_date = os.system("git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + version = os.system("cat pytorch/version.txt").strip()[:-2] build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={version}.dev{build_date} PYTORCH_BUILD_NUMBER=1" if branch.startswith("v1."): build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={branch[1:branch.find('-')]} PYTORCH_BUILD_NUMBER=1" @@ -244,23 +231,23 @@ def start_build(branch="master", build_ArmComputeLibrary(git_clone_flags) print("build pytorch with mkldnn+acl backend") build_vars += " USE_MKLDNN=ON USE_MKLDNN_ACL=ON" - run_cmd(f"cd /pytorch ; export ACL_ROOT_DIR=$HOME/acl; {build_vars} python3 setup.py bdist_wheel") + os.system(f"cd /pytorch ; export ACL_ROOT_DIR=$HOME/acl; {build_vars} python3 setup.py bdist_wheel") print('Repair the wheel') pytorch_wheel_name = list_dir("pytorch/dist")[0] - run_cmd(f"export LD_LIBRARY_PATH=/acl/build:/pytorch/build/lib; auditwheel repair $HOME/pytorch/dist/{pytorch_wheel_name}") + os.system(f"export LD_LIBRARY_PATH=/acl/build:/pytorch/build/lib; auditwheel repair $HOME/pytorch/dist/{pytorch_wheel_name}") print('replace the original wheel with the repaired one') pytorch_repaired_wheel_name = list_dir("wheelhouse")[0] - run_cmd(f"cp /wheelhouse/{pytorch_repaired_wheel_name} /pytorch/dist/{pytorch_wheel_name}") + os.system(f"cp /wheelhouse/{pytorch_repaired_wheel_name} /pytorch/dist/{pytorch_wheel_name}") else: print("build pytorch without mkldnn backend") - run_cmd(f"cd pytorch ; {build_vars} python3 setup.py bdist_wheel") + os.system(f"cd pytorch ; {build_vars} python3 setup.py bdist_wheel") print("Deleting build folder") - run_cmd("cd /pytorch; rm -rf build") - pytorch_wheel_name = run_cmd("ls /pytorch/dist")[0] + os.system("cd /pytorch; rm -rf build") + pytorch_wheel_name = os.system("ls /pytorch/dist")[0] embed_libgomp(use_conda, os.path.join('pytorch', 'dist', pytorch_wheel_name)) print('Move PyTorch wheel to artfacts') - run_cmd(f"mv /pytorch/dist/{pytorch_wheel_name} /artifacts/") + os.system(f"mv /pytorch/dist/{pytorch_wheel_name} /artifacts/") vision_wheel_name = build_torchvision(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) audio_wheel_name = build_torchaudio(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) From fe7ea6c7b04c83b026b4abc96133e9c11e39662f Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Wed, 25 Jan 2023 23:23:16 +0000 Subject: [PATCH 03/40] change to ubuntu arm64. add OS dependencies --- aarch64_ci_build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index 010c2fff0..0f05b4b65 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -3,6 +3,12 @@ set -eux -o pipefail PYTHON_VERSION=3.10 +############################################################################### +# Install conda +############################################################################### +echo "Install builder OS dependencies" +apt-get install -y ninja-build g++ git cmake gfortran unzip curl + ############################################################################### # Install conda ############################################################################### From f3f00c07f727f68a906b5f1b82e9556e57e64341 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 26 Jan 2023 00:06:25 +0000 Subject: [PATCH 04/40] env updates and remove popd pushd --- aarch64_ci_build.sh | 13 ++++++++----- aarch64_wheel_ci_build.py | 18 +++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index 0f05b4b65..8fddfa96d 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -1,12 +1,15 @@ #!/bin/bash set -eux -o pipefail -PYTHON_VERSION=3.10 +CONDA_PYTHON_EXE=/opt/conda/bin/python +CONDA_EXE=/opt/conda/bin/conda +PATH=/opt/conda/bin:$PATH ############################################################################### -# Install conda +# Install OS dependencies ############################################################################### echo "Install builder OS dependencies" +apt-get update apt-get install -y ninja-build g++ git cmake gfortran unzip curl ############################################################################### @@ -18,9 +21,9 @@ chmod +x ~/mambaforge.sh ~/mambaforge.sh -b -p /opt/conda rm ~/mambaforge.sh /opt/conda/bin/conda install -y -c conda-forge python=${PYTHON_VERSION} numpy pyyaml setuptools -export CONDA_PYTHON_EXE='/opt/conda/bin/python' -export CONDA_EXE='/opt/conda/bin/conda' -export PATH='/opt/conda/bin:$PATH' +export CONDA_PYTHON_EXE=/opt/conda/bin/python +export CONDA_EXE=/opt/conda/bin/conda +export PATH=/opt/conda/bin:$PATH python --version conda --version diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 2be77b05d..c82f06700 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -12,21 +12,21 @@ def list_dir(path: str) -> List[str]: def build_OpenBLAS(git_clone_flags: str = "") -> None: print('Building OpenBLAS') - os.system(f"git clone https://github.com/xianyi/OpenBLAS -b v0.3.19 {git_clone_flags}") + os.system(f"cd /; git clone https://github.com/xianyi/OpenBLAS -b v0.3.19 {git_clone_flags}") make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8" - os.system(f"pushd OpenBLAS; make {make_flags} -j8; make {make_flags} install; popd; rm -rf OpenBLAS") + os.system(f"cd OpenBLAS; make {make_flags} -j8; make {make_flags} install; cd /; rm -rf OpenBLAS") def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: print('Building Arm Compute Library') - os.system("mkdir /acl") + os.system("cd / && mkdir /acl") os.system(f"git clone https://github.com/ARM-software/ComputeLibrary.git -b v22.05 {git_clone_flags}") - os.system(f"pushd ComputeLibrary; export acl_install_dir=/acl; " \ - f"scons Werror=1 -j8 debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native build_dir=$acl_install_dir/build; " \ - f"cp -r arm_compute $acl_install_dir; " \ - f"cp -r include $acl_install_dir; " \ - f"cp -r utils $acl_install_dir; " \ - f"cp -r support $acl_install_dir; popd") + os.system(f"cd ComputeLibrary; export acl_install_dir=/acl && " \ + f"scons Werror=1 -j8 debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native build_dir=$acl_install_dir/build && " \ + f"cp -r arm_compute $acl_install_dir && " \ + f"cp -r include $acl_install_dir && " \ + f"cp -r utils $acl_install_dir && " \ + f"cp -r support $acl_install_dir && cd /") def embed_libgomp(use_conda, wheel_name) -> None: From 9cbbf4de8efa48a94a6ed514420741b3c0b91d3a Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 26 Jan 2023 00:33:31 +0000 Subject: [PATCH 05/40] verifying paths --- aarch64_wheel_ci_build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index c82f06700..aaa5bc74b 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -231,10 +231,10 @@ def start_build(branch="master", build_ArmComputeLibrary(git_clone_flags) print("build pytorch with mkldnn+acl backend") build_vars += " USE_MKLDNN=ON USE_MKLDNN_ACL=ON" - os.system(f"cd /pytorch ; export ACL_ROOT_DIR=$HOME/acl; {build_vars} python3 setup.py bdist_wheel") + os.system(f"cd /pytorch ; export ACL_ROOT_DIR=/acl; {build_vars} python3 setup.py bdist_wheel") print('Repair the wheel') pytorch_wheel_name = list_dir("pytorch/dist")[0] - os.system(f"export LD_LIBRARY_PATH=/acl/build:/pytorch/build/lib; auditwheel repair $HOME/pytorch/dist/{pytorch_wheel_name}") + os.system(f"export LD_LIBRARY_PATH=/acl/build:/pytorch/build/lib; auditwheel repair /pytorch/dist/{pytorch_wheel_name}") print('replace the original wheel with the repaired one') pytorch_repaired_wheel_name = list_dir("wheelhouse")[0] os.system(f"cp /wheelhouse/{pytorch_repaired_wheel_name} /pytorch/dist/{pytorch_wheel_name}") From 3a934b4ea72cb92772dea9d4894549a754de4b71 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 26 Jan 2023 00:50:32 +0000 Subject: [PATCH 06/40] install pytorch requirements --- aarch64_wheel_ci_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index aaa5bc74b..cb48d39e7 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -220,7 +220,7 @@ def start_build(branch="master", print('Building PyTorch wheel') # Breakpad build fails on aarch64 build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " - os.system(f"cd /pytorch") + os.system(f"cd /pytorch; pip install requirements.txt") if branch == 'nightly': build_date = os.system("git log --pretty=format:%s -1").strip().split()[0].replace("-", "") version = os.system("cat pytorch/version.txt").strip()[:-2] From 3ea7563d098934553e499f2b9d226f823fe1f4b8 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 26 Jan 2023 02:08:49 +0000 Subject: [PATCH 07/40] missing pip option for requirements --- aarch64_wheel_ci_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index cb48d39e7..2971f2d2a 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -220,7 +220,7 @@ def start_build(branch="master", print('Building PyTorch wheel') # Breakpad build fails on aarch64 build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " - os.system(f"cd /pytorch; pip install requirements.txt") + os.system(f"cd /pytorch; pip install -r requirements.txt") if branch == 'nightly': build_date = os.system("git log --pretty=format:%s -1").strip().split()[0].replace("-", "") version = os.system("cat pytorch/version.txt").strip()[:-2] From cf71783d4c14cbfa005eb078e44a9913497896c5 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 26 Jan 2023 13:11:57 +0000 Subject: [PATCH 08/40] adding build-essential to os --- aarch64_ci_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index 8fddfa96d..bec786e38 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -10,7 +10,7 @@ PATH=/opt/conda/bin:$PATH ############################################################################### echo "Install builder OS dependencies" apt-get update -apt-get install -y ninja-build g++ git cmake gfortran unzip curl +apt-get install -y ninja-build g++ git cmake gfortran unzip curl build-essential ############################################################################### # Install conda From a124ab52d23c57c5eb5b3128ba178e109c1ae6d0 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 26 Jan 2023 14:00:18 +0000 Subject: [PATCH 09/40] move library export sooner --- aarch64_wheel_ci_build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 2971f2d2a..ac5e13c30 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -230,11 +230,12 @@ def start_build(branch="master", if enable_mkldnn: build_ArmComputeLibrary(git_clone_flags) print("build pytorch with mkldnn+acl backend") + os.system(f"export ACL_ROOT_DIR=/acl; export LD_LIBRARY_PATH=/acl/build") build_vars += " USE_MKLDNN=ON USE_MKLDNN_ACL=ON" - os.system(f"cd /pytorch ; export ACL_ROOT_DIR=/acl; {build_vars} python3 setup.py bdist_wheel") + os.system(f"cd /pytorch; {build_vars} python3 setup.py bdist_wheel") print('Repair the wheel') pytorch_wheel_name = list_dir("pytorch/dist")[0] - os.system(f"export LD_LIBRARY_PATH=/acl/build:/pytorch/build/lib; auditwheel repair /pytorch/dist/{pytorch_wheel_name}") + os.system(f"export LD_LIBRARY_PATH=/pytorch/build/lib:$LD_LIBRARY_PATH; auditwheel repair /pytorch/dist/{pytorch_wheel_name}") print('replace the original wheel with the repaired one') pytorch_repaired_wheel_name = list_dir("wheelhouse")[0] os.system(f"cp /wheelhouse/{pytorch_repaired_wheel_name} /pytorch/dist/{pytorch_wheel_name}") From fcad7c831438baff8e4903c4821db52e893ce42b Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 26 Jan 2023 16:33:23 +0000 Subject: [PATCH 10/40] move pytorch repo pull to script --- aarch64_ci_build.sh | 3 --- aarch64_wheel_ci_build.py | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index bec786e38..0a73211fd 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -21,9 +21,6 @@ chmod +x ~/mambaforge.sh ~/mambaforge.sh -b -p /opt/conda rm ~/mambaforge.sh /opt/conda/bin/conda install -y -c conda-forge python=${PYTHON_VERSION} numpy pyyaml setuptools -export CONDA_PYTHON_EXE=/opt/conda/bin/python -export CONDA_EXE=/opt/conda/bin/conda -export PATH=/opt/conda/bin:$PATH python --version conda --version diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index ac5e13c30..7fd18439e 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -215,8 +215,12 @@ def start_build(branch="master", git_clone_flags = " --depth 1 --shallow-submodules" if shallow_clone else "" os.system(f"conda install -y ninja scons") + print("Build and Install OpenBLAS") build_OpenBLAS(git_clone_flags) + print('Checking out PyTorch repo') + os.system(f"cd /; git clone --recurse-submodules -b {branch} https://github.com/pytorch/pytorch {git_clone_flags}") + print('Building PyTorch wheel') # Breakpad build fails on aarch64 build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " From afa4861f70a2ce4bb98c36f2a7ad2e950b2bfa58 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 26 Jan 2023 17:13:38 +0000 Subject: [PATCH 11/40] still working on mkldnn-acl --- aarch64_wheel_ci_build.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 7fd18439e..fa2afe37e 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -21,12 +21,12 @@ def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: print('Building Arm Compute Library') os.system("cd / && mkdir /acl") os.system(f"git clone https://github.com/ARM-software/ComputeLibrary.git -b v22.05 {git_clone_flags}") - os.system(f"cd ComputeLibrary; export acl_install_dir=/acl && " \ - f"scons Werror=1 -j8 debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native build_dir=$acl_install_dir/build && " \ - f"cp -r arm_compute $acl_install_dir && " \ - f"cp -r include $acl_install_dir && " \ - f"cp -r utils $acl_install_dir && " \ - f"cp -r support $acl_install_dir && cd /") + os.system(f"cd ComputeLibrary; export acl_install_dir=/acl; " \ + f"scons Werror=1 -j8 debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native build_dir=$acl_install_dir/build; " \ + f"cp -r arm_compute $acl_install_dir; " \ + f"cp -r include $acl_install_dir; " \ + f"cp -r utils $acl_install_dir; " \ + f"cp -r support $acl_install_dir; cd /") def embed_libgomp(use_conda, wheel_name) -> None: @@ -218,9 +218,6 @@ def start_build(branch="master", print("Build and Install OpenBLAS") build_OpenBLAS(git_clone_flags) - print('Checking out PyTorch repo') - os.system(f"cd /; git clone --recurse-submodules -b {branch} https://github.com/pytorch/pytorch {git_clone_flags}") - print('Building PyTorch wheel') # Breakpad build fails on aarch64 build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " From cbceac3569bfa978390de1923b6f51e5677e1d36 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 26 Jan 2023 19:12:02 +0000 Subject: [PATCH 12/40] Updated OpenBLAS and ACL versions --- aarch64_wheel_ci_build.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index fa2afe37e..5126c279d 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -12,7 +12,7 @@ def list_dir(path: str) -> List[str]: def build_OpenBLAS(git_clone_flags: str = "") -> None: print('Building OpenBLAS') - os.system(f"cd /; git clone https://github.com/xianyi/OpenBLAS -b v0.3.19 {git_clone_flags}") + os.system(f"cd /; git clone https://github.com/xianyi/OpenBLAS -b v0.3.21 {git_clone_flags}") make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8" os.system(f"cd OpenBLAS; make {make_flags} -j8; make {make_flags} install; cd /; rm -rf OpenBLAS") @@ -20,7 +20,7 @@ def build_OpenBLAS(git_clone_flags: str = "") -> None: def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: print('Building Arm Compute Library') os.system("cd / && mkdir /acl") - os.system(f"git clone https://github.com/ARM-software/ComputeLibrary.git -b v22.05 {git_clone_flags}") + os.system(f"git clone https://github.com/ARM-software/ComputeLibrary.git -b v22.11 {git_clone_flags}") os.system(f"cd ComputeLibrary; export acl_install_dir=/acl; " \ f"scons Werror=1 -j8 debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native build_dir=$acl_install_dir/build; " \ f"cp -r arm_compute $acl_install_dir; " \ @@ -224,14 +224,14 @@ def start_build(branch="master", os.system(f"cd /pytorch; pip install -r requirements.txt") if branch == 'nightly': build_date = os.system("git log --pretty=format:%s -1").strip().split()[0].replace("-", "") - version = os.system("cat pytorch/version.txt").strip()[:-2] + version = os.system("cat /pytorch/version.txt").strip()[:-2] build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={version}.dev{build_date} PYTORCH_BUILD_NUMBER=1" if branch.startswith("v1."): build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={branch[1:branch.find('-')]} PYTORCH_BUILD_NUMBER=1" if enable_mkldnn: build_ArmComputeLibrary(git_clone_flags) print("build pytorch with mkldnn+acl backend") - os.system(f"export ACL_ROOT_DIR=/acl; export LD_LIBRARY_PATH=/acl/build") + os.system(f"export ACL_ROOT_DIR=/acl; export LD_LIBRARY_PATH=/acl/build; export ACL_LIBRARY=/acl/build") build_vars += " USE_MKLDNN=ON USE_MKLDNN_ACL=ON" os.system(f"cd /pytorch; {build_vars} python3 setup.py bdist_wheel") print('Repair the wheel') From 550fbe9e53fe08d35068a67d62aae4071c7b87c9 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 26 Jan 2023 23:06:23 +0000 Subject: [PATCH 13/40] copy header to acl --- aarch64_wheel_ci_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 5126c279d..c53ff4cb6 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -26,7 +26,8 @@ def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: f"cp -r arm_compute $acl_install_dir; " \ f"cp -r include $acl_install_dir; " \ f"cp -r utils $acl_install_dir; " \ - f"cp -r support $acl_install_dir; cd /") + f"cp -r support $acl_install_dir; " \ + f"cp -r src $acl_install_dir; cd /") def embed_libgomp(use_conda, wheel_name) -> None: From 0ee58afe76066ab41f0600bd213b227282f02091 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Fri, 27 Jan 2023 00:14:56 +0000 Subject: [PATCH 14/40] ass glfortran flag for OpenBlas links into PT --- aarch64_wheel_ci_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index c53ff4cb6..acfe0a6f3 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -221,7 +221,7 @@ def start_build(branch="master", print('Building PyTorch wheel') # Breakpad build fails on aarch64 - build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " + build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 ADD_LDFLAGS=-lgfortran" os.system(f"cd /pytorch; pip install -r requirements.txt") if branch == 'nightly': build_date = os.system("git log --pretty=format:%s -1").strip().split()[0].replace("-", "") From cf417714b39275ac76ce66b712983af7c17d509d Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Fri, 27 Jan 2023 17:08:16 +0000 Subject: [PATCH 15/40] changing docker image used --- aarch64_ci_build.sh | 16 ++++++++-------- aarch64_wheel_ci_build.py | 3 +-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index 0a73211fd..739a2c6d7 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -8,19 +8,19 @@ PATH=/opt/conda/bin:$PATH ############################################################################### # Install OS dependencies ############################################################################### -echo "Install builder OS dependencies" -apt-get update -apt-get install -y ninja-build g++ git cmake gfortran unzip curl build-essential +# echo "Install builder OS dependencies" +# apt-get update +# apt-get install -y ninja-build g++ git cmake gfortran unzip curl build-essential ############################################################################### # Install conda ############################################################################### echo 'Installing conda-forge' -curl -L -o ~/mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh -chmod +x ~/mambaforge.sh -~/mambaforge.sh -b -p /opt/conda -rm ~/mambaforge.sh -/opt/conda/bin/conda install -y -c conda-forge python=${PYTHON_VERSION} numpy pyyaml setuptools +curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh +chmod +x /mambaforge.sh +/mambaforge.sh -b -p /opt/conda +rm /mambaforge.sh +/opt/conda/bin/conda install -y -c conda-forge python=${PYTHON_VERSION} numpy pyyaml setuptools patchelf python --version conda --version diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index acfe0a6f3..195b6008a 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -31,8 +31,6 @@ def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: def embed_libgomp(use_conda, wheel_name) -> None: - os.system("pip3 install auditwheel") - os.system("conda install -y patchelf") from tempfile import NamedTemporaryFile with NamedTemporaryFile() as tmp: tmp.write(embed_library_script.encode('utf-8')) @@ -223,6 +221,7 @@ def start_build(branch="master", # Breakpad build fails on aarch64 build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 ADD_LDFLAGS=-lgfortran" os.system(f"cd /pytorch; pip install -r requirements.txt") + os.system(f"pip install auditwheel") if branch == 'nightly': build_date = os.system("git log --pretty=format:%s -1").strip().split()[0].replace("-", "") version = os.system("cat /pytorch/version.txt").strip()[:-2] From 5a1612c65b967357da9ee9ea85743c0ec6ceea57 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Fri, 27 Jan 2023 18:11:55 +0000 Subject: [PATCH 16/40] removing added lgfortran env --- aarch64_wheel_ci_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 195b6008a..082744a69 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -219,7 +219,7 @@ def start_build(branch="master", print('Building PyTorch wheel') # Breakpad build fails on aarch64 - build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 ADD_LDFLAGS=-lgfortran" + build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " # removed: ADD_LDFLAGS=-lgfortran os.system(f"cd /pytorch; pip install -r requirements.txt") os.system(f"pip install auditwheel") if branch == 'nightly': From 1b9d4321c6039fcd116e0b1c93834b46bce8796d Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 00:53:29 +0000 Subject: [PATCH 17/40] removing extra cmake flags --- aarch64_wheel_ci_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 082744a69..fb709e232 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -219,7 +219,7 @@ def start_build(branch="master", print('Building PyTorch wheel') # Breakpad build fails on aarch64 - build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " # removed: ADD_LDFLAGS=-lgfortran + build_vars = " " # USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " # removed: ADD_LDFLAGS=-lgfortran os.system(f"cd /pytorch; pip install -r requirements.txt") os.system(f"pip install auditwheel") if branch == 'nightly': From f019e079fe4338c14fe6136a3c810b374bc1dfd6 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 00:56:01 +0000 Subject: [PATCH 18/40] revert change --- aarch64_wheel_ci_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index fb709e232..082744a69 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -219,7 +219,7 @@ def start_build(branch="master", print('Building PyTorch wheel') # Breakpad build fails on aarch64 - build_vars = " " # USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " # removed: ADD_LDFLAGS=-lgfortran + build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " # removed: ADD_LDFLAGS=-lgfortran os.system(f"cd /pytorch; pip install -r requirements.txt") os.system(f"pip install auditwheel") if branch == 'nightly': From 4f83134d8a922dfe56785dc1e6612771e0a31c4b Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 02:30:53 +0000 Subject: [PATCH 19/40] adding fPIC to openblas --- aarch64_wheel_ci_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 082744a69..921e38b19 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -13,7 +13,7 @@ def list_dir(path: str) -> List[str]: def build_OpenBLAS(git_clone_flags: str = "") -> None: print('Building OpenBLAS') os.system(f"cd /; git clone https://github.com/xianyi/OpenBLAS -b v0.3.21 {git_clone_flags}") - make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8" + make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8 -fPIC" os.system(f"cd OpenBLAS; make {make_flags} -j8; make {make_flags} install; cd /; rm -rf OpenBLAS") From ffad868baa6094dccaec308a6cfaed6f8f2324f3 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 02:44:21 +0000 Subject: [PATCH 20/40] ervert -fPIE --- aarch64_wheel_ci_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 921e38b19..4d21b5286 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -13,7 +13,7 @@ def list_dir(path: str) -> List[str]: def build_OpenBLAS(git_clone_flags: str = "") -> None: print('Building OpenBLAS') os.system(f"cd /; git clone https://github.com/xianyi/OpenBLAS -b v0.3.21 {git_clone_flags}") - make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8 -fPIC" + make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8ß" os.system(f"cd OpenBLAS; make {make_flags} -j8; make {make_flags} install; cd /; rm -rf OpenBLAS") From a83f4142b53f0ecbab9000ce115990195faa4f1e Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 15:11:52 +0000 Subject: [PATCH 21/40] more refactor --- aarch64_ci_build.sh | 9 +++------ aarch64_wheel_ci_build.py | 11 +++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index 739a2c6d7..bddccaf75 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -1,16 +1,13 @@ #!/bin/bash set -eux -o pipefail +# This script is used to prepare the Docker container for aarch64_ci_wheel_build.py python script +# as we need to install conda and setup the python version for the build. + CONDA_PYTHON_EXE=/opt/conda/bin/python CONDA_EXE=/opt/conda/bin/conda PATH=/opt/conda/bin:$PATH -############################################################################### -# Install OS dependencies -############################################################################### -# echo "Install builder OS dependencies" -# apt-get update -# apt-get install -y ninja-build g++ git cmake gfortran unzip curl build-essential ############################################################################### # Install conda diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 4d21b5286..72e36963f 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import subprocess from typing import Dict, List, Optional, Tuple, Union @@ -13,7 +12,7 @@ def list_dir(path: str) -> List[str]: def build_OpenBLAS(git_clone_flags: str = "") -> None: print('Building OpenBLAS') os.system(f"cd /; git clone https://github.com/xianyi/OpenBLAS -b v0.3.21 {git_clone_flags}") - make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8ß" + make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8 CMAKE_CXX_FLAGS=-fPIC " os.system(f"cd OpenBLAS; make {make_flags} -j8; make {make_flags} install; cd /; rm -rf OpenBLAS") @@ -222,11 +221,11 @@ def start_build(branch="master", build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " # removed: ADD_LDFLAGS=-lgfortran os.system(f"cd /pytorch; pip install -r requirements.txt") os.system(f"pip install auditwheel") - if branch == 'nightly': - build_date = os.system("git log --pretty=format:%s -1").strip().split()[0].replace("-", "") - version = os.system("cat /pytorch/version.txt").strip()[:-2] + if branch == 'nightly' or branch == 'master': + build_date = str(os.system("git log --pretty=format:%cs -1")).replace("-", "") + version = str(os.system("cat /pytorch/version.txt")).strip()[:-2] build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={version}.dev{build_date} PYTORCH_BUILD_NUMBER=1" - if branch.startswith("v1."): + if branch.startswith("v1.") or branch.startswith("v2."): build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={branch[1:branch.find('-')]} PYTORCH_BUILD_NUMBER=1" if enable_mkldnn: build_ArmComputeLibrary(git_clone_flags) From 76e878421918ea37234411c62c01362c17b8a70e Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 15:22:19 +0000 Subject: [PATCH 22/40] add git safe.directory --- aarch64_ci_build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index bddccaf75..0b4b7c359 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -25,4 +25,7 @@ conda --version # Run aarch64 builder python ############################################################################### cd / +# adding safe directory for git as the permissions will be +# on the mounted pytorch repo +git config --global --add safe.directory /pytorch python /builder/aarch64_wheel_ci_build.py --python-version ${PYTHON_VERSION} --enable-mkldnn From d5f9d8b8613835a6051f8482cb9b73c3fb2450d4 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 16:21:32 +0000 Subject: [PATCH 23/40] fix version parsing --- aarch64_ci_build.sh | 4 ++++ aarch64_wheel_ci_build.py | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index 0b4b7c359..ebe1a818b 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -8,6 +8,10 @@ CONDA_PYTHON_EXE=/opt/conda/bin/python CONDA_EXE=/opt/conda/bin/conda PATH=/opt/conda/bin:$PATH +############################################################################### +# Install OS dependent packages +############################################################################### +yum -y install less ############################################################################### # Install conda diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 72e36963f..33727aac0 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import subprocess from typing import Dict, List, Optional, Tuple, Union @@ -222,8 +223,8 @@ def start_build(branch="master", os.system(f"cd /pytorch; pip install -r requirements.txt") os.system(f"pip install auditwheel") if branch == 'nightly' or branch == 'master': - build_date = str(os.system("git log --pretty=format:%cs -1")).replace("-", "") - version = str(os.system("cat /pytorch/version.txt")).strip()[:-2] + build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1']).decode().replace('-','') + version = subprocess.check_output(['cat','/pytorch/version.txt']).decode().strip()[:-2] build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={version}.dev{build_date} PYTORCH_BUILD_NUMBER=1" if branch.startswith("v1.") or branch.startswith("v2."): build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={branch[1:branch.find('-')]} PYTORCH_BUILD_NUMBER=1" From a534e22bc7ad33d3d7ae934ec672ae596a4f07c8 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 16:38:32 +0000 Subject: [PATCH 24/40] set cwd for check_output --- aarch64_wheel_ci_build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 33727aac0..3acac9f2e 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -223,8 +223,8 @@ def start_build(branch="master", os.system(f"cd /pytorch; pip install -r requirements.txt") os.system(f"pip install auditwheel") if branch == 'nightly' or branch == 'master': - build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1']).decode().replace('-','') - version = subprocess.check_output(['cat','/pytorch/version.txt']).decode().strip()[:-2] + build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1'], cwd='/pytorch').decode().replace('-','') + version = subprocess.check_output(['cat','version.txt'], cwd='/pytorch').decode().strip()[:-2] build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={version}.dev{build_date} PYTORCH_BUILD_NUMBER=1" if branch.startswith("v1.") or branch.startswith("v2."): build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={branch[1:branch.find('-')]} PYTORCH_BUILD_NUMBER=1" From fa3269513a28179f5fca0fb1050b77cbe4a4d18c Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 22:55:20 +0000 Subject: [PATCH 25/40] put in glibfortran.a hack --- aarch64_ci_build.sh | 16 +++++++++++++++- aarch64_wheel_ci_build.py | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index ebe1a818b..d33bac958 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -11,7 +11,7 @@ PATH=/opt/conda/bin:$PATH ############################################################################### # Install OS dependent packages ############################################################################### -yum -y install less +yum -y install less wget zstd ############################################################################### # Install conda @@ -25,6 +25,20 @@ rm /mambaforge.sh python --version conda --version + +############################################################################### +# Exec libglfortran.a hack +# +# libgfortran.a from quay.io/pypa/manylinux2014_aarch64 was not compiled with -fPIC. +# This causes __stack_chk_guard@@GLIBC_2.17 on pytorch build. To solve, get +# ubuntu's libgfortran.a which was compiled with -fPIC +############################################################################### +cd ~/ +wget wget http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/gcc-10/libgfortran-10-dev_10.4.0-6ubuntu1_arm64.deb +ar x libgfortran-10-dev_10.4.0-6ubuntu1_arm64.deb +tar --use-compress-program=unzstd -xvf data.tar.zst +cp -f ~/usr/lib/gcc/aarch64-linux-gnu/10/libgfortran.a /opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10/ + ############################################################################### # Run aarch64 builder python ############################################################################### diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 3acac9f2e..ed405bd6e 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -13,7 +13,7 @@ def list_dir(path: str) -> List[str]: def build_OpenBLAS(git_clone_flags: str = "") -> None: print('Building OpenBLAS') os.system(f"cd /; git clone https://github.com/xianyi/OpenBLAS -b v0.3.21 {git_clone_flags}") - make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8 CMAKE_CXX_FLAGS=-fPIC " + make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8 " os.system(f"cd OpenBLAS; make {make_flags} -j8; make {make_flags} install; cd /; rm -rf OpenBLAS") From 94f19cf3670debf6f89be4f8e6f6f34c59ba8b84 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 22:59:19 +0000 Subject: [PATCH 26/40] adding epel for zstd --- aarch64_ci_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index d33bac958..4637f99f2 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -11,6 +11,7 @@ PATH=/opt/conda/bin:$PATH ############################################################################### # Install OS dependent packages ############################################################################### +yum install epel-release yum -y install less wget zstd ############################################################################### From dbdc2cffbdceb72518dccaa95deb5cc3649a77e1 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 22:59:57 +0000 Subject: [PATCH 27/40] yes on epel --- aarch64_ci_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index 4637f99f2..db8306232 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -11,7 +11,7 @@ PATH=/opt/conda/bin:$PATH ############################################################################### # Install OS dependent packages ############################################################################### -yum install epel-release +yum -y install epel-release yum -y install less wget zstd ############################################################################### From 7ef26db7ec079045389bb6b55254c5d97d145f82 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sat, 28 Jan 2023 23:05:06 +0000 Subject: [PATCH 28/40] more work on the hack --- aarch64_ci_build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index db8306232..6d55c5533 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -35,9 +35,9 @@ conda --version # ubuntu's libgfortran.a which was compiled with -fPIC ############################################################################### cd ~/ -wget wget http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/gcc-10/libgfortran-10-dev_10.4.0-6ubuntu1_arm64.deb -ar x libgfortran-10-dev_10.4.0-6ubuntu1_arm64.deb -tar --use-compress-program=unzstd -xvf data.tar.zst +curl -L -o ~/libgfortran-10-dev.deb http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/gcc-10/libgfortran-10-dev_10.4.0-6ubuntu1_arm64.deb +ar x ~/libgfortran-10-dev.deb +tar --use-compress-program=unzstd -xvf data.tar.zst -C ~/ cp -f ~/usr/lib/gcc/aarch64-linux-gnu/10/libgfortran.a /opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10/ ############################################################################### From d32d8634c233f428242da809ad49fb12f557d066 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sun, 29 Jan 2023 00:09:38 +0000 Subject: [PATCH 29/40] reset list_dir --- aarch64_ci_build.sh | 3 +-- aarch64_wheel_ci_build.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/aarch64_ci_build.sh b/aarch64_ci_build.sh index 6d55c5533..8ba548600 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_ci_build.sh @@ -12,7 +12,7 @@ PATH=/opt/conda/bin:$PATH # Install OS dependent packages ############################################################################### yum -y install epel-release -yum -y install less wget zstd +yum -y install less zstd ############################################################################### # Install conda @@ -26,7 +26,6 @@ rm /mambaforge.sh python --version conda --version - ############################################################################### # Exec libglfortran.a hack # diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index ed405bd6e..dd3d6a102 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -7,7 +7,7 @@ def list_dir(path: str) -> List[str]: - return os.system(["ls", "-1", path]).split("\n") + return subprocess.check_output(["ls", "-1", path]).decode().split("\n") def build_OpenBLAS(git_clone_flags: str = "") -> None: @@ -219,7 +219,7 @@ def start_build(branch="master", print('Building PyTorch wheel') # Breakpad build fails on aarch64 - build_vars = "USE_BREAKPAD=0 CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " # removed: ADD_LDFLAGS=-lgfortran + build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " os.system(f"cd /pytorch; pip install -r requirements.txt") os.system(f"pip install auditwheel") if branch == 'nightly' or branch == 'master': From de2c4e167be4fc71e84a3c76e9009c91d1dd65c6 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sun, 29 Jan 2023 00:11:57 +0000 Subject: [PATCH 30/40] add a clean before build --- aarch64_wheel_ci_build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index dd3d6a102..1ef8cd3b8 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -222,6 +222,7 @@ def start_build(branch="master", build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " os.system(f"cd /pytorch; pip install -r requirements.txt") os.system(f"pip install auditwheel") + os.system(f"python setup.py clean") if branch == 'nightly' or branch == 'master': build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1'], cwd='/pytorch').decode().replace('-','') version = subprocess.check_output(['cat','version.txt'], cwd='/pytorch').decode().strip()[:-2] From 768f0970d5d9a2023cc5d293573aa290fad62393 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sun, 29 Jan 2023 03:18:50 +0000 Subject: [PATCH 31/40] new way to get versions. --- aarch64_wheel_ci_build.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/aarch64_wheel_ci_build.py b/aarch64_wheel_ci_build.py index 1ef8cd3b8..a418570af 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_wheel_ci_build.py @@ -80,11 +80,13 @@ def build_torchvision(branch: str = "main", print('Building TorchVision wheel') build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " if branch == 'nightly': - version = os.system(["if [ -f vision/version.txt ]; then cat vision/version.txt; fi"]).strip() + version = '' + if os.path.exists('/vision/version.txt'): + version = subprocess.check_output(['cat', '/vision/version.txt']).decode().strip() if len(version) == 0: # In older revisions, version was embedded in setup.py - version = os.system(["grep", "\"version = '\"", "vision/setup.py"]).strip().split("'")[1][:-2] - build_date = os.system("cd /pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + version = subprocess.check_output(['grep', 'version', 'setup.py']).decode().strip().split('\'')[1][:-2] + build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1'], cwd='/vision').decode().replace('-','') build_vars += f"BUILD_VERSION={version}.dev{build_date}" elif build_version is not None: build_vars += f"BUILD_VERSION={build_version}" @@ -121,8 +123,10 @@ def build_torchtext(branch: str = "main", print('Building TorchText wheel') build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " if branch == 'nightly': - version = os.system(["if [ -f text/version.txt ]; then cat text/version.txt; fi"]).strip() - build_date = os.system("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + version = '' + if os.path.exists('/text/version.txt'): + version = subprocess.check_output(['cat', '/text/version.txt']).decode().strip() + build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1'], cwd='/text').decode().replace('-','') build_vars += f"BUILD_VERSION={version}.dev{build_date}" elif build_version is not None: build_vars += f"BUILD_VERSION={build_version}" @@ -158,8 +162,10 @@ def build_torchaudio(branch: str = "main", print('Building TorchAudio wheel') build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " if branch == 'nightly': - version = os.system(["grep", "\"version = '\"", "audio/setup.py"]).strip().split("'")[1][:-2] - build_date = os.system("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + version = '' + if os.path.exists('/audio/version.txt'): + version = subprocess.check_output(['cat', '/audio/version.txt']).decode().strip() + build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1'], cwd='/audio').decode().replace('-','') build_vars += f"BUILD_VERSION={version}.dev{build_date}" elif build_version is not None: build_vars += f"BUILD_VERSION={build_version}" @@ -190,8 +196,10 @@ def build_torchdata(branch: str = "main", print('Building TorchData wheel') build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " if branch == 'nightly': - version = os.system(["grep", "\"version = '\"", "audio/setup.py"]).strip().split("'")[1][:-2] - build_date = os.system("cd /pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "") + version = '' + if os.path.exists('/data/version.txt'): + version = subprocess.check_output(['cat', '/data/version.txt']).decode().strip() + build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1'], cwd='/data').decode().replace('-','') build_vars += f"BUILD_VERSION={version}.dev{build_date}" elif build_version is not None: build_vars += f"BUILD_VERSION={build_version}" @@ -247,7 +255,7 @@ def start_build(branch="master", print("Deleting build folder") os.system("cd /pytorch; rm -rf build") - pytorch_wheel_name = os.system("ls /pytorch/dist")[0] + pytorch_wheel_name = list_dir("/pytorch/dist")[0] embed_libgomp(use_conda, os.path.join('pytorch', 'dist', pytorch_wheel_name)) print('Move PyTorch wheel to artfacts') os.system(f"mv /pytorch/dist/{pytorch_wheel_name} /artifacts/") From 6beeb4735d22d4ecd1e9fa111f05ba71c720f7d5 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sun, 29 Jan 2023 04:47:48 +0000 Subject: [PATCH 32/40] new folder for aarch64 --- .../aarch64_ci_build.sh | 2 +- .../aarch64_wheel_ci_build.py | 88 +------------------ .../build_aarch64_wheel.py | 0 aarch64_linux/embed_library.py | 72 +++++++++++++++ 4 files changed, 76 insertions(+), 86 deletions(-) rename aarch64_ci_build.sh => aarch64_linux/aarch64_ci_build.sh (95%) rename aarch64_wheel_ci_build.py => aarch64_linux/aarch64_wheel_ci_build.py (81%) rename build_aarch64_wheel.py => aarch64_linux/build_aarch64_wheel.py (100%) create mode 100644 aarch64_linux/embed_library.py diff --git a/aarch64_ci_build.sh b/aarch64_linux/aarch64_ci_build.sh similarity index 95% rename from aarch64_ci_build.sh rename to aarch64_linux/aarch64_ci_build.sh index 8ba548600..e1bb6b3f4 100644 --- a/aarch64_ci_build.sh +++ b/aarch64_linux/aarch64_ci_build.sh @@ -46,4 +46,4 @@ cd / # adding safe directory for git as the permissions will be # on the mounted pytorch repo git config --global --add safe.directory /pytorch -python /builder/aarch64_wheel_ci_build.py --python-version ${PYTHON_VERSION} --enable-mkldnn +python /builder/aarch64_linux/aarch64_wheel_ci_build.py --python-version ${PYTHON_VERSION} --enable-mkldnn diff --git a/aarch64_wheel_ci_build.py b/aarch64_linux/aarch64_wheel_ci_build.py similarity index 81% rename from aarch64_wheel_ci_build.py rename to aarch64_linux/aarch64_wheel_ci_build.py index a418570af..29ea5ba54 100755 --- a/aarch64_wheel_ci_build.py +++ b/aarch64_linux/aarch64_wheel_ci_build.py @@ -30,15 +30,9 @@ def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: f"cp -r src $acl_install_dir; cd /") -def embed_libgomp(use_conda, wheel_name) -> None: - from tempfile import NamedTemporaryFile - with NamedTemporaryFile() as tmp: - tmp.write(embed_library_script.encode('utf-8')) - tmp.flush() - os.system(f"mv {tmp.name} ./embed_library.py") - +def embed_libgomp(wheel_name) -> None: print('Embedding libgomp into wheel') - os.system(f"python3 embed_library.py {wheel_name} --update-tag") + os.system(f"python3 /builder/aarch64_linux/embed_library.py {wheel_name} --update-tag") def checkout_repo(branch: str = "master", @@ -256,7 +250,7 @@ def start_build(branch="master", print("Deleting build folder") os.system("cd /pytorch; rm -rf build") pytorch_wheel_name = list_dir("/pytorch/dist")[0] - embed_libgomp(use_conda, os.path.join('pytorch', 'dist', pytorch_wheel_name)) + embed_libgomp(f"/pytorch/dist/{pytorch_wheel_name}") print('Move PyTorch wheel to artfacts') os.system(f"mv /pytorch/dist/{pytorch_wheel_name} /artifacts/") @@ -266,82 +260,6 @@ def start_build(branch="master", data_wheel_name = build_torchdata(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) return [pytorch_wheel_name, vision_wheel_name, audio_wheel_name, text_wheel_name, data_wheel_name] - -embed_library_script = """ -#!/usr/bin/env python3 - -from auditwheel.patcher import Patchelf -from auditwheel.wheeltools import InWheelCtx -from auditwheel.elfutils import elf_file_filter -from auditwheel.repair import copylib -from auditwheel.lddtree import lddtree -from subprocess import check_call -import os -import shutil -import sys -from tempfile import TemporaryDirectory - - -def replace_tag(filename): - with open(filename, 'r') as f: - lines = f.read().split("\\n") - for i,line in enumerate(lines): - if not line.startswith("Tag: "): - continue - lines[i] = line.replace("-linux_", "-manylinux2014_") - print(f'Updated tag from {line} to {lines[i]}') - - with open(filename, 'w') as f: - f.write("\\n".join(lines)) - - -class AlignedPatchelf(Patchelf): - def set_soname(self, file_name: str, new_soname: str) -> None: - check_call(['patchelf', '--page-size', '65536', '--set-soname', new_soname, file_name]) - - def replace_needed(self, file_name: str, soname: str, new_soname: str) -> None: - check_call(['patchelf', '--page-size', '65536', '--replace-needed', soname, new_soname, file_name]) - - -def embed_library(whl_path, lib_soname, update_tag=False): - patcher = AlignedPatchelf() - out_dir = TemporaryDirectory() - whl_name = os.path.basename(whl_path) - tmp_whl_name = os.path.join(out_dir.name, whl_name) - with InWheelCtx(whl_path) as ctx: - torchlib_path = os.path.join(ctx._tmpdir.name, 'torch', 'lib') - ctx.out_wheel=tmp_whl_name - new_lib_path, new_lib_soname = None, None - for filename, elf in elf_file_filter(ctx.iter_files()): - if not filename.startswith('torch/lib'): - continue - libtree = lddtree(filename) - if lib_soname not in libtree['needed']: - continue - lib_path = libtree['libs'][lib_soname]['path'] - if lib_path is None: - print(f"Can't embed {lib_soname} as it could not be found") - break - if lib_path.startswith(torchlib_path): - continue - - if new_lib_path is None: - new_lib_soname, new_lib_path = copylib(lib_path, torchlib_path, patcher) - patcher.replace_needed(filename, lib_soname, new_lib_soname) - print(f'Replacing {lib_soname} with {new_lib_soname} for {filename}') - if update_tag: - # Add manylinux2014 tag - for filename in ctx.iter_files(): - if os.path.basename(filename) != 'WHEEL': - continue - replace_tag(filename) - shutil.move(tmp_whl_name, whl_path) - - -if __name__ == '__main__': - embed_library(sys.argv[1], 'libgomp.so.1', len(sys.argv) > 2 and sys.argv[2] == '--update-tag') -""" - def parse_arguments(): from argparse import ArgumentParser parser = ArgumentParser("AARCH64 wheels python CD") diff --git a/build_aarch64_wheel.py b/aarch64_linux/build_aarch64_wheel.py similarity index 100% rename from build_aarch64_wheel.py rename to aarch64_linux/build_aarch64_wheel.py diff --git a/aarch64_linux/embed_library.py b/aarch64_linux/embed_library.py new file mode 100644 index 000000000..978970d45 --- /dev/null +++ b/aarch64_linux/embed_library.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +from auditwheel.patcher import Patchelf +from auditwheel.wheeltools import InWheelCtx +from auditwheel.elfutils import elf_file_filter +from auditwheel.repair import copylib +from auditwheel.lddtree import lddtree +from subprocess import check_call +import os +import shutil +import sys +from tempfile import TemporaryDirectory + + +def replace_tag(filename): + with open(filename, 'r') as f: + lines = f.read().split("\\n") + for i,line in enumerate(lines): + if not line.startswith("Tag: "): + continue + lines[i] = line.replace("-linux_", "-manylinux2014_") + print(f'Updated tag from {line} to {lines[i]}') + + with open(filename, 'w') as f: + f.write("\\n".join(lines)) + + +class AlignedPatchelf(Patchelf): + def set_soname(self, file_name: str, new_soname: str) -> None: + check_call(['patchelf', '--page-size', '65536', '--set-soname', new_soname, file_name]) + + def replace_needed(self, file_name: str, soname: str, new_soname: str) -> None: + check_call(['patchelf', '--page-size', '65536', '--replace-needed', soname, new_soname, file_name]) + + +def embed_library(whl_path, lib_soname, update_tag=False): + patcher = AlignedPatchelf() + out_dir = TemporaryDirectory() + whl_name = os.path.basename(whl_path) + tmp_whl_name = os.path.join(out_dir.name, whl_name) + with InWheelCtx(whl_path) as ctx: + torchlib_path = os.path.join(ctx._tmpdir.name, 'torch', 'lib') + ctx.out_wheel=tmp_whl_name + new_lib_path, new_lib_soname = None, None + for filename, elf in elf_file_filter(ctx.iter_files()): + if not filename.startswith('torch/lib'): + continue + libtree = lddtree(filename) + if lib_soname not in libtree['needed']: + continue + lib_path = libtree['libs'][lib_soname]['path'] + if lib_path is None: + print(f"Can't embed {lib_soname} as it could not be found") + break + if lib_path.startswith(torchlib_path): + continue + + if new_lib_path is None: + new_lib_soname, new_lib_path = copylib(lib_path, torchlib_path, patcher) + patcher.replace_needed(filename, lib_soname, new_lib_soname) + print(f'Replacing {lib_soname} with {new_lib_soname} for {filename}') + if update_tag: + # Add manylinux2014 tag + for filename in ctx.iter_files(): + if os.path.basename(filename) != 'WHEEL': + continue + replace_tag(filename) + shutil.move(tmp_whl_name, whl_path) + + +if __name__ == '__main__': + embed_library(sys.argv[1], 'libgomp.so.1', len(sys.argv) > 2 and sys.argv[2] == '--update-tag') From 031b05d76fd312c9d72e692cfe748c74f2ad19fd Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sun, 29 Jan 2023 14:15:41 +0000 Subject: [PATCH 33/40] fix embed_lobgomp and cleanup --- aarch64_linux/aarch64_wheel_ci_build.py | 30 +++++++++++-------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/aarch64_linux/aarch64_wheel_ci_build.py b/aarch64_linux/aarch64_wheel_ci_build.py index 29ea5ba54..15c719a76 100755 --- a/aarch64_linux/aarch64_wheel_ci_build.py +++ b/aarch64_linux/aarch64_wheel_ci_build.py @@ -2,7 +2,7 @@ import os import subprocess -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple @@ -51,7 +51,6 @@ def checkout_repo(branch: str = "master", def build_torchvision(branch: str = "main", - use_conda: bool = True, git_clone_flags: str = "") -> str: print('Checking out TorchVision repo') build_version = checkout_repo(branch=branch, @@ -86,8 +85,8 @@ def build_torchvision(branch: str = "main", build_vars += f"BUILD_VERSION={build_version}" os.system(f"cd /vision; {build_vars} python3 setup.py bdist_wheel") - wheel_name = os.system("ls /vision/dist")[0] - embed_libgomp(use_conda, os.path.join('vision', 'dist', wheel_name)) + wheel_name = list_dir("ls /vision/dist")[0] + embed_libgomp(f"/vision/dist/{wheel_name}") print('Move TorchVision wheel to artfacts') os.system(f"mv /vision/dist/{wheel_name} /artifacts/") @@ -95,7 +94,6 @@ def build_torchvision(branch: str = "main", def build_torchtext(branch: str = "main", - use_conda: bool = True, git_clone_flags: str = "") -> str: print('Checking out TorchText repo') os.system(f"cd /") @@ -126,8 +124,8 @@ def build_torchtext(branch: str = "main", build_vars += f"BUILD_VERSION={build_version}" os.system(f"cd text; {build_vars} python3 setup.py bdist_wheel") - wheel_name = os.system("ls /text/dist")[0] - embed_libgomp(use_conda, os.path.join('text', 'dist', wheel_name)) + wheel_name = list_dir("ls /text/dist")[0] + embed_libgomp(f"/text/dist/{wheel_name}") print('Move TorchText wheel to artfacts') os.system(f"mv /text/dist/{wheel_name} /artifacts/") @@ -135,7 +133,6 @@ def build_torchtext(branch: str = "main", def build_torchaudio(branch: str = "main", - use_conda: bool = True, git_clone_flags: str = "") -> str: print('Checking out TorchAudio repo') git_clone_flags += " --recurse-submodules" @@ -165,8 +162,8 @@ def build_torchaudio(branch: str = "main", build_vars += f"BUILD_VERSION={build_version}" os.system(f"cd /audio; {build_vars} python3 setup.py bdist_wheel") - wheel_name = os.system("ls /audio/dist")[0] - embed_libgomp(use_conda, os.path.join('audio', 'dist', wheel_name)) + wheel_name = list_dir("ls /audio/dist")[0] + embed_libgomp(f"/audio/dist/{wheel_name}") print('Move TorchAudio wheel to artfacts') os.system(f"mv /audio/dist/{wheel_name} /artifacts/") @@ -174,7 +171,6 @@ def build_torchaudio(branch: str = "main", def build_torchdata(branch: str = "main", - use_conda: bool = True, git_clone_flags: str = "") -> str: print('Checking out TorchData repo') git_clone_flags += " --recurse-submodules" @@ -199,8 +195,8 @@ def build_torchdata(branch: str = "main", build_vars += f"BUILD_VERSION={build_version}" os.system(f"cd /data; {build_vars} python3 setup.py bdist_wheel") - wheel_name = os.system("ls /data/dist")[0] - embed_libgomp(use_conda, os.path.join('data', 'dist', wheel_name)) + wheel_name = list_dir("ls /data/dist")[0] + embed_libgomp(f"/data/dist/{wheel_name}") print('Move TorchAudio wheel to artfacts') os.system(f"mv /data/dist/{wheel_name} /artifacts/") @@ -254,10 +250,10 @@ def start_build(branch="master", print('Move PyTorch wheel to artfacts') os.system(f"mv /pytorch/dist/{pytorch_wheel_name} /artifacts/") - vision_wheel_name = build_torchvision(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) - audio_wheel_name = build_torchaudio(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) - text_wheel_name = build_torchtext(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) - data_wheel_name = build_torchdata(branch=branch, use_conda=use_conda, git_clone_flags=git_clone_flags) + vision_wheel_name = build_torchvision(branch=branch, git_clone_flags=git_clone_flags) + audio_wheel_name = build_torchaudio(branch=branch, git_clone_flags=git_clone_flags) + text_wheel_name = build_torchtext(branch=branch, git_clone_flags=git_clone_flags) + data_wheel_name = build_torchdata(branch=branch, git_clone_flags=git_clone_flags) return [pytorch_wheel_name, vision_wheel_name, audio_wheel_name, text_wheel_name, data_wheel_name] def parse_arguments(): From 137527beebc3989bdf1e0816ab8110fe19ad1e83 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sun, 29 Jan 2023 15:20:43 +0000 Subject: [PATCH 34/40] installing torch wheel for other builds --- aarch64_linux/aarch64_wheel_ci_build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aarch64_linux/aarch64_wheel_ci_build.py b/aarch64_linux/aarch64_wheel_ci_build.py index 15c719a76..811624f7c 100755 --- a/aarch64_linux/aarch64_wheel_ci_build.py +++ b/aarch64_linux/aarch64_wheel_ci_build.py @@ -249,13 +249,16 @@ def start_build(branch="master", embed_libgomp(f"/pytorch/dist/{pytorch_wheel_name}") print('Move PyTorch wheel to artfacts') os.system(f"mv /pytorch/dist/{pytorch_wheel_name} /artifacts/") - + print("Installing Pytorch wheel") + os.system(f"pip install /artifacts/{pytorch_wheel_name}") + vision_wheel_name = build_torchvision(branch=branch, git_clone_flags=git_clone_flags) audio_wheel_name = build_torchaudio(branch=branch, git_clone_flags=git_clone_flags) text_wheel_name = build_torchtext(branch=branch, git_clone_flags=git_clone_flags) data_wheel_name = build_torchdata(branch=branch, git_clone_flags=git_clone_flags) return [pytorch_wheel_name, vision_wheel_name, audio_wheel_name, text_wheel_name, data_wheel_name] + def parse_arguments(): from argparse import ArgumentParser parser = ArgumentParser("AARCH64 wheels python CD") From 521e3233de60a8df5cefaa93809a1594cdf56b0c Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Sun, 29 Jan 2023 16:23:53 +0000 Subject: [PATCH 35/40] fix calls to list_dir --- aarch64_linux/aarch64_wheel_ci_build.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aarch64_linux/aarch64_wheel_ci_build.py b/aarch64_linux/aarch64_wheel_ci_build.py index 811624f7c..2453bd751 100755 --- a/aarch64_linux/aarch64_wheel_ci_build.py +++ b/aarch64_linux/aarch64_wheel_ci_build.py @@ -85,7 +85,7 @@ def build_torchvision(branch: str = "main", build_vars += f"BUILD_VERSION={build_version}" os.system(f"cd /vision; {build_vars} python3 setup.py bdist_wheel") - wheel_name = list_dir("ls /vision/dist")[0] + wheel_name = list_dir("/vision/dist")[0] embed_libgomp(f"/vision/dist/{wheel_name}") print('Move TorchVision wheel to artfacts') @@ -124,7 +124,7 @@ def build_torchtext(branch: str = "main", build_vars += f"BUILD_VERSION={build_version}" os.system(f"cd text; {build_vars} python3 setup.py bdist_wheel") - wheel_name = list_dir("ls /text/dist")[0] + wheel_name = list_dir("/text/dist")[0] embed_libgomp(f"/text/dist/{wheel_name}") print('Move TorchText wheel to artfacts') @@ -162,7 +162,7 @@ def build_torchaudio(branch: str = "main", build_vars += f"BUILD_VERSION={build_version}" os.system(f"cd /audio; {build_vars} python3 setup.py bdist_wheel") - wheel_name = list_dir("ls /audio/dist")[0] + wheel_name = list_dir("/audio/dist")[0] embed_libgomp(f"/audio/dist/{wheel_name}") print('Move TorchAudio wheel to artfacts') @@ -195,7 +195,7 @@ def build_torchdata(branch: str = "main", build_vars += f"BUILD_VERSION={build_version}" os.system(f"cd /data; {build_vars} python3 setup.py bdist_wheel") - wheel_name = list_dir("ls /data/dist")[0] + wheel_name = list_dir("/data/dist")[0] embed_libgomp(f"/data/dist/{wheel_name}") print('Move TorchAudio wheel to artfacts') From 83089d98e5b93c9e677d7659bb71e8b1533a5de1 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 2 Feb 2023 17:17:01 +0000 Subject: [PATCH 36/40] Refactor python and env update --- aarch64_linux/aarch64_ci_build.sh | 8 +- aarch64_linux/aarch64_wheel_ci_build.py | 193 ++++++++++++++---------- 2 files changed, 114 insertions(+), 87 deletions(-) diff --git a/aarch64_linux/aarch64_ci_build.sh b/aarch64_linux/aarch64_ci_build.sh index e1bb6b3f4..122793716 100644 --- a/aarch64_linux/aarch64_ci_build.sh +++ b/aarch64_linux/aarch64_ci_build.sh @@ -22,16 +22,16 @@ curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/late chmod +x /mambaforge.sh /mambaforge.sh -b -p /opt/conda rm /mambaforge.sh -/opt/conda/bin/conda install -y -c conda-forge python=${PYTHON_VERSION} numpy pyyaml setuptools patchelf +/opt/conda/bin/conda install -y -c conda-forge python=${DESIRED_PYTHON} numpy pyyaml setuptools patchelf python --version conda --version ############################################################################### # Exec libglfortran.a hack # -# libgfortran.a from quay.io/pypa/manylinux2014_aarch64 was not compiled with -fPIC. +# libgfortran.a from quay.io/pypa/manylinux2014_aarch64 is not compiled with -fPIC. # This causes __stack_chk_guard@@GLIBC_2.17 on pytorch build. To solve, get -# ubuntu's libgfortran.a which was compiled with -fPIC +# ubuntu's libgfortran.a which is compiled with -fPIC ############################################################################### cd ~/ curl -L -o ~/libgfortran-10-dev.deb http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/gcc-10/libgfortran-10-dev_10.4.0-6ubuntu1_arm64.deb @@ -46,4 +46,4 @@ cd / # adding safe directory for git as the permissions will be # on the mounted pytorch repo git config --global --add safe.directory /pytorch -python /builder/aarch64_linux/aarch64_wheel_ci_build.py --python-version ${PYTHON_VERSION} --enable-mkldnn +python /builder/aarch64_linux/aarch64_wheel_ci_build.py --python-version ${DESIRED_PYTHON} --enable-mkldnn diff --git a/aarch64_linux/aarch64_wheel_ci_build.py b/aarch64_linux/aarch64_wheel_ci_build.py index 2453bd751..bc891d2ae 100755 --- a/aarch64_linux/aarch64_wheel_ci_build.py +++ b/aarch64_linux/aarch64_wheel_ci_build.py @@ -5,11 +5,34 @@ from typing import Dict, List, Optional, Tuple - +'''' +Helper for getting paths for Python +''' def list_dir(path: str) -> List[str]: return subprocess.check_output(["ls", "-1", path]).decode().split("\n") +''' +Helper to get repo branches for specific versions +''' +def checkout_repo(branch: str = "main", + url: str = "", + git_clone_flags: str = "", + mapping: Dict[str, Tuple[str, str]] = []) -> Optional[str]: + for prefix in mapping: + if not branch.startswith(prefix): + continue + tag = f"v{mapping[prefix][0]}-{mapping[prefix][1]}" + os.system(f"git clone {url} -b {tag} {git_clone_flags}") + return mapping[prefix][0] + + os.system(f"git clone {url} {git_clone_flags}") + return None + + +''' +Using OpenBLAS with PyTorch +''' def build_OpenBLAS(git_clone_flags: str = "") -> None: print('Building OpenBLAS') os.system(f"cd /; git clone https://github.com/xianyi/OpenBLAS -b v0.3.21 {git_clone_flags}") @@ -17,6 +40,9 @@ def build_OpenBLAS(git_clone_flags: str = "") -> None: os.system(f"cd OpenBLAS; make {make_flags} -j8; make {make_flags} install; cd /; rm -rf OpenBLAS") +''' +Using ArmComputeLibrary for aarch64 PyTorch +''' def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: print('Building Arm Compute Library') os.system("cd / && mkdir /acl") @@ -30,26 +56,17 @@ def build_ArmComputeLibrary(git_clone_flags: str = "") -> None: f"cp -r src $acl_install_dir; cd /") +''' +Script to embed libgomp to the wheels +''' def embed_libgomp(wheel_name) -> None: print('Embedding libgomp into wheel') os.system(f"python3 /builder/aarch64_linux/embed_library.py {wheel_name} --update-tag") -def checkout_repo(branch: str = "master", - url: str = "", - git_clone_flags: str = "", - mapping: Dict[str, Tuple[str, str]] = []) -> Optional[str]: - for prefix in mapping: - if not branch.startswith(prefix): - continue - tag = f"v{mapping[prefix][0]}-{mapping[prefix][1]}" - os.system(f"git clone {url} -b {tag} {git_clone_flags}") - return mapping[prefix][0] - - os.system(f"git clone {url} {git_clone_flags}") - return None - - +''' +Build TorchVision wheel +''' def build_torchvision(branch: str = "main", git_clone_flags: str = "") -> str: print('Checking out TorchVision repo') @@ -93,6 +110,50 @@ def build_torchvision(branch: str = "main", return wheel_name +''' +Build TorchAudio wheel +''' +def build_torchaudio(branch: str = "main", + git_clone_flags: str = "") -> str: + print('Checking out TorchAudio repo') + git_clone_flags += " --recurse-submodules" + build_version = checkout_repo(branch=branch, + url="https://github.com/pytorch/audio", + git_clone_flags=git_clone_flags, + mapping={ + "v1.9.0": ("0.9.0", "rc2"), + "v1.10.0": ("0.10.0", "rc5"), + "v1.10.1": ("0.10.1", "rc1"), + "v1.10.2": ("0.10.2", "rc1"), + "v1.11.0": ("0.11.0", "rc1"), + "v1.12.0": ("0.12.0", "rc3"), + "v1.12.1": ("0.12.1", "rc5"), + "v1.13.0": ("0.13.0", "rc4"), + "v1.13.1": ("0.13.1", "rc2"), + }) + print('Building TorchAudio wheel') + build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " + if branch == 'nightly': + version = '' + if os.path.exists('/audio/version.txt'): + version = subprocess.check_output(['cat', '/audio/version.txt']).decode().strip() + build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1'], cwd='/audio').decode().replace('-','') + build_vars += f"BUILD_VERSION={version}.dev{build_date}" + elif build_version is not None: + build_vars += f"BUILD_VERSION={build_version}" + + os.system(f"cd /audio; {build_vars} python3 setup.py bdist_wheel") + wheel_name = list_dir("/audio/dist")[0] + embed_libgomp(f"/audio/dist/{wheel_name}") + + print('Move TorchAudio wheel to artfacts') + os.system(f"mv /audio/dist/{wheel_name} /artifacts/") + return wheel_name + + +''' +Build TorchText wheel +''' def build_torchtext(branch: str = "main", git_clone_flags: str = "") -> str: print('Checking out TorchText repo') @@ -132,44 +193,9 @@ def build_torchtext(branch: str = "main", return wheel_name -def build_torchaudio(branch: str = "main", - git_clone_flags: str = "") -> str: - print('Checking out TorchAudio repo') - git_clone_flags += " --recurse-submodules" - build_version = checkout_repo(branch=branch, - url="https://github.com/pytorch/audio", - git_clone_flags=git_clone_flags, - mapping={ - "v1.9.0": ("0.9.0", "rc2"), - "v1.10.0": ("0.10.0", "rc5"), - "v1.10.1": ("0.10.1", "rc1"), - "v1.10.2": ("0.10.2", "rc1"), - "v1.11.0": ("0.11.0", "rc1"), - "v1.12.0": ("0.12.0", "rc3"), - "v1.12.1": ("0.12.1", "rc5"), - "v1.13.0": ("0.13.0", "rc4"), - "v1.13.1": ("0.13.1", "rc2"), - }) - print('Building TorchAudio wheel') - build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " - if branch == 'nightly': - version = '' - if os.path.exists('/audio/version.txt'): - version = subprocess.check_output(['cat', '/audio/version.txt']).decode().strip() - build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1'], cwd='/audio').decode().replace('-','') - build_vars += f"BUILD_VERSION={version}.dev{build_date}" - elif build_version is not None: - build_vars += f"BUILD_VERSION={build_version}" - - os.system(f"cd /audio; {build_vars} python3 setup.py bdist_wheel") - wheel_name = list_dir("/audio/dist")[0] - embed_libgomp(f"/audio/dist/{wheel_name}") - - print('Move TorchAudio wheel to artfacts') - os.system(f"mv /audio/dist/{wheel_name} /artifacts/") - return wheel_name - - +''' +Build TorchData wheel +''' def build_torchdata(branch: str = "main", git_clone_flags: str = "") -> str: print('Checking out TorchData repo') @@ -203,24 +229,40 @@ def build_torchdata(branch: str = "main", return wheel_name -def start_build(branch="master", - compiler="gcc-8", - use_conda=True, - python_version="3.8", - shallow_clone=True, - enable_mkldnn=False) -> Tuple[str, str]: - git_clone_flags = " --depth 1 --shallow-submodules" if shallow_clone else "" +def parse_arguments(): + from argparse import ArgumentParser + parser = ArgumentParser("AARCH64 wheels python CD") + parser.add_argument("--debug", action="store_true") + parser.add_argument("--build-only", action="store_true") + parser.add_argument("--test-only", type=str) + parser.add_argument("--python-version", type=str, choices=['3.6', '3.7', '3.8', '3.9', '3.10'], default=None) + parser.add_argument("--branch", type=str, default="master") + parser.add_argument("--compiler", type=str, choices=['gcc-7', 'gcc-8', 'gcc-9', 'clang'], default="gcc-8") + parser.add_argument("--enable-mkldnn", action="store_true") + return parser.parse_args() + + +''' +Entry Point +''' +if __name__ == '__main__': + + args = parse_arguments() + branch = args.branch + enable_mkldnn = args.enable_mkldnn + + git_clone_flags = " --depth 1 --shallow-submodules" os.system(f"conda install -y ninja scons") print("Build and Install OpenBLAS") build_OpenBLAS(git_clone_flags) print('Building PyTorch wheel') - # Breakpad build fails on aarch64 build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 " os.system(f"cd /pytorch; pip install -r requirements.txt") os.system(f"pip install auditwheel") os.system(f"python setup.py clean") + if branch == 'nightly' or branch == 'master': build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1'], cwd='/pytorch').decode().replace('-','') version = subprocess.check_output(['cat','version.txt'], cwd='/pytorch').decode().strip()[:-2] @@ -256,25 +298,10 @@ def start_build(branch="master", audio_wheel_name = build_torchaudio(branch=branch, git_clone_flags=git_clone_flags) text_wheel_name = build_torchtext(branch=branch, git_clone_flags=git_clone_flags) data_wheel_name = build_torchdata(branch=branch, git_clone_flags=git_clone_flags) - return [pytorch_wheel_name, vision_wheel_name, audio_wheel_name, text_wheel_name, data_wheel_name] - - -def parse_arguments(): - from argparse import ArgumentParser - parser = ArgumentParser("AARCH64 wheels python CD") - parser.add_argument("--debug", action="store_true") - parser.add_argument("--build-only", action="store_true") - parser.add_argument("--test-only", type=str) - parser.add_argument("--python-version", type=str, choices=['3.6', '3.7', '3.8', '3.9', '3.10'], default=None) - parser.add_argument("--branch", type=str, default="master") - parser.add_argument("--compiler", type=str, choices=['gcc-7', 'gcc-8', 'gcc-9', 'clang'], default="gcc-8") - parser.add_argument("--enable-mkldnn", action="store_true") - return parser.parse_args() - -if __name__ == '__main__': - args = parse_arguments() - start_build(branch=args.branch, - compiler=args.compiler, - python_version=args.python_version, - enable_mkldnn=args.enable_mkldnn) + print(f"Wheels Created:\n" \ + f"{pytorch_wheel_name}\n" \ + f"{vision_wheel_name}\n" \ + f"{audio_wheel_name}\n" \ + f"{text_wheel_name}\n" \ + f"{data_wheel_name}\n") From 5b46184043c754fe014799649c9ac9d359805045 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 2 Feb 2023 17:25:59 +0000 Subject: [PATCH 37/40] adding fix for conda cert issue --- aarch64_linux/aarch64_ci_build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aarch64_linux/aarch64_ci_build.sh b/aarch64_linux/aarch64_ci_build.sh index 122793716..3c011de4d 100644 --- a/aarch64_linux/aarch64_ci_build.sh +++ b/aarch64_linux/aarch64_ci_build.sh @@ -7,6 +7,7 @@ set -eux -o pipefail CONDA_PYTHON_EXE=/opt/conda/bin/python CONDA_EXE=/opt/conda/bin/conda PATH=/opt/conda/bin:$PATH +REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt ############################################################################### # Install OS dependent packages @@ -18,6 +19,7 @@ yum -y install less zstd # Install conda ############################################################################### echo 'Installing conda-forge' + curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh chmod +x /mambaforge.sh /mambaforge.sh -b -p /opt/conda From 10be16d8243df02e4ea9c83f77c32965c0b8ee3e Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 2 Feb 2023 17:37:46 +0000 Subject: [PATCH 38/40] correct cert path for image used --- aarch64_linux/aarch64_ci_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_linux/aarch64_ci_build.sh b/aarch64_linux/aarch64_ci_build.sh index 3c011de4d..774731e3d 100644 --- a/aarch64_linux/aarch64_ci_build.sh +++ b/aarch64_linux/aarch64_ci_build.sh @@ -7,7 +7,7 @@ set -eux -o pipefail CONDA_PYTHON_EXE=/opt/conda/bin/python CONDA_EXE=/opt/conda/bin/conda PATH=/opt/conda/bin:$PATH -REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt ############################################################################### # Install OS dependent packages From 9f29ef31f2a51fcf980136ffcd018d545386a6f7 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 2 Feb 2023 17:54:22 +0000 Subject: [PATCH 39/40] trying to get SSL working for conda --- aarch64_linux/aarch64_ci_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aarch64_linux/aarch64_ci_build.sh b/aarch64_linux/aarch64_ci_build.sh index 774731e3d..949ca3f31 100644 --- a/aarch64_linux/aarch64_ci_build.sh +++ b/aarch64_linux/aarch64_ci_build.sh @@ -19,7 +19,7 @@ yum -y install less zstd # Install conda ############################################################################### echo 'Installing conda-forge' - +mkdir -p /etc/pki/tls/certs && cp /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt /etc/pki/tls/certs/ca-bundle.crt curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh chmod +x /mambaforge.sh /mambaforge.sh -b -p /opt/conda From a1ed2c36b35c084d532f40b373c523bebb3c6e28 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Thu, 2 Feb 2023 18:02:38 +0000 Subject: [PATCH 40/40] handle non-latest python versions --- aarch64_linux/aarch64_ci_build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aarch64_linux/aarch64_ci_build.sh b/aarch64_linux/aarch64_ci_build.sh index 949ca3f31..352ddb128 100644 --- a/aarch64_linux/aarch64_ci_build.sh +++ b/aarch64_linux/aarch64_ci_build.sh @@ -7,7 +7,6 @@ set -eux -o pipefail CONDA_PYTHON_EXE=/opt/conda/bin/python CONDA_EXE=/opt/conda/bin/conda PATH=/opt/conda/bin:$PATH -REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt ############################################################################### # Install OS dependent packages @@ -17,13 +16,15 @@ yum -y install less zstd ############################################################################### # Install conda +# disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path" +# when using Python version, less than the conda latest ############################################################################### echo 'Installing conda-forge' -mkdir -p /etc/pki/tls/certs && cp /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt /etc/pki/tls/certs/ca-bundle.crt curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh chmod +x /mambaforge.sh /mambaforge.sh -b -p /opt/conda rm /mambaforge.sh +/opt/conda/bin/conda config --set ssl_verify False /opt/conda/bin/conda install -y -c conda-forge python=${DESIRED_PYTHON} numpy pyyaml setuptools patchelf python --version conda --version