[BE] Error handling in build_aarch64_wheel

malfet · malfet · commit 85075c17c64c · 2023-03-23T16:53:38.000-07:00
I've noticed that build errors in `build_ArmComputeLibrary` would be
ignored as semicolon is used between the commands, instead of &amp;&amp;
Also, replace nightly version evaluation by relying on torch, to rely on
individual libraries
diff --git a/aarch64_linux/build_aarch64_wheel.py b/aarch64_linux/build_aarch64_wheel.py
@@ -128,7 +128,7 @@ def run_cmd(self, args: Union[str, List[str]]) -> None:
         assert self.container_id is not None
         docker_cmd = self._gen_ssh_prefix() + ['docker', 'exec', '-i', self.container_id, 'bash']
         p = subprocess.Popen(docker_cmd, stdin=subprocess.PIPE)
-        p.communicate(input=" ".join(["source .bashrc;"] + self._split_cmd(args)).encode("utf-8"))
+        p.communicate(input=" ".join(["source .bashrc && "] + self._split_cmd(args)).encode("utf-8"))
         rc = p.wait()
         if rc != 0:
             raise subprocess.CalledProcessError(rc, docker_cmd)
@@ -139,7 +139,7 @@ def check_output(self, args: Union[str, List[str]]) -> str:
         assert self.container_id is not None
         docker_cmd = self._gen_ssh_prefix() + ['docker', 'exec', '-i', self.container_id, 'bash']
         p = subprocess.Popen(docker_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        (out, err) = p.communicate(input=" ".join(["source .bashrc;"] + self._split_cmd(args)).encode("utf-8"))
+        (out, err) = p.communicate(input=" ".join(["source .bashrc && "] + self._split_cmd(args)).encode("utf-8"))
         rc = p.wait()
         if rc != 0:
             raise subprocess.CalledProcessError(rc, docker_cmd, output=out, stderr=err)
@@ -225,23 +225,16 @@ def build_OpenBLAS(host: RemoteHost, git_clone_flags: str = "") -> None:
     print('Building OpenBLAS')
     host.run_cmd(f"git clone https://github.com/xianyi/OpenBLAS -b v0.3.19 {git_clone_flags}")
     make_flags = "NUM_THREADS=64 USE_OPENMP=1 NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=ARMV8"
-    host.run_cmd(f"pushd OpenBLAS; make {make_flags} -j8; sudo make {make_flags} install; popd; rm -rf OpenBLAS")
+    host.run_cmd(f"pushd OpenBLAS && make {make_flags} -j8 && sudo make {make_flags} install && popd && rm -rf OpenBLAS")
 
 
 def build_ArmComputeLibrary(host: RemoteHost, git_clone_flags: str = "") -> None:
     print('Building Arm Compute Library')
-    host.run_cmd("mkdir $HOME/acl")
+    acl_install_dir="${HOME}/acl"
+    acl_build_flags="debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native"
+    host.run_cmd(f"mkdir {acl_install_dir}")
     host.run_cmd(f"git clone https://github.com/ARM-software/ComputeLibrary.git -b v22.11 {git_clone_flags}")
-    host.run_cmd("pushd ComputeLibrary; export acl_install_dir=$HOME/acl; scons Werror=1 -j8 debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native build_dir=$acl_install_dir/build; cp -r arm_compute $acl_install_dir; cp -r include $acl_install_dir; cp -r utils $acl_install_dir; cp -r support $acl_install_dir; popd")
-
-
-def build_FFTW(host: RemoteHost, git_clone_flags: str = "") -> None:
-    print("Building FFTW3")
-    host.run_cmd("sudo apt-get install -y ocaml ocamlbuild autoconf automake indent libtool fig2dev texinfo")
-    # TODO: fix a version to build
-    # TODO: consider adding flags --host=arm-linux-gnueabi --enable-single --enable-neon CC=arm-linux-gnueabi-gcc -march=armv7-a -mfloat-abi=softfp
-    host.run_cmd(f"git clone https://github.com/FFTW/fftw3 {git_clone_flags}")
-    host.run_cmd("pushd fftw3; sh bootstrap.sh; make -j8; sudo make install; popd")
+    host.run_cmd(f"cd ComputeLibrary && scons Werror=1 -j8 {acl_build_flags} build_dir={acl_install_dir}/build")
 
 
 def embed_libgomp(host: RemoteHost, use_conda, wheel_name) -> None:
@@ -261,7 +254,7 @@ def embed_libgomp(host: RemoteHost, use_conda, wheel_name) -> None:
 
 
 def checkout_repo(host: RemoteHost, *,
-                  branch: str = "master",
+                  branch: str = "main",
                   url: str,
                   git_clone_flags: str,
                   mapping: Dict[str, Tuple[str, str]]) -> Optional[str]:
@@ -272,12 +265,16 @@ def checkout_repo(host: RemoteHost, *,
         host.run_cmd(f"git clone {url} -b {tag} {git_clone_flags}")
         return mapping[prefix][0]
 
-    host.run_cmd(f"git clone {url} {git_clone_flags}")
+    # Map master to main
+    if branch == "master" and url.rsplit("/")[-1] in ['vision', 'text', 'audio', 'data']:
+        branch = "main"
+
+    host.run_cmd(f"git clone {url} -b {branch} {git_clone_flags}")
     return None
 
 
 def build_torchvision(host: RemoteHost, *,
-                      branch: str = "master",
+                      branch: str = "main",
                       use_conda: bool = True,
                       git_clone_flags: str,
                       run_smoke_tests: bool = True) -> str:
@@ -317,14 +314,14 @@ def build_torchvision(host: RemoteHost, *,
         if len(version) == 0:
             # In older revisions, version was embedded in setup.py
             version = host.check_output(["grep", "\"version = '\"", "vision/setup.py"]).strip().split("'")[1][:-2]
-        build_date = host.check_output("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "")
+        build_date = host.check_output("cd vision && git log --pretty=format:%s -1").strip().split()[0].replace("-", "")
         build_vars += f"BUILD_VERSION={version}.dev{build_date}"
     elif build_version is not None:
         build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
     if host.using_docker():
         build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
 
-    host.run_cmd(f"cd vision; {build_vars} python3 setup.py bdist_wheel")
+    host.run_cmd(f"cd vision && {build_vars} python3 setup.py bdist_wheel")
     vision_wheel_name = host.list_dir("vision/dist")[0]
     embed_libgomp(host, use_conda, os.path.join('vision', 'dist', vision_wheel_name))
 
@@ -357,14 +354,14 @@ def build_torchdata(host: RemoteHost, *,
     build_vars = ""
     if branch == 'nightly':
         version = host.check_output(["if [ -f data/version.txt ]; then cat data/version.txt; fi"]).strip()
-        build_date = host.check_output("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "")
+        build_date = host.check_output("cd data && git log --pretty=format:%s -1").strip().split()[0].replace("-", "")
         build_vars += f"BUILD_VERSION={version}.dev{build_date}"
     elif build_version is not None:
         build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
     if host.using_docker():
         build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
 
-    host.run_cmd(f"cd data; {build_vars} python3 setup.py bdist_wheel")
+    host.run_cmd(f"cd data && {build_vars} python3 setup.py bdist_wheel")
     wheel_name = host.list_dir("data/dist")[0]
     embed_libgomp(host, use_conda, os.path.join('data', 'dist', wheel_name))
 
@@ -400,14 +397,14 @@ def build_torchtext(host: RemoteHost, *,
     build_vars = ""
     if branch == 'nightly':
         version = host.check_output(["if [ -f text/version.txt ]; then cat text/version.txt; fi"]).strip()
-        build_date = host.check_output("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "")
+        build_date = host.check_output("cd text && git log --pretty=format:%s -1").strip().split()[0].replace("-", "")
         build_vars += f"BUILD_VERSION={version}.dev{build_date}"
     elif build_version is not None:
         build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
     if host.using_docker():
         build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
 
-    host.run_cmd(f"cd text; {build_vars} python3 setup.py bdist_wheel")
+    host.run_cmd(f"cd text && {build_vars} python3 setup.py bdist_wheel")
     wheel_name = host.list_dir("text/dist")[0]
     embed_libgomp(host, use_conda, os.path.join('text', 'dist', wheel_name))
 
@@ -443,14 +440,14 @@ def build_torchaudio(host: RemoteHost, *,
     build_vars = ""
     if branch == 'nightly':
         version = host.check_output(["grep", "\"version = '\"", "audio/setup.py"]).strip().split("'")[1][:-2]
-        build_date = host.check_output("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "")
+        build_date = host.check_output("cd audio && git log --pretty=format:%s -1").strip().split()[0].replace("-", "")
         build_vars += f"BUILD_VERSION={version}.dev{build_date}"
     elif build_version is not None:
         build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
     if host.using_docker():
         build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
 
-    host.run_cmd(f"cd audio; {build_vars} python3 setup.py bdist_wheel")
+    host.run_cmd(f"cd audio && {build_vars} python3 setup.py bdist_wheel")
     wheel_name = host.list_dir("audio/dist")[0]
     embed_libgomp(host, use_conda, os.path.join('audio', 'dist', wheel_name))
 
@@ -523,7 +520,6 @@ def start_build(host: RemoteHost, *,
                      use_conda=use_conda,
                      python_version=python_version)
     build_OpenBLAS(host, git_clone_flags)
-    # build_FFTW(host, git_clone_flags)
 
     if host.using_docker():
         print("Move libgfortant.a into a standard location")
@@ -546,7 +542,7 @@ def start_build(host: RemoteHost, *,
     # Breakpad build fails on aarch64
     build_vars = "USE_BREAKPAD=0 "
     if branch == 'nightly':
-        build_date = host.check_output("cd pytorch ; git log --pretty=format:%s -1").strip().split()[0].replace("-", "")
+        build_date = host.check_output("cd pytorch && git log --pretty=format:%s -1").strip().split()[0].replace("-", "")
         version = host.check_output("cat pytorch/version.txt").strip()[:-2]
         build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={version}.dev{build_date} PYTORCH_BUILD_NUMBER=1"
     if branch.startswith("v1.") or branch.startswith("v2."):
@@ -557,19 +553,19 @@ def start_build(host: RemoteHost, *,
         build_ArmComputeLibrary(host, git_clone_flags)
         print("build pytorch with mkldnn+acl backend")
         build_vars += " USE_MKLDNN=ON USE_MKLDNN_ACL=ON"
-        host.run_cmd(f"cd pytorch ; export ACL_ROOT_DIR=$HOME/ComputeLibrary:$HOME/acl; {build_vars} python3 setup.py bdist_wheel{build_opts}")
+        host.run_cmd(f"cd pytorch && export ACL_ROOT_DIR=$HOME/ComputeLibrary:$HOME/acl && {build_vars} python3 setup.py bdist_wheel{build_opts}")
         print('Repair the wheel')
         pytorch_wheel_name = host.list_dir("pytorch/dist")[0]
-        host.run_cmd(f"export LD_LIBRARY_PATH=$HOME/acl/build:$HOME/pytorch/build/lib; auditwheel repair $HOME/pytorch/dist/{pytorch_wheel_name}")
+        host.run_cmd(f"export LD_LIBRARY_PATH=$HOME/acl/build:$HOME/pytorch/build/lib && auditwheel repair $HOME/pytorch/dist/{pytorch_wheel_name}")
         print('replace the original wheel with the repaired one')
         pytorch_repaired_wheel_name = host.list_dir("wheelhouse")[0]
         host.run_cmd(f"cp $HOME/wheelhouse/{pytorch_repaired_wheel_name} $HOME/pytorch/dist/{pytorch_wheel_name}")
     else:
         print("build pytorch without mkldnn backend")
-        host.run_cmd(f"cd pytorch ; {build_vars} python3 setup.py bdist_wheel{build_opts}")
+        host.run_cmd(f"cd pytorch && {build_vars} python3 setup.py bdist_wheel{build_opts}")
 
     print("Deleting build folder")
-    host.run_cmd("cd pytorch; rm -rf build")
+    host.run_cmd("cd pytorch && rm -rf build")
     pytorch_wheel_name = host.list_dir("pytorch/dist")[0]
     embed_libgomp(host, use_conda, os.path.join('pytorch', 'dist', pytorch_wheel_name))
     print('Copying the wheel')
