Remove LLVM submodule and patch instead (#110)

* gh: bump code version

* llvm: remove submodule and just track our patch

* fix typo

* llvm: apply patch in the right directory

* llvm: call clean-libc before very-clean

* docker: get llvm sources from parent image

* docker: cleanup

* gha: use access token instead of password

Author: csegarragonz

.env

@@ -1,3 +1,3 @@
-SYSROOT_VERSION=0.2.2
-SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:0.2.2
+SYSROOT_VERSION=0.2.3
+SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:0.2.3
 COMPOSE_PROJECT_NAME=cpp-dev

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+          password: ${{ secrets.DOCKER_TOKEN }}
       - name: "Build and push cpp-sysroot container"
         id: docker_build
         uses: docker/build-push-action@v3

.github/workflows/tests.yml

@@ -19,7 +19,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/cpp-sysroot:0.2.2
+      image: faasm/cpp-sysroot:0.2.3
     steps:
       # --- Update code ---
       - name: "Checkout code"

.gitmodules

@@ -2,10 +2,6 @@
     path = third-party/wasi-libc
     url = https://github.com/faasm/wasi-libc
     branch = faasm
-[submodule "third-party/llvm-project"]
-    path = third-party/llvm-project
-    url = https://github.com/faasm/llvm-project.git
-    branch = faasm
 [submodule "third-party/libffi"]
     path = third-party/libffi
     url = https://github.com/faasm/libffi.git

LLVM.makefile

@@ -30,7 +30,7 @@ clean-libc:
 	rm -rf $(BUILD_DIR)/libc.BUILT $(WASI_LIBC_DIR)/build
 
 .PHONY: very-clean-libc
-very-clean-libc:
+very-clean-libc: clean-libc
 	# WARNING: this is going to remove _everything_ in FAASM_SYSROOT
 	cd $(WASI_LIBC_DIR) && SYSROOT=$(FAASM_SYSROOT) make clean
 

VERSION

@@ -1 +1 @@
-0.2.2
+0.2.3

docker/cpp-sysroot.dockerfile

@@ -1,13 +1,14 @@
 # llvm image is not re-built often, so the tag may be behind
-FROM faasm/llvm:0.2.2 as llvm
+FROM faasm/llvm:0.2.3 as llvm
 
 # faabric-base image is not re-built often, so tag may be behind
 FROM faasm/faabric-base:0.4.1
 SHELL ["/bin/bash", "-c"]
 ENV CPP_DOCKER="on"
 
-# Copy the toolchain in from the LLVM container
+# Copy the toolchain and LLVM sources from the LLVM container
 COPY --from=llvm /usr/local/faasm /usr/local/faasm
+COPY --from=llvm /opt/llvm-project /opt/llvm-project
 
 # Update APT dependencies
 RUN apt update && apt install -y autotools-dev
@@ -18,7 +19,6 @@ RUN mkdir -p /code \
         https://github.com/faasm/cpp \
         /code/cpp \
     && cd /code/cpp \
-    && git submodule update --init -f third-party/llvm-project \
     && git submodule update --init -f third-party/faabric \
     && git submodule update --init -f third-party/faasm-clapack \
     && git submodule update --init -f third-party/libffi \

docker/llvm.dockerfile

@@ -38,7 +38,6 @@ RUN mkdir -p /code \
         https://github.com/faasm/cpp \
         /code/cpp \
     && cd /code/cpp \
-    && git submodule update --init -f third-party/llvm-project \
     && git submodule update --init -f third-party/wasi-libc \
     && ./bin/create_venv.sh \
     && source venv/bin/activate \

faasmtools/env.py

@@ -14,7 +14,7 @@
 
 # LLVM variables
 LLVM_VERSION = "13.0.1"
-LLVM_DIR = join(THIRD_PARTY_DIR, "llvm-project")
+LLVM_DIR = join("/opt", "llvm-project")
 LLVM_MAKEFILE = join(PROJ_ROOT, "LLVM.makefile")
 WASI_LIBC_DIR = join(THIRD_PARTY_DIR, "wasi-libc")
 

patches/llvm-project/0001-Remove-__EMSCRIPTEN__-definitions-from-emscripten-ta.patch

@@ -0,0 +1,44 @@
+From 4d755c1f417a09cbe5156af4c04782a10cb047aa Mon Sep 17 00:00:00 2001
+From: Simon Shillaker <[email protected]>
+Date: Wed, 27 Apr 2022 16:55:11 +0000
+Subject: [PATCH] Remove __EMSCRIPTEN__ definitions from emscripten target
+
+---
+ clang/lib/Basic/Targets/OSTargets.h | 14 ++++++++++----
+ 1 file changed, 10 insertions(+), 4 deletions(-)
+
+diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
+index 12df95c19f35..5aaf0e7fc84c 100644
+--- a/clang/lib/Basic/Targets/OSTargets.h
++++ b/clang/lib/Basic/Targets/OSTargets.h
+@@ -291,7 +291,7 @@ protected:
+     Builder.defineMacro("__HAIKU__");
+     Builder.defineMacro("__ELF__");
+     DefineStd(Builder, "unix", Opts);
+-    if (this->HasFloat128) 
++    if (this->HasFloat128)
+       Builder.defineMacro("__FLOAT128__");
+   }
+ 
+@@ -946,9 +946,15 @@ class LLVM_LIBRARY_VISIBILITY EmscriptenTargetInfo
+   void getOSDefines(const LangOptions &Opts, const llvm::Triple &Triple,
+                     MacroBuilder &Builder) const final {
+     WebAssemblyOSTargetInfo<Target>::getOSDefines(Opts, Triple, Builder);
+-    Builder.defineMacro("__EMSCRIPTEN__");
+-    if (Opts.POSIXThreads)
+-      Builder.defineMacro("__EMSCRIPTEN_PTHREADS__");
++    // For Faasm, we need to use the Emscripten target to build shared 
++    // libraries, but we don't actually want any of the emscripten features.
++    // See htps://bugs.llvm.org/show_bug.cgi?id=42714
++    //
++    Builder.defineMacro("__wasi__");
++
++    //Builder.defineMacro("__EMSCRIPTEN__");
++    //if (Opts.POSIXThreads)
++    //  Builder.defineMacro("__EMSCRIPTEN_PTHREADS__");
+   }
+ 
+ public:
+-- 
+2.34.1
+

tasks/llvm.py

@@ -13,9 +13,10 @@
     WASI_LIBC_DIR,
 )
 from invoke import task
-from os import environ
-from os.path import join
+from os import environ, listdir
+from os.path import exists, join
 from subprocess import run
+from tasks.util.env import PATCHES_DIR
 
 LLVM_BUILD_DIR = join(LLVM_DIR, "build")
 LLVM_BUILD_ENV_VARS = {
@@ -29,6 +30,27 @@
 }
 
 
+def do_llvm_clone():
+    """
+    Clone the llvm-project repo and apply faasm-specific patches
+    """
+    print("WARNING: Cloning llvm-project, this can take a while...")
+    git_cmd = [
+        "git clone",
+        "-b llvmorg-{}".format(LLVM_VERSION),
+        "--depth 1",
+        "https://github.com/llvm/llvm-project",
+        LLVM_DIR,
+    ]
+    git_cmd = " ".join(git_cmd)
+    run(git_cmd, shell=True, check=True)
+
+    llvm_patches_dir = join(PATCHES_DIR, "llvm-project")
+    for patch in listdir(llvm_patches_dir):
+        git_cmd = "git apply {}".format(join(llvm_patches_dir, patch))
+        run(git_cmd, shell=True, check=True, cwd=LLVM_DIR)
+
+
 def do_llvm_build(target, clean_target=None):
     build_env = environ.copy()
     # These env. variables are required for LLVM's build configuration (and
@@ -61,6 +83,9 @@ def build(ctx, clean=False):
     """
     Build LLVM and all targets (including libc)
     """
+    if not exists(LLVM_DIR):
+        do_llvm_clone()
+
     do_llvm_build("", "clean-all" if clean else None)
 
 

tasks/util/env.py

@@ -1,3 +1,5 @@
-from os.path import dirname, realpath
+from os.path import dirname, join, realpath
 
 PROJ_ROOT = dirname(dirname(dirname(realpath(__file__))))
+
+PATCHES_DIR = join(PROJ_ROOT, "patches")