Latest wasi-libc (#109)

* gh: bump code version

* llvm: bump wasi-libc to latest commit

* wasi-libc: rebase to latest commit cross-compiling all functions

* makefile: update toolchain file to use ldflags w/out quotes in clapack

* cmake: add linker flags in wasi toolchain

* faasmtools: set the compilationflags and linking flags required to use wasi with emulation

* func: set wasi emulated libs in toolchain

* zlib: only build static library

* clapack and zlib: fix compilation with new wasi-libc

* docker: tag llvm image with cpp tag as well

* libc: changes to cross-compile cpython

* wasi-libc: more changes

* wasi-libc: more updates

* libc: more changes

* wasi-libc: emulated mman

* libfake: fix task

* func: fix calloc function by passing the right flags to mmap (as required by wasi-libc's calloc implementation

* dynlink: temporarily disable one failing check

* func: remove PROT_EXEC flag from mmap as it is not supported in wasi-libc

* clang-format

* wasi-libc: add --purge flag to clean the sysroot and apply patches

* wasi-libc: fix patch

* docker: make sure to re-install llvm headers after purge

* py: make linter happy

* wasi-libc: more cleanup

* wasi-libc: remove patch-oritented style

* cleanup

* wasi-libc: prepare for merge

* nits: self-review

* dynlink: undo patch

Author: csegarragonz

.env

@@ -1,3 +1,3 @@
-SYSROOT_VERSION=0.2.1
-SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:0.2.1
+SYSROOT_VERSION=0.2.2
+SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:0.2.2
 COMPOSE_PROJECT_NAME=cpp-dev

.github/workflows/tests.yml

@@ -19,7 +19,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/cpp-sysroot:0.2.1
+      image: faasm/cpp-sysroot:0.2.2
     steps:
       # --- Update code ---
       - name: "Checkout code"

LLVM.makefile

@@ -29,6 +29,11 @@ default: build
 clean-libc:
 	rm -rf $(BUILD_DIR)/libc.BUILT $(WASI_LIBC_DIR)/build
 
+.PHONY: very-clean-libc
+very-clean-libc:
+	# WARNING: this is going to remove _everything_ in FAASM_SYSROOT
+	cd $(WASI_LIBC_DIR) && SYSROOT=$(FAASM_SYSROOT) make clean
+
 .PHONY: clean-libs
 clean-libs: clean-libc
 	rm -rf $(BUILD_DIR)/compiler-rt $(BUILD_DIR)/compiler-rt.BUILT
@@ -72,9 +77,9 @@ $(BUILD_DIR)/libc.BUILT: $(BUILD_DIR)/llvm.BUILT
 	mkdir -p $(WASI_LIBC_DIR)/build
 	cd $(WASI_LIBC_DIR); $(MAKE) \
 		THREAD_MODEL=faasm \
-		WASM_CC=$(PREFIX)/bin/clang \
-		WASM_AR=$(PREFIX)/bin/llvm-ar \
-		WASM_NM=$(PREFIX)/bin/llvm-nm \
+		CC=$(PREFIX)/bin/clang \
+		AR=$(PREFIX)/bin/llvm-ar \
+		NM=$(PREFIX)/bin/llvm-nm \
 		SYSROOT=$(FAASM_SYSROOT)
 	touch $(BUILD_DIR)/libc.BUILT
 

Makefile.envs

@@ -16,6 +16,7 @@ export WASM_RANLIB=${FAASM_WASM_RANLIB}
 export WASM_LD=${FAASM_WASM_CC}
 export WASM_LDSHARED=${FAASM_WASM_CC}
 export WASM_LDFLAGS="${FAASM_WASM_STATIC_LINKER_FLAGS}"
+export WASM_LDFLAGS_NO_QUOTES=${FAASM_WASM_STATIC_LINKER_FLAGS}
 export WASM_EXE_LDFLAGS="${FAASM_WASM_EXE_LINKER_FLAGS}"
 export WASM_EXE_LDFLAGS_NO_QUOTES=${FAASM_WASM_EXE_LINKER_FLAGS}
 

VERSION

@@ -1 +1 @@
-0.2.1
+0.2.2

WasiToolchain.cmake

@@ -58,6 +58,7 @@ endif()
 set(CMAKE_SYSROOT ${FAASM_SYSROOT} CACHE STRING "faasm build")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FAASM_COMPILER_FLAGS}" CACHE STRING "faasm build")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FAASM_COMPILER_FLAGS}" CACHE STRING "faasm build")
+set(CMAKE_LINKER_FLAGS $ENV{FAASM_WASM_LINKER_FLAGS} CACHE STRING "faasm build")
 set(CMAKE_SHARED_LINKER_FLAGS $ENV{FAASM_WASM_SHARED_LINKER_FLAGS} CACHE STRING "faasm build")
 set(CMAKE_EXE_LINKER_FLAGS $ENV{FAASM_WASM_EXE_LINKER_FLAGS} CACHE STRING "faasm build")
 

docker/cpp-sysroot.dockerfile

@@ -1,4 +1,5 @@
-FROM faasm/llvm:13.0.1 as llvm
+# llvm image is not re-built often, so the tag may be behind
+FROM faasm/llvm:0.2.2 as llvm
 
 # faabric-base image is not re-built often, so tag may be behind
 FROM faasm/faabric-base:0.4.1
@@ -44,8 +45,9 @@ RUN cd /code/cpp \
         libfaasmpi --native --shared \
     # Install toolchain files
     && inv install \
-    # First build libc
-    && inv llvm.libc \
+    # Build wasi-libc and reset the sysroot. The second call to LLVM just
+    # installs some headers that are purged
+    && inv llvm.libc --purge llvm \
     # Build Faasm WASM libraries
     && inv \
         libemscripten \

faasmtools/build.py

@@ -81,6 +81,15 @@
     "-D__faasm",
 ]
 
+# Wasi-libc specific flags that are needed to include certain emulation headers
+WASM_WASI_LIBC_CFLAGS = [
+    "-D_WASI_EMULATED_GETPID",
+    "-D_WASI_EMULATED_PROCESS_CLOCKS",
+    "-D_WASI_EMULATED_SIGNAL",
+]
+
+WASM_CFLAGS += WASM_WASI_LIBC_CFLAGS
+
 WASM_CXXFLAGS = WASM_CFLAGS
 
 # Flags for shared libraries
@@ -112,6 +121,17 @@
     "-Xlinker --no-check-features",
 ]
 
+# Wasi-libc specific libraries we need to link with to enable certain emulated
+# functionalities
+WASM_WASI_LIBC_LDFLAGS = [
+    "-lc-printscan-long-double",
+    "-lwasi-emulated-getpid",
+    "-lwasi-emulated-process-clocks",
+    "-lwasi-emulated-signal",
+]
+
+WASM_LDFLAGS += WASM_WASI_LIBC_LDFLAGS
+
 # Flags for executables
 WASM_EXE_LDFLAGS = [
     "-Xlinker --stack-first",
@@ -125,6 +145,8 @@
     "-Wl,--initial-memory={}".format(FAASM_WASM_INITIAL_MEMORY_SIZE),
 ]
 
+WASM_EXE_LDFLAGS += WASM_WASI_LIBC_LDFLAGS
+
 # These are the compiler and linker flags required for functions that will also
 # do dynamic linking.  We need to export all symbols to make them available to
 # the dynamically loaded modules
@@ -133,6 +155,8 @@
     "-Xlinker --no-gc-sections",
 ]
 
+WASM_EXE_LDFLAGS_SHARED += WASM_WASI_LIBC_LDFLAGS
+
 # Flags for shared libraries
 # See notes in README about WebAssembly and shared libraries
 WASM_LDFLAGS_SHARED = [

func/CMakeLists.txt

@@ -25,7 +25,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Wasm")
     # Wasm build
     # ----------------------
 
-    set(FAASM_FUNC_LIBS faasm c-printscan-long-double)
+    set(FAASM_FUNC_LIBS faasm)
 
     function(faasm_func_c func_name file_name)
         add_executable(${func_name} ${file_name})

func/demo/calloc.cpp

@@ -26,8 +26,11 @@ int main(int argc, char* argv[])
         strcpy(callocPtr, s.c_str());
 
         // Add in a call to mmap to deliberately fragment things
-        char* mmapPtr =
-          (char*)mmap(nullptr, mmapLen, PROT_WRITE, MAP_ANONYMOUS, -1, 0);
+        char* mmapPtr = (char*)mmap(
+          nullptr, mmapLen, PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
+        if (mmapPtr == MAP_FAILED) {
+            printf("ERROR - mmap failed: %s\n", strerror(errno));
+        }
         strcpy(mmapPtr, "mmapSpace");
 
         callocPtrs[i] = callocPtr;

func/demo/getcwd.cpp

@@ -4,6 +4,9 @@
 #include <string.h>
 #include <unistd.h>
 
+// 09/02/2022 - wasi-libc implements getcwd now, so we don't need to implement
+// it ourselves in Faasm. Thus, this test is now redundant as it is essentially
+// testing wasi-libc's functionality. We keep it for backwards-compatibility.
 int main(int argc, char* argv[])
 {
     char buf[1024];
@@ -14,7 +17,7 @@ int main(int argc, char* argv[])
         return 1;
     }
 
-    const char* expected = "/bin";
+    const char* expected = "/";
     if (strcmp(res, expected) != 0) {
         printf("Unexpected working dir (expected %s, got %s)\n", expected, res);
         return 1;

func/demo/mmap.cpp

@@ -22,7 +22,7 @@ int main(int argc, char* argv[])
         // Map some memory
         void* p = mmap(nullptr,
                        memLen,
-                       PROT_READ | PROT_WRITE | PROT_EXEC,
+                       PROT_READ | PROT_WRITE,
                        MAP_ANON | MAP_PRIVATE,
                        -1,
                        0);

func/demo/mmap_big.cpp

@@ -18,7 +18,7 @@ int main(int argc, char* argv[])
     for (int i = 0; i < nChunks; i++) {
         void* p = mmap(nullptr,
                        chunkLen,
-                       PROT_READ | PROT_WRITE | PROT_EXEC,
+                       PROT_READ | PROT_WRITE,
                        MAP_ANON | MAP_PRIVATE,
                        -1,
                        0);

tasks/docker.py

@@ -10,7 +10,6 @@
 from faasmtools.env import (
     get_version,
     PROJ_ROOT,
-    LLVM_VERSION,
 )
 
 LLVM_IMAGE_NAME = "faasm/llvm"
@@ -25,7 +24,8 @@ def get_sysroot_tag():
 
 
 def get_llvm_tag():
-    return "{}:{}".format(LLVM_IMAGE_NAME, LLVM_VERSION)
+    version = get_version()
+    return "{}:{}".format(LLVM_IMAGE_NAME, version)
 
 
 @task(iterable=["c"])

tasks/lib.py

@@ -37,6 +37,9 @@ def build_faasm_lib(subdir, clean=False, native=False, shared=False):
         extras = [
             "-DCMAKE_C_COMPILER=/usr/bin/clang-{}".format(llvm_major),
             "-DCMAKE_CXX_COMPILER=/usr/bin/clang++-{}".format(llvm_major),
+            "-DCMAKE_AR=/usr/bin/llvm-ar-{}".format(llvm_major),
+            "-DCMAKE_NM=/usr/bin/llvm-nm-{}".format(llvm_major),
+            "-DCMAKE_RANLIB=/usr/bin/llvm-ranlib-{}".format(llvm_major),
         ]
     else:
         extras = [

tasks/libfake.py

@@ -43,8 +43,8 @@ def fake(ctx, clean=False):
         check=True,
         env=work_env,
     )
-    run("ninja", shell=True, cwd=build_dir, check=True)
-    run("ninja install", shell=True, cwd=build_dir, check=True)
+    run("ninja", shell=True, cwd=build_dir, check=True, env=work_env)
+    run("ninja install", shell=True, cwd=build_dir, check=True, env=work_env)
 
     # Copy shared object into place
     sysroot_files = join(WASM_SYSROOT, "lib", "wasm32-wasi", "libfake*.so")

tasks/llvm.py

@@ -2,6 +2,7 @@
     FAASM_BUILD_ENV_DICT,
     FAASM_LOCAL_DIR,
     WASM_LIB_INSTALL,
+    WASM_SYSROOT,
 )
 from faasmtools.env import (
     LLVM_DIR,
@@ -24,6 +25,7 @@
     "FAASM_LLVM_VERSION": LLVM_VERSION,
     "FAASM_LOCAL_DIR_ENV": FAASM_LOCAL_DIR,
     "FAASM_WASI_LIBC_DIR": WASI_LIBC_DIR,
+    "FAASM_SYSROOT": WASM_SYSROOT,
 }
 
 
@@ -73,11 +75,22 @@ def libs(ctx, clean=False):
 
 
 @task()
-def libc(ctx, clean=False):
+def libc(ctx, clean=False, purge=False):
     """
     Builds the wasi libc fork in this directory
+
+    Clean removes the built targets but does not wipe the sysroot. Purge wipes
+    the sysroot altogether.
     """
-    do_llvm_build("libc", "clean-libc" if clean else None)
+    clean_args = None
+    if purge:
+        if clean:
+            print("WARN: Set --clean and --purge flag. Purge preceeds.")
+        clean_args = "very-clean-libc"
+    elif clean:
+        clean_args = "clean-libc"
+
+    do_llvm_build("libc", clean_args)
 
     # Copy the import files into place
     copy_cmd = "cp -r sysroot_extras/* {}".format(WASM_LIB_INSTALL)

tasks/zlib.py

@@ -46,7 +46,8 @@ def zlib(ctx, clean=False):
     configure_cmd = " ".join(configure_cmd)
     run(configure_cmd, shell=True, cwd=zlib_dir, check=True)
 
-    run("make -j", shell=True, cwd=zlib_dir, check=True)
+    # Only build the static library, not the examples nor tests
+    run("make -j libz.a", shell=True, cwd=zlib_dir, check=True)
     run("make install", shell=True, cwd=zlib_dir, check=True)
 
     # Move the library from /usr/local/faasm/llvm-sysroot/lib into wasm32-wasi