Skip to content

Upgrade source-build to BB2 for Julia 1.3 #450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Dec 12, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,7 @@
.DS_Store
docs/build
docs/site
deps/usr
deps/deps.jl
deps/build.log
Manifest.toml
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ Note that Windows support is still in its early stage, so feel free to submit fe
### Build from source
#### Build requirements

In addition to the [system requirements](https://github.com/JuliaLang/julia#required-build-tools-and-external-libraries) to build julia itself(note you need to build julia with `USE_BINARYBUILDER=0`), the following are required:
In addition to the [system requirements](https://github.com/JuliaLang/julia#required-build-tools-and-external-libraries) to build julia itself, the following are required:

- *Debian/Ubuntu*: `libedit-dev`, `libncurses5-dev`
- *RedHat/CentOS*: `libedit-devel`
Expand Down
252 changes: 95 additions & 157 deletions deps/BuildBootstrap.Makefile
Original file line number Diff line number Diff line change
@@ -1,145 +1,99 @@
JULIA_SRC := $(subst \,/,$(BASE_JULIA_SRC))
JULIA_BIN := $(subst \,/,$(BASE_JULIA_BIN))
# Cxx source build
# download sources
LLVM_VER := 6.0.1
LLVM_URL_PREFIX := http://releases.llvm.org/$(LLVM_VER)
LLVM_TAR := llvm-$(LLVM_VER).src.tar.xz
CLANG_TAR := cfe-$(LLVM_VER).src.tar.xz
COMPILER_RT_TAR := compiler-rt-$(LLVM_VER).src.tar.xz
LIBCXX_TAR := libcxx-$(LLVM_VER).src.tar.xz
LIBCXXABI_TAR := libcxxabi-$(LLVM_VER).src.tar.xz
POLLY_TAR := polly-$(LLVM_VER).src.tar.xz
LIBUNWIND_TAR := libunwind-$(LLVM_VER).src.tar.xz
LLD_TAR := lld-$(LLVM_VER).src.tar.xz

TIMEOUT := 180
CURL := curl -fkL --connect-timeout $(TIMEOUT) -y $(TIMEOUT)

usr/download:
@[ -d usr ] || mkdir usr
mkdir -p $@

ifeq ($(LLVM_VER),)
BUILDROOT=$(JULIA_BIN)/../..
include $(JULIA_SRC)/deps/Versions.make
ifeq (exists, $(shell [ -e $(BUILDROOT)/Make.user ] && echo exists ))
include $(BUILDROOT)/Make.user
endif
endif
include Make.inc
LLVM_TARS := $(LLVM_TAR) $(CLANG_TAR) $(COMPILER_RT_TAR) $(LIBCXX_TAR) $(LIBCXXABI_TAR) $(POLLY_TAR) $(LIBUNWIND_TAR) $(LLD_TAR)

LLVM_VER_MAJ:=$(word 1, $(subst ., ,$(LLVM_VER)))
LLVM_VER_MIN:=$(word 2, $(subst ., ,$(LLVM_VER)))
# define a "short" LLVM version for easy comparisons
ifeq ($(LLVM_VER),svn)
LLVM_VER_SHORT:=svn
else
LLVM_VER_SHORT:=$(LLVM_VER_MAJ).$(LLVM_VER_MIN)
endif
LLVM_VER_PATCH:=$(word 3, $(subst ., ,$(LLVM_VER)))
ifeq ($(LLVM_VER_PATCH),)
LLVM_VER_PATCH := 0
endif
llvm_tars: usr/download
@for tar in $(LLVM_TARS); do \
if [ -e $</$$tar ]; then \
echo "$$tar already exists, please delete it manually if you'd like to re-download it."; \
else \
$(CURL) -o $</$$tar $(LLVM_URL_PREFIX)/$$tar ; \
fi \
done

ifeq ($(LLVM_VER_SHORT),$(filter $(LLVM_VER_SHORT),3.3 3.4 3.5 3.6 3.7 3.8))
LLVM_USE_CMAKE := 0
else
LLVM_USE_CMAKE := 1
endif
# unzip
usr/src: usr/download
mkdir -p $@

all: usr/lib/libcxxffi.$(SHLIB_EXT) usr/lib/libcxxffi-debug.$(SHLIB_EXT) build/clang_constants.jl
llvm-$(LLVM_VER): usr/src llvm_tars
@[ -d $</$@ ] || mkdir -p $</$@
tar -C $</$@ --strip-components=1 -xf usr/download/$(LLVM_TAR)
@[ -d $</$@/tools/clang ] || mkdir -p $</$@/tools/clang
tar -C $</$@/tools/clang --strip-components=1 -xf usr/download/$(CLANG_TAR)
@[ -d $</$@/tools/polly ] || mkdir -p $</$@/tools/polly
tar -C $</$@/tools/polly --strip-components=1 -xf usr/download/$(POLLY_TAR)
@[ -d $</$@/tools/lld ] || mkdir -p $</$@/tools/lld
tar -C $</$@/tools/lld --strip-components=1 -xf usr/download/$(LLD_TAR)
@[ -d $</$@/projects/compiler-rt ] || mkdir -p $</$@/projects/compiler-rt
tar -C $</$@/projects/compiler-rt --strip-components=1 -xf usr/download/$(COMPILER_RT_TAR)
@[ -d $</$@/projects/libcxx ] || mkdir -p $</$@/projects/libcxx
tar -C $</$@/projects/libcxx --strip-components=1 -xf usr/download/$(LIBCXX_TAR)
@[ -d $</$@/projects/libcxxabi ] || mkdir -p $</$@/projects/libcxxabi
tar -C $</$@/projects/libcxxabi --strip-components=1 -xf usr/download/$(LIBCXXABI_TAR)
@[ -d $</$@/projects/libunwind ] || mkdir -p $</$@/projects/libunwind
tar -C $</$@/projects/libunwind --strip-components=1 -xf usr/download/$(LIBUNWIND_TAR)

# apply patches
llvm-patched: llvm-$(LLVM_VER)
cd usr/src/$< && \
for p in ../../../llvm_patches/*.patch; do \
echo "Applying patch $$p"; \
patch -p1 < $$p ; \
done

# build libcxxffi
include Make.inc

JULIA_SRC := $(subst \,/,$(BASE_JULIA_SRC))
JULIA_BIN := $(subst \,/,$(BASE_JULIA_BIN))

ifeq ($(OLD_CXX_ABI),1)
CXX_ABI_SETTING=-D_GLIBCXX_USE_CXX11_ABI=0
else
CXX_ABI_SETTING=-D_GLIBCXX_USE_CXX11_ABI=1
endif

CXXJL_CPPFLAGS = -I$(JULIA_SRC)/src/support -I$(BASE_JULIA_BIN)/../include
JULIA_SOURCE_INCLUDE_DIRS := $(JULIA_SRC)/src/support
JULIA_INCLUDE_DIRS := $(JULIA_BIN)/../include
CLANG_SOURCE_INCLUDE_DIRS := usr/src/llvm-$(LLVM_VER)/tools/clang/include
CLANG_SOURCE_INCLUDE_DIRS += usr/src/llvm-$(LLVM_VER)/tools/clang/lib
CLANG_INCLUDE_DIRS := $(JULIA_BIN)/../include $(JULIA_BIN)/../include/clang
INCLUDE_DIRS := $(JULIA_SOURCE_INCLUDE_DIRS) $(JULIA_INCLUDE_DIRS) $(CLANG_SOURCE_INCLUDE_DIRS) $(CLANG_INCLUDE_DIRS)
CXXJL_CPPFLAGS = $(addprefix -I, $(INCLUDE_DIRS))

ifeq ($(JULIA_BINARY_BUILD),1)
LIBDIR := $(BASE_JULIA_BIN)/../lib/julia
else
LIBDIR := $(BASE_JULIA_BIN)/../lib
endif

CLANG_LIBS = clangFrontendTool clangBasic clangLex clangDriver clangFrontend clangParse \
clangAST clangASTMatchers clangSema clangAnalysis clangEdit \
clangRewriteFrontend clangRewrite clangSerialization clangStaticAnalyzerCheckers \
clangStaticAnalyzerCore clangStaticAnalyzerFrontend clangTooling clangToolingCore \
clangCodeGen clangARCMigrate clangFormat

ifneq ($(PREBUILT_CI_BINARIES),1)
# If clang is not built by base julia, build it ourselves
ifeq ($(BUILD_LLVM_CLANG),)
ifeq ($(LLVM_VER),svn)
$(error For julia built against llvm-svn, please built clang in tree)
endif

LLVM_TAR_EXT:=$(LLVM_VER).src.tar.xz
LLVM_CLANG_TAR:=src/cfe-$(LLVM_TAR_EXT)
LLVM_SRC_TAR:=src/llvm-$(LLVM_TAR_EXT)
LLVM_COMPILER_RT_TAR:=src/compiler-rt-$(LLVM_TAR_EXT)
LLVM_SRC_URL := http://releases.llvm.org/$(LLVM_VER)

src:
mkdir $@

# Also build a new copy of LLVM, so we get headers, tools, etc.
ifeq ($(JULIA_BINARY_BUILD),1)
LLVM_SRC_DIR := src/llvm-$(LLVM_VER)
include llvm-patches/apply-llvm-patches.mk
$(LLVM_SRC_TAR): | src
curl -Lo $@ $(LLVM_SRC_URL)/$(notdir $@)
src/llvm-$(LLVM_VER): $(LLVM_SRC_TAR)
mkdir -p $@
tar -C $@ --strip-components=1 -xf $<
build/llvm-$(LLVM_VER)/Makefile: src/llvm-$(LLVM_VER) $(LLVM_PATCH_LIST)
mkdir -p $(dir $@)
cd $(dir $@) && \
cmake -G "Unix Makefiles" -DLLVM_TARGETS_TO_BUILD="X86" \
-DLLVM_BUILD_LLVM_DYLIB=ON -DCMAKE_BUILD_TYPE=Release \
-DLLVM_LINK_LLVM_DYLIB=ON -DLLVM_ENABLE_THREADS=OFF \
-DCMAKE_CXX_COMPILER_ARG1="$(CXX_ABI_SETTING)" \
../../src/llvm-$(LLVM_VER)
build/llvm-$(LLVM_VER)/bin/llvm-config: build/llvm-$(LLVM_VER)/Makefile
cd build/llvm-$(LLVM_VER) && $(MAKE)
LLVM_HEADER_DIRS = src/llvm-$(LLVM_VER)/include build/llvm-$(LLVM_VER)/include
CLANG_CMAKE_DEP = build/llvm-$(LLVM_VER)/bin/llvm-config
LLVM_CONFIG = ../llvm-$(LLVM_VER)/bin/llvm-config
else
CLANG_CMAKE_OPTS += -DLLVM_TABLEGEN_EXE=$(BASE_JULIA_BIN)/../tools/llvm-tblgen
endif

JULIA_LDFLAGS = -L$(BASE_JULIA_BIN)/../lib -L$(BASE_JULIA_BIN)/../lib/julia
CLANG_LIBS := clangFrontendTool clangBasic clangLex clangDriver clangFrontend clangParse
CLANG_LIBS += clangAST clangASTMatchers clangSema clangAnalysis clangEdit
CLANG_LIBS += clangRewriteFrontend clangRewrite clangSerialization clangStaticAnalyzerCheckers
CLANG_LIBS += clangStaticAnalyzerCore clangStaticAnalyzerFrontend clangTooling clangToolingCore
CLANG_LIBS += clangCodeGen clangARCMigrate clangFormat
LINKED_LIBS = $(addprefix -l,$(CLANG_LIBS))

$(LLVM_CLANG_TAR): | src
curl -Lo $@ $(LLVM_SRC_URL)/$(notdir $@)
$(LLVM_COMPILER_RT_TAR): | src
$(JLDOWNLOAD) $@ $(LLVM_SRC_URL)/$(notdir $@)
src/clang-$(LLVM_VER): $(LLVM_CLANG_TAR)
mkdir -p $@
tar -C $@ --strip-components=1 -xf $<
build/clang-$(LLVM_VER)/Makefile: src/clang-$(LLVM_VER) $(CLANG_CMAKE_DEP)
mkdir -p $(dir $@)
cd $(dir $@) && \
cmake -G "Unix Makefiles" \
-DLLVM_BUILD_LLVM_DYLIB=ON -DCMAKE_BUILD_TYPE=Release \
-DLLVM_LINK_LLVM_DYLIB=ON -DLLVM_ENABLE_THREADS=OFF \
-DCMAKE_CXX_COMPILER_ARG1="$(CXX_ABI_SETTING)" \
-DLLVM_CONFIG=$(LLVM_CONFIG) $(CLANG_CMAKE_OPTS) ../../src/clang-$(LLVM_VER)
build/clang-$(LLVM_VER)/lib/libclangCodeGen.a: build/clang-$(LLVM_VER)/Makefile
cd build/clang-$(LLVM_VER) && $(MAKE)
LIB_DEPENDENCY += build/clang-$(LLVM_VER)/lib/libclangCodeGen.a
JULIA_LDFLAGS += -Lbuild/clang-$(LLVM_VER)/lib
CXXJL_CPPFLAGS += -Isrc/clang-$(LLVM_VER)/lib -Ibuild/clang-$(LLVM_VER)/include \
-Isrc/clang-$(LLVM_VER)/include
else # BUILD_LLVM_CLANG
JULIA_LDFLAGS = -L$(BASE_JULIA_BIN)/../lib -L$(BASE_JULIA_BIN)/../lib/julia
CXXJL_CPPFLAGS += -I$(JULIA_SRC)/deps/srccache/llvm-$(LLVM_VER)/tools/clang/lib \
-I$(JULIA_SRC)/deps/llvm-$(LLVM_VER)/tools/clang/lib
endif
else
LLVM_SRC_DIR := src/llvm-$(LLVM_VER)
CLANG_CMAKE_DEP = build/llvm-$(LLVM_VER)/bin/llvm-config
LLVM_CONFIG = ../llvm-$(LLVM_VER)/bin/llvm-config
JULIA_LDFLAGS += -Lbuild/clang-$(LLVM_VER)/lib
LLVM_HEADER_DIRS = src/llvm-$(LLVM_VER)/include build/llvm-$(LLVM_VER)/include
CXXJL_CPPFLAGS += -Isrc/clang-$(LLVM_VER)/lib -Ibuild/clang-$(LLVM_VER)/include \
-Isrc/clang-$(LLVM_VER)/include
JULIA_LDFLAGS += -L$(BASE_JULIA_BIN)/../lib -L$(BASE_JULIA_BIN)/../lib/julia
endif # PREBUILD_CI_BINARIES

CXX_LLVM_VER := $(LLVM_VER)
ifeq ($(CXX_LLVM_VER),svn)
CXX_LLVM_VER := $(shell $(BASE_JULIA_BIN)/../tools/llvm-config --version)
LIB_DIRS := $(JULIA_BIN)/../lib
LIB_DIRS += $(JULIA_BIN)/../lib/julia
ifneq ($(USEMSVC), 1)
LIB_DIRS += $(JULIA_BIN)
endif
JULIA_LDFLAGS = $(addprefix -L,$(LIB_DIRS))
# JULIA_LDFLAGS += -Lbuild/clang-$(LLVM_VER)/lib

ifneq ($(LLVM_HEADER_DIRS),)
CXXJL_CPPFLAGS += $(addprefix -I,$(LLVM_HEADER_DIRS))
endif

FLAGS = -std=c++11 $(CPPFLAGS) $(CFLAGS) $(CXXJL_CPPFLAGS)

Expand All @@ -149,58 +103,42 @@ else
CPP_STDOUT := $(CPP) -E
endif

ifeq ($(LLVM_USE_CMAKE),1)
LLVM_LIB_NAME := LLVM
else ifeq ($(LLVM_VER),svn)
LLVM_LIB_NAME := LLVM
else
LLVM_LIB_NAME := LLVM-$(CXX_LLVM_VER)
endif
LDFLAGS += -l$(LLVM_LIB_NAME)
LIB_DEPENDENCY += $(JULIA_BIN)/../lib/lib$(LLVM_LIB_NAME).$(SHLIB_EXT)
LLVM_EXTRA_CPPFLAGS = -DLLVM_NDEBUG
JULIA_LIB := $(JULIA_BIN)/../lib/libjulia.$(SHLIB_EXT)
JULIA_DEBUG_LIB := $(JULIA_BIN)/../lib/libjulia-debug.$(SHLIB_EXT)

LIB_DEPENDENCY += $(LIBDIR)/lib$(LLVM_LIB_NAME).$(SHLIB_EXT)
all: usr/lib/libcxxffi.$(SHLIB_EXT) usr/lib/libcxxffi-debug.$(SHLIB_EXT) usr/clang_constants.jl

usr/lib:
@mkdir -p $(CURDIR)/usr/lib/

build:
@mkdir -p $(CURDIR)/build

LLVM_EXTRA_CPPFLAGS =
ifneq ($(LLVM_ASSERTIONS),1)
LLVM_EXTRA_CPPFLAGS += -DLLVM_NDEBUG
endif
usr/lib: usr/src
mkdir $@

build/bootstrap.o: ../src/bootstrap.cpp BuildBootstrap.Makefile $(LIB_DEPENDENCY) | build
usr/lib/bootstrap.o: ../src/bootstrap.cpp BuildBootstrap.Makefile $(LIB_DEPENDENCY) llvm-patched | usr/lib
@$(call PRINT_CC, $(CXX) $(CXX_ABI_SETTING) -fno-rtti -DLIBRARY_EXPORTS -fPIC -O0 -g $(FLAGS) $(LLVM_EXTRA_CPPFLAGS) -c ../src/bootstrap.cpp -o $@)


LINKED_LIBS = $(addprefix -l,$(CLANG_LIBS))
ifeq ($(BUILD_LLDB),1)
LINKED_LIBS += $(LLDB_LIBS)
endif

ifneq (,$(wildcard $(BASE_JULIA_BIN)/../lib/libjulia.$(SHLIB_EXT)))
usr/lib/libcxxffi.$(SHLIB_EXT): build/bootstrap.o $(LIB_DEPENDENCY) | usr/lib
ifneq (,$(wildcard $(JULIA_LIB)))
usr/lib/libcxxffi.$(SHLIB_EXT): usr/lib/bootstrap.o $(LIB_DEPENDENCY) | usr/lib
@$(call PRINT_LINK, $(CXX) -shared -fPIC $(JULIA_LDFLAGS) -ljulia $(LDFLAGS) -o $@ $(WHOLE_ARCHIVE) $(LINKED_LIBS) $(NO_WHOLE_ARCHIVE) $< )
else
usr/lib/libcxxffi.$(SHLIB_EXT):
@echo "Not building release library because corresponding julia RELEASE library does not exist."
@echo "To build, simply run the build again once the library at"
@echo $(build_libdir)/libjulia.$(SHLIB_EXT)
@echo $(JULIA_LIB)
@echo "has been built."
endif

ifneq (,$(wildcard $(BASE_JULIA_BIN)/../lib/libjulia-debug.$(SHLIB_EXT)))
ifneq (,$(wildcard $(JULIA_DEBUG_LIB)))
usr/lib/libcxxffi-debug.$(SHLIB_EXT): build/bootstrap.o $(LIB_DEPENDENCY) | usr/lib
@$(call PRINT_LINK, $(CXX) -shared -fPIC $(JULIA_LDFLAGS) -ljulia-debug $(LDFLAGS) -o $@ $(WHOLE_ARCHIVE) $(LINKED_LIBS) $(NO_WHOLE_ARCHIVE) $< )
else
usr/lib/libcxxffi-debug.$(SHLIB_EXT):
@echo "Not building debug library because corresponding julia DEBUG library does not exist."
@echo "To build, simply run the build again once the library at"
@echo $(build_libdir)/libjulia-debug.$(SHLIB_EXT)
@echo $(JULIA_DEBUG_LIB)
@echo "has been built."
endif

build/clang_constants.jl: ../src/cenumvals.jl.h usr/lib/libcxxffi.$(SHLIB_EXT)
usr/clang_constants.jl: ../src/cenumvals.jl.h usr/lib/libcxxffi.$(SHLIB_EXT)
@$(call PRINT_PERL, $(CPP_STDOUT) $(CXXJL_CPPFLAGS) -DJULIA ../src/cenumvals.jl.h > $@)
30 changes: 9 additions & 21 deletions deps/build.jl
Original file line number Diff line number Diff line change
@@ -1,22 +1,11 @@
using Libdl

if haskey(ENV, "PREBUILT_CI_BINARIES") && ENV["PREBUILT_CI_BINARIES"] == "1"
# Try to download pre-built binaries
if !isdir("build") || length(readdir("build")) == 0
os_tag = Sys.isapple() ? "osx" : "linux"
run(`rm -rf build/ src/`)
filename = "llvm-$(os_tag)-$(Base.libllvm_version).tgz"
run(`wget https://s3.amazonaws.com/julia-cxx/$filename`)
run(`tar xzf $filename --strip-components=1`)
end
end

#in case we have specified the path to the julia installation
#that contains the headers etc, use that
BASE_JULIA_BIN = get(ENV, "BASE_JULIA_BIN", Sys.BINDIR)
BASE_JULIA_SRC = get(ENV, "BASE_JULIA_SRC", joinpath(BASE_JULIA_BIN, "..", ".."))
# in case we have specified the path to the julia installation
# that contains the headers etc, use that
BASE_JULIA_BIN = get(ENV, "BASE_JULIA_BIN", Sys.BINDIR) |> normpath
BASE_JULIA_SRC = get(ENV, "BASE_JULIA_SRC", joinpath(BASE_JULIA_BIN, "..", "..")) |> normpath

#write a simple include file with that path
# write a simple include file with that path
println("writing path.jl file")
s = """
const BASE_JULIA_BIN=$(sprint(show, BASE_JULIA_BIN))
Expand All @@ -28,21 +17,20 @@ export BASE_JULIA_SRC

println("Tuning for julia installation at $BASE_JULIA_BIN with sources possibly at $BASE_JULIA_SRC")

# Try to autodetect C++ ABI in use
# try to autodetect C++ ABI in use
llvm_path = Sys.iswindows() ? "LLVM" :
(Sys.isapple() && Base.libllvm_version >= v"3.8") ? "libLLVM" : "libLLVM-$(Base.libllvm_version)"

Sys.isapple() ? "libLLVM" : "libLLVM-$(Base.libllvm_version)"
llvm_lib_path = Libdl.dlpath(llvm_path)
old_cxx_abi = findfirst("_ZN4llvm3sys16getProcessTripleEv", String(open(read, llvm_lib_path))) !== nothing
old_cxx_abi && (ENV["OLD_CXX_ABI"] = "1")

llvm_config_path = joinpath(BASE_JULIA_BIN,"..","tools","llvm-config")
llvm_config_path = joinpath(BASE_JULIA_BIN, "..", "tools", "llvm-config")
if isfile(llvm_config_path)
@info "Building julia source build"
ENV["LLVM_CONFIG"] = llvm_config_path
delete!(ENV,"LLVM_VER")
make = Sys.isbsd() && !Sys.isapple() ? `gmake` : `make`
run(`$make -j$(Sys.CPU_THREADS) -f BuildBootstrap.Makefile BASE_JULIA_BIN=$BASE_JULIA_BIN BASE_JULIA_SRC=$BASE_JULIA_SRC`)
run(`$make all -j$(Sys.CPU_THREADS) -f BuildBootstrap.Makefile BASE_JULIA_BIN=$BASE_JULIA_BIN BASE_JULIA_SRC=$BASE_JULIA_SRC`)
s = s * "\n const IS_BINARYBUILD = false"
else
@info "Building julia binary build"
Expand Down
6 changes: 0 additions & 6 deletions deps/llvm-patches/README

This file was deleted.

Loading