Skip to content

Commit b9107b4

Browse files
ShillakerShillaker
committed
Switch to a copy-in Docker approach
1 parent c4cf4ed commit b9107b4

14 files changed

+184
-110
lines changed

‎.dockerignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Ignore local builds
2+
third-party/llvm-project/build
3+
third-party/wasi-libc/build

‎.github/workflows/build.yml

+13-17
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,30 @@ name: Build
22

33
on:
44
push:
5-
tags:
6-
- 'v*.*.*'
5+
branches: [master]
6+
pull_request:
7+
branches: [master]
78

89
jobs:
910
build:
1011
runs-on: ubuntu-20.04
1112
steps:
12-
- name: "Get the tag"
13-
run: echo ::set-env name=TAG_VERSION::${GITHUB_REF#refs/*/}
14-
- name: "Print tag"
15-
run: echo ${{ env.TAG_VERSION }}
13+
- name: "Get the latest code"
14+
uses: actions/checkout@v2
15+
- name: "Get LLVM version"
16+
id: llvm_ver
17+
run: echo "::set-output name=version::$(cat LLVM_VERSION)"
18+
shell: bash
1619
- name: "Set up QEMU"
1720
uses: docker/setup-qemu-action@v1
1821
- name: "Set up Docker Buildx"
1922
uses: docker/setup-buildx-action@v1
20-
- name: "Log in to DockerHub"
21-
uses: docker/login-action@v1
22-
with:
23-
username: ${{ secrets.DOCKER_USERNAME }}
24-
password: ${{ secrets.DOCKER_PASSWORD }}
25-
- name: "Build and push sysroot container"
23+
- name: "Build the sysroot container"
2624
id: docker_build
2725
uses: docker/build-push-action@v2
2826
with:
29-
push: true
27+
push: false
3028
file: docker/sysroot.dockerfile
31-
tags: faasm/sysroot:${{ env.TAG_VERSION }}
3229
build-args:
33-
- TOOLCHAIN_VERSION=${{ env.TAG_VERSION }}
34-
- name: "Image digest"
35-
run: echo ${{ steps.docker_build.outputs.digest }}
30+
- LLVM_VERSION=${{ steps.llvm_ver.outputs.version }}
31+

‎.github/workflows/release.yml

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Release
2+
3+
on:
4+
push:
5+
tags:
6+
- 'v*.*.*'
7+
8+
jobs:
9+
build:
10+
runs-on: ubuntu-20.04
11+
steps:
12+
- name: "Get the code"
13+
uses: actions/checkout@v2
14+
- name: "Get LLVM version"
15+
id: llvm_ver
16+
run: echo "::set-output name=version::$(cat LLVM_VERSION)"
17+
shell: bash
18+
- name: "Get toolchain version"
19+
id: toolchain_ver
20+
run: echo "::set-output name=version::$(cat VERSION)"
21+
shell: bash
22+
- name: "Set up QEMU"
23+
uses: docker/setup-qemu-action@v1
24+
- name: "Set up Docker Buildx"
25+
uses: docker/setup-buildx-action@v1
26+
- name: "Log in to DockerHub"
27+
uses: docker/login-action@v1
28+
with:
29+
username: ${{ secrets.DOCKER_USERNAME }}
30+
password: ${{ secrets.DOCKER_PASSWORD }}
31+
- name: "Build and push sysroot container"
32+
id: docker_build
33+
uses: docker/build-push-action@v2
34+
with:
35+
push: true
36+
file: docker/sysroot.dockerfile
37+
tags: faasm/sysroot:${{ steps.toolchain_ver.outputs.version }}
38+
build-args:
39+
- LLVM_VERSION=${{ steps.llvm_ver.outputs.version }}

‎LLVM_VERSION

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
10.0.1

‎Makefile

+1-2
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,7 @@ FAASM_LOCAL_DIR=/usr/local/faasm
1010
PREFIX=$(FAASM_LOCAL_DIR)/toolchain
1111
FAASM_SYSROOT=/usr/local/faasm/llvm-sysroot
1212

13-
# NOTE - upgrading LLVM itself is done by updating the submodule in this project
14-
CLANG_VERSION=10.0.1
13+
CLANG_VERSION=$(shell cat LLVM_VERSION)
1514

1615
BUILD_DIR=$(LLVM_PROJ_DIR)/build
1716
LLVM_CONFIG=$(BUILD_DIR)/llvm/bin/llvm-config

‎docker/sysroot.dockerfile

+16-10
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,20 @@
1-
ARG TOOLCHAIN_VERSION
2-
FROM faasm/toolchain:${TOOLCHAIN_VERSION}
3-
ARG TOOLCHAIN_VERSION
1+
ARG LLVM_VERSION
2+
FROM faasm/toolchain:${LLVM_VERSION}
3+
ARG LLVM_VERSION
44

5-
# Update existing code checkout to latest tag
6-
WORKDIR /code/faasm-toolchain
7-
RUN git fetch --all
8-
RUN git checkout v${TOOLCHAIN_VERSION}
9-
RUN git submodule update --init --recursive
5+
# Install Python tooling
6+
RUN apt update
7+
RUN apt install \
8+
python3-dev \
9+
python3-pip
10+
11+
WORKDIR /code
12+
COPY requirements.txt .
13+
RUN pip3 install -r requirements.txt
14+
15+
# Copy the code in
16+
COPY . .
1017

1118
# Rebuild libc
12-
RUN make clean-libc
13-
RUN make
19+
RUN inv libc --clean
1420

‎docker/toolchain.dockerfile

+13-14
Original file line numberDiff line numberDiff line change
@@ -8,36 +8,35 @@ RUN apt upgrade -y
88
RUN apt install -y \
99
autoconf \
1010
clang-10 \
11-
clang-format-10 \
1211
build-essential \
1312
git \
1413
ninja-build \
15-
python3-dev \
16-
python3-pip \
1714
pkg-config \
1815
wget
1916

20-
# Python deps
21-
RUN pip3 install black
22-
2317
# Up-to-date CMake
2418
RUN apt remove --purge --auto-remove cmake
2519
WORKDIR /setup
2620
RUN wget -q -O cmake-linux.sh https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2-Linux-x86_64.sh
2721
RUN sh cmake-linux.sh -- --skip-license --prefix=/usr/local
2822

29-
# Run the build from the master branch
23+
# Tidy up
24+
RUN apt-get clean autoclean
25+
RUN apt-get autoremove
26+
27+
# Copy the code in
3028
WORKDIR /code
31-
RUN git clone https://github.com/faasm/faasm-toolchain/
32-
WORKDIR /code/faasm-toolchain
33-
RUN git submodule update --init --recursive
29+
COPY . .
30+
31+
# Run the main make
3432
RUN make
3533

36-
# Tidy up
34+
# Print the clang version
35+
RUN /usr/local/faasm/toolchain/bin/clang --version
36+
37+
# Remove the code
3738
WORKDIR /
38-
RUN rm -r /setup
39-
RUN apt-get clean autoclean
40-
RUN apt-get autoremove
39+
RUN rm -r /code
4140

4241
CMD /bin/bash
4342

‎docs/release.md

+37-18
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,51 @@
11
# Building and Releasing
22

3-
This repo has two types of releases:
3+
The toolchain repo is based on two Docker images:
44

5-
- LLVM releases e.g. `vllvm-10.0.1`, which include the bundled LLVM tooling
6-
- Normal releases e.g. `v0.0.1`, which include a bundled sysroot
5+
- `faasm/toolchain` - the base image holding just the custom LLVM tools
6+
- `faasm/sysroot` - the image holding both the tools and sysroot
77

8-
## Docker
8+
See the [Actions page](https://github.com/faasm/faasm-toolchain/actions) and
9+
[Dockerfiles](docker) for more info.
910

10-
The toolchain uses two Docker images:
11+
You only need to rebuild the `toolchain` image when upgrading LLVM (see
12+
[the docs](docs/upgrade-llvm.md).
1113

12-
- `faasm/toolchain` - used to build the custom LLVM tools, not rebuilt regularly
13-
- `faasm/sysroot` - used to build and package the sysroot
14+
The `sysroot` image is rebuilt as part of the CI and tagging process.
1415

15-
See the [Actions page](https://github.com/faasm/faasm-toolchain/actions) and
16-
[Dockerfiles](docker) for more info.
16+
## Rebuilding `toolchain`
17+
18+
You should only need to manually rebuild the `toolchain` image, the `sysroot`
19+
image is built through GH Actions.
20+
21+
```bash
22+
# Build the toolchain image
23+
inv container.toolchain` - rebuilds the base toolchain image
24+
25+
# Push the latest toolchain image build
26+
inv container.push-toolchain
27+
```
28+
29+
## Rebuilding `sysroot`
30+
31+
If you do want to build `sysroot` locally (e.g. for debugging issues):
32+
33+
```bash
34+
inv container.sysroot
35+
```
1736

18-
## Building Docker images
37+
## CI build
1938

20-
We have two tasks for building the Docker images:
39+
The CI build will run through the `sysroot` build but not push any new images.
2140

22-
- `inv container.toolchain` - rebuilds the base toolchain image
23-
- `inv container.sysroot` - rebuilds the sysroot image
41+
## Release build
2442

25-
## Releasing
43+
The release build will run the `sysroot` build and push the Docker image to
44+
Dockerhub.
2645

27-
To create a new release:
46+
To do this:
2847

29-
- Update the versions in `VERSION` and the Github Actions build
30-
- Run `inv git.tag` from the branch you want to tag
31-
- Push and let the CI do the rest
48+
- Update the version in `VERSION`
49+
- Run `inv git.tag` to create the tag (from the head of the current branch)
50+
- Let the CI build run through and build the container
3251

‎docs/upgrade-llvm.md

+14-27
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,16 @@
11
# Upgrading LLVM
22

3-
To upgrade the underlying LLVM version you need to update the _submodule_ in
4-
this project. Find the commit related to tag name for the desired release in
3+
To upgrade the underlying LLVM version you need to:
4+
5+
- Update the _submodule_ in this project (see below)
6+
- Update the `LLVM_VERSION` file
7+
- Run the `inv container.toolchain` task
8+
- Test it out
9+
- Run the `inv container.push-toolchain` task
10+
11+
## Updating the LLVM submodule
12+
13+
Find the commit related to tag name for the desired release in
514
[the LLVM repo](https://github.com/llvm/llvm-project/releases) (e.g.
615
`llvmorg-10.0.1`), then:
716

@@ -12,38 +21,16 @@ git fetch origin
1221
git checkout <tag-name>
1322
```
1423

15-
Once done, check the [Makefile](Makefile) for any references to LLVM
16-
versions and update accordingly.
17-
18-
You can then follow the steps below as normal.
19-
20-
## Building From Scratch
21-
22-
To build from scratch:
23-
24-
```bash
25-
make clean-all
26-
make
27-
```
28-
29-
This a while as it builds a lot of the LLVM toolchain from scratch.
30-
31-
When it finishes, check things are up to date with:
32-
33-
```bash
34-
/usr/local/faasm/toolchain/bin/clang --version
35-
```
36-
37-
### Rebuilding
24+
# Rebuilding LLVM
3825

39-
To rebuild, there different options. The first is just rebuilding libc:
26+
To rebuild LLVM, there a couple of options. To rebuild just libc:
4027

4128
```bash
4229
make clean-libc
4330
make
4431
```
4532

46-
Another is rebuilding all the libs, i.e. libc, libcxx, libcxxabi and
33+
To rebuilding all the libs, i.e. libc, libcxx, libcxxabi and
4734
compiler-rt:
4835

4936
```bash

‎pyproject.toml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[tool.black]
2+
line-length = 79
3+

‎requirements.txt

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
invoke
2+

0 commit comments

Comments
 (0)