Skip to content

refactor: keep sequences compressed in the query engine to allow sorting of large sequence sets #3140

refactor: keep sequences compressed in the query engine to allow sorting of large sequence sets

refactor: keep sequences compressed in the query engine to allow sorting of large sequence sets #3140

Workflow file for this run

name: LAPIS-SILO
on:
pull_request:
workflow_run:
workflows: [Release SILO]
types: [completed]
branches: [main]
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
env:
DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies
DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo
jobs:
dockerImage:
name: Build docker images
strategy:
matrix:
arch: ["amd64", "arm64"]
runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }}
permissions:
packages: write
steps:
- uses: actions/checkout@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Determine HEAD_SHA
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
else
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
fi
- name: Generate dependency files hash
run: |
DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', './Dockerfile_dependencies') }})
echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
- name: Check if dependency image exists
run: |
EXISTS=$(docker manifest inspect \
${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} > /dev/null 2>&1 &&
echo "true" || echo "false")
echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
- name: Build and push dependency image if input files changed
if: env.CACHE_HIT == 'false'
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile_dependencies
push: true
tags: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }}
cache-from: type=gha,ref=dependency-image-cache-${{ matrix.arch }}-${{ hashFiles('conanfile.py', 'Dockerfile') }}
cache-to: type=gha,mode=min,ref=dependency-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
- name: Tag dependency image with commit hash
run: |
docker buildx imagetools create \
--tag ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} \
${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }}
- name: Write release_version.txt
run: |
git fetch origin 'refs/tags/*:refs/tags/*'
echo "Available tags:"
git tag -l
if git describe --tags --exact-match >/dev/null 2>&1; then
echo "Using release version from git tag"
VERSION=$(git describe --tags --exact-match | sed 's/^v//')
else
echo "Using release version from commit hash"
VERSION="${{ env.HEAD_SHA }}"
fi
echo "Using release version: $VERSION"
echo "$VERSION" > release_version.txt
- name: Build unit test image
uses: docker/build-push-action@v6
with:
context: .
target: builder
tags: builder
load: true
cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
build-args: |
DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }}
- name: Run unit tests
run: |
docker run \
--entrypoint "./silo_test" \
builder
- name: Build and push production image
uses: docker/build-push-action@v6
with:
context: .
push: true
cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
tags: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }}
build-args: |
DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }}
multiPlatformImages:
name: Create multi-platform images
needs: dockerImage
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- uses: actions/checkout@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Determine HEAD_SHA
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
else
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
fi
- name: Determine whether to use a release version
run: |
git fetch origin 'refs/tags/*:refs/tags/*'
echo "Available tags:"
git tag -l
if git describe --tags --exact-match >/dev/null 2>&1; then
echo "Using release version from git tag"
VERSION=$(git describe --tags --exact-match | sed 's/^v//')
echo "SILO_VERSION=${VERSION}" >> $GITHUB_ENV
fi
- name: Docker metadata
id: dockerMetadataImage
uses: docker/metadata-action@v5
with:
images: ${{ env.DOCKER_IMAGE_NAME }}
tags: |
type=ref,event=pr
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=raw,value=commit-${{ env.HEAD_SHA }}
type=semver,value=${{ env.SILO_VERSION }},enable=${{ env.SILO_VERSION != '' }},pattern={{version}}
type=semver,value=${{ env.SILO_VERSION }},enable=${{ env.SILO_VERSION != '' }},pattern={{major}}.{{minor}}
- name: Tag images
run: |
TAGS=(${{ steps.dockerMetadataImage.outputs.tags }})
for TAG in "${TAGS[@]}"; do
docker buildx imagetools create --tag $TAG \
"${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-arm64" \
"${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64"
done
endToEndTests:
name: Run End To End Tests
needs: dockerImage
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: 18.x
- name: Determine HEAD_SHA
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
else
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
fi
- uses: actions/cache@v4
with:
path: ~/.npm
key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}"
- name: npm install
run: cd endToEndTests && npm ci
- name: Check Format
run: cd endToEndTests && npm run check-format
- name: Start Docker Container and preprocess data
run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up
env:
SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64
- name: Start Docker Container and run api
run: docker compose -f docker-compose-for-tests-api.yml up -d --wait
env:
SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64
- name: Run Tests
run: cd endToEndTests && SILO_URL=localhost:8080 npm run test
linterChanges:
name: Build/Run linter on changed files
needs: dockerImage
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Determine HEAD_SHA
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
else
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
fi
- name: Run linter script
run: bash ./linter_changes.bash ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}-amd64
linterAll:
name: Build/Run linter on all files
needs: dockerImage
if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'trigger-linter')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Determine HEAD_SHA
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
else
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
fi
- shell: bash
name: Build all files with clang-tidy
run: |
docker build -f ./Dockerfile_linter \
--build-arg DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}-amd64 .