refactor: keep sequences compressed in the query engine to allow sorting of large sequence sets #3140
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LAPIS-SILO | |
| on: | |
| pull_request: | |
| workflow_run: | |
| workflows: [Release SILO] | |
| types: [completed] | |
| branches: [main] | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies | |
| DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo | |
| jobs: | |
| dockerImage: | |
| name: Build docker images | |
| strategy: | |
| matrix: | |
| arch: ["amd64", "arm64"] | |
| runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }} | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - name: Generate dependency files hash | |
| run: | | |
| DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', './Dockerfile_dependencies') }}) | |
| echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV | |
| - name: Check if dependency image exists | |
| run: | | |
| EXISTS=$(docker manifest inspect \ | |
| ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} > /dev/null 2>&1 && | |
| echo "true" || echo "false") | |
| echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV | |
| - name: Build and push dependency image if input files changed | |
| if: env.CACHE_HIT == 'false' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: Dockerfile_dependencies | |
| push: true | |
| tags: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} | |
| cache-from: type=gha,ref=dependency-image-cache-${{ matrix.arch }}-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
| cache-to: type=gha,mode=min,ref=dependency-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
| - name: Tag dependency image with commit hash | |
| run: | | |
| docker buildx imagetools create \ | |
| --tag ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} \ | |
| ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} | |
| - name: Write release_version.txt | |
| run: | | |
| git fetch origin 'refs/tags/*:refs/tags/*' | |
| echo "Available tags:" | |
| git tag -l | |
| if git describe --tags --exact-match >/dev/null 2>&1; then | |
| echo "Using release version from git tag" | |
| VERSION=$(git describe --tags --exact-match | sed 's/^v//') | |
| else | |
| echo "Using release version from commit hash" | |
| VERSION="${{ env.HEAD_SHA }}" | |
| fi | |
| echo "Using release version: $VERSION" | |
| echo "$VERSION" > release_version.txt | |
| - name: Build unit test image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| target: builder | |
| tags: builder | |
| load: true | |
| cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
| cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
| build-args: | | |
| DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
| - name: Run unit tests | |
| run: | | |
| docker run \ | |
| --entrypoint "./silo_test" \ | |
| builder | |
| - name: Build and push production image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
| cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
| tags: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
| build-args: | | |
| DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
| multiPlatformImages: | |
| name: Create multi-platform images | |
| needs: dockerImage | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - name: Determine whether to use a release version | |
| run: | | |
| git fetch origin 'refs/tags/*:refs/tags/*' | |
| echo "Available tags:" | |
| git tag -l | |
| if git describe --tags --exact-match >/dev/null 2>&1; then | |
| echo "Using release version from git tag" | |
| VERSION=$(git describe --tags --exact-match | sed 's/^v//') | |
| echo "SILO_VERSION=${VERSION}" >> $GITHUB_ENV | |
| fi | |
| - name: Docker metadata | |
| id: dockerMetadataImage | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=pr | |
| type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} | |
| type=raw,value=commit-${{ env.HEAD_SHA }} | |
| type=semver,value=${{ env.SILO_VERSION }},enable=${{ env.SILO_VERSION != '' }},pattern={{version}} | |
| type=semver,value=${{ env.SILO_VERSION }},enable=${{ env.SILO_VERSION != '' }},pattern={{major}}.{{minor}} | |
| - name: Tag images | |
| run: | | |
| TAGS=(${{ steps.dockerMetadataImage.outputs.tags }}) | |
| for TAG in "${TAGS[@]}"; do | |
| docker buildx imagetools create --tag $TAG \ | |
| "${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-arm64" \ | |
| "${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64" | |
| done | |
| endToEndTests: | |
| name: Run End To End Tests | |
| needs: dockerImage | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18.x | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.npm | |
| key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}" | |
| - name: npm install | |
| run: cd endToEndTests && npm ci | |
| - name: Check Format | |
| run: cd endToEndTests && npm run check-format | |
| - name: Start Docker Container and preprocess data | |
| run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up | |
| env: | |
| SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64 | |
| - name: Start Docker Container and run api | |
| run: docker compose -f docker-compose-for-tests-api.yml up -d --wait | |
| env: | |
| SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64 | |
| - name: Run Tests | |
| run: cd endToEndTests && SILO_URL=localhost:8080 npm run test | |
| linterChanges: | |
| name: Build/Run linter on changed files | |
| needs: dockerImage | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - name: Run linter script | |
| run: bash ./linter_changes.bash ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}-amd64 | |
| linterAll: | |
| name: Build/Run linter on all files | |
| needs: dockerImage | |
| if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'trigger-linter') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - shell: bash | |
| name: Build all files with clang-tidy | |
| run: | | |
| docker build -f ./Dockerfile_linter \ | |
| --build-arg DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}-amd64 . |