feat: Improve selected columns storage mechanism #7262
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests (E2E) | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths: | |
| - "keep/**" | |
| - "keep-ui/**" | |
| - "tests/**" | |
| # Add permissions for GitHub Container Registry | |
| permissions: | |
| contents: read | |
| packages: write | |
| concurrency: | |
| group: ${{ github.event_name }}-${{ github.workflow }}-${{ github.head_ref }} | |
| cancel-in-progress: true | |
| env: | |
| PYTHON_VERSION: 3.11 | |
| STORAGE_MANAGER_DIRECTORY: /tmp/storage-manager | |
| # MySQL server environment variables | |
| MYSQL_ROOT_PASSWORD: keep | |
| MYSQL_DATABASE: keep | |
| # Postgres environment variables | |
| POSTGRES_USER: keepuser | |
| POSTGRES_PASSWORD: keeppassword | |
| POSTGRES_DB: keepdb | |
| # To test if imports are working properly | |
| EE_ENABLED: true | |
| # Docker Compose project name | |
| COMPOSE_PROJECT_NAME: keep | |
| # Check if PR is from fork (external contributor) | |
| IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork }} | |
| jobs: | |
| # Prepare test environment in parallel with Docker builds | |
| prepare-test-environment: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| - name: Cache dependencies | |
| id: cache-deps | |
| uses: actions/[email protected] | |
| with: | |
| path: .venv | |
| key: pydeps-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install dependencies using poetry | |
| run: poetry install --no-interaction --no-root --with dev | |
| - name: Get Playwright version from poetry.lock | |
| id: playwright-version | |
| run: | | |
| PLAYWRIGHT_VERSION=$(grep "playwright" poetry.lock -A 5 | grep "version" | head -n 1 | cut -d'"' -f2) | |
| echo "version=$PLAYWRIGHT_VERSION" >> $GITHUB_OUTPUT | |
| - name: Cache Playwright browsers | |
| id: playwright-cache | |
| uses: actions/[email protected] | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: playwright-${{ steps.playwright-version.outputs.version }} | |
| - name: Install Playwright and dependencies | |
| run: | | |
| if [ "${{ steps.playwright-cache.outputs.cache-hit }}" != "true" ]; then | |
| poetry run playwright install --with-deps | |
| else | |
| poetry run playwright install-deps | |
| fi | |
| # Build images in parallel | |
| build-frontend: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_name: ${{ steps.set-image-name.outputs.image_name }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Set image name | |
| id: set-image-name | |
| run: | | |
| if [[ "${{ env.IS_FORK }}" == "true" ]]; then | |
| echo "image_name=keep-frontend:local" >> $GITHUB_OUTPUT | |
| else | |
| echo "image_name=ghcr.io/${{ github.repository_owner }}/keep-frontend:${{ github.sha }}" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to GitHub Container Registry | |
| if: ${{ env.IS_FORK != 'true' }} | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Set cache key variables | |
| id: cache-keys | |
| run: | | |
| # Create a safe branch name for cache key (replace / with - and remove special chars) | |
| SAFE_BRANCH=$(echo "${{ github.head_ref || github.ref_name }}" | sed 's/\//-/g' | sed 's/[^a-zA-Z0-9._-]//g') | |
| echo "SAFE_BRANCH_NAME=${SAFE_BRANCH}" >> $GITHUB_OUTPUT | |
| # Create a hash ONLY of the dependencies section of package.json and package-lock.json | |
| # This ensures the hash only changes when dependencies change | |
| DEPS_HASH=$(jq '.dependencies' keep-ui/package.json | sha256sum | cut -d ' ' -f 1) | |
| echo "DEPS_HASH=${DEPS_HASH:0:8}" >> $GITHUB_OUTPUT | |
| - name: Debug repository and cache info | |
| run: | | |
| echo "Repository: ${{ github.repository }}" | |
| echo "Repository owner: ${{ github.repository_owner }}" | |
| echo "Branch: ${{ github.head_ref || github.ref_name }}" | |
| echo "Safe branch name: ${{ steps.cache-keys.outputs.SAFE_BRANCH_NAME }}" | |
| echo "Dependencies hash: ${{ steps.cache-keys.outputs.DEPS_HASH }}" | |
| echo "Is fork: ${{ env.IS_FORK }}" | |
| # Pre-check if branch cache exists (only for non-forks) | |
| - name: Check if branch cache exists | |
| id: branch-cache-exists | |
| if: ${{ env.IS_FORK != 'true' }} | |
| continue-on-error: true | |
| run: | | |
| BRANCH_CACHE_TAG="ghcr.io/${{ github.repository_owner }}/keep-frontend:cache-${{ steps.cache-keys.outputs.SAFE_BRANCH_NAME }}" | |
| if docker buildx imagetools inspect "$BRANCH_CACHE_TAG" &>/dev/null; then | |
| echo "Branch cache exists: $BRANCH_CACHE_TAG" | |
| echo "cache_exists=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Branch cache does not exist: $BRANCH_CACHE_TAG" | |
| echo "cache_exists=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Log frontend cache status | |
| if: ${{ env.IS_FORK != 'true' }} | |
| run: | | |
| if [ "${{ steps.branch-cache-exists.outputs.cache_exists }}" == "true" ]; then | |
| echo "FRONTEND CACHE HIT ✅" | |
| echo "Cache tag: ghcr.io/${{ github.repository_owner }}/keep-frontend:cache-${{ steps.cache-keys.outputs.SAFE_BRANCH_NAME }}" | |
| else | |
| echo "FRONTEND CACHE MISS ❌" | |
| echo "Will attempt to use main branch cache and create a new branch cache" | |
| fi | |
| # For non-forks: Build and push to registry | |
| - name: Build and push frontend image with registry cache | |
| if: ${{ env.IS_FORK != 'true' }} | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: keep-ui | |
| file: ./docker/Dockerfile.ui | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository_owner }}/keep-frontend:${{ github.sha }} | |
| # Use registry-based caching with branch-specific tags | |
| cache-from: | | |
| type=registry,ref=ghcr.io/${{ github.repository_owner }}/keep-frontend:cache-${{ steps.cache-keys.outputs.SAFE_BRANCH_NAME }} | |
| type=registry,ref=ghcr.io/${{ github.repository_owner }}/keep-frontend:cache-${{ steps.cache-keys.outputs.DEPS_HASH }} | |
| type=registry,ref=ghcr.io/${{ github.repository_owner }}/keep-frontend:cache-main | |
| cache-to: | | |
| type=registry,ref=ghcr.io/${{ github.repository_owner }}/keep-frontend:cache-${{ steps.cache-keys.outputs.SAFE_BRANCH_NAME }},mode=max | |
| type=registry,ref=ghcr.io/${{ github.repository_owner }}/keep-frontend:cache-${{ steps.cache-keys.outputs.DEPS_HASH }},mode=max | |
| # Add build args for better caching | |
| build-args: | | |
| BUILDKIT_INLINE_CACHE=1 | |
| # Verbose output | |
| outputs: type=image,push=true | |
| build-backend: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_name: ${{ steps.set-image-name.outputs.image_name }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Set image name | |
| id: set-image-name | |
| run: | | |
| if [[ "${{ env.IS_FORK }}" == "true" ]]; then | |
| echo "image_name=keep-backend:local" >> $GITHUB_OUTPUT | |
| else | |
| echo "image_name=ghcr.io/${{ github.repository_owner }}/keep-backend:${{ github.sha }}" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to GitHub Container Registry | |
| if: ${{ env.IS_FORK != 'true' }} | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Set cache key variables | |
| id: cache-keys | |
| run: | | |
| # Create a safe branch name for cache key (replace / with - and remove special chars) | |
| SAFE_BRANCH=$(echo "${{ github.head_ref || github.ref_name }}" | sed 's/\//-/g' | sed 's/[^a-zA-Z0-9._-]//g') | |
| echo "SAFE_BRANCH_NAME=${SAFE_BRANCH}" >> $GITHUB_OUTPUT | |
| # Create a hash of poetry files for version-specific caching | |
| DEPS_HASH=$(cat poetry.lock pyproject.toml | sha256sum | cut -d ' ' -f 1) | |
| echo "DEPS_HASH=${DEPS_HASH:0:8}" >> $GITHUB_OUTPUT | |
| - name: Debug repository and cache info | |
| run: | | |
| echo "Repository: ${{ github.repository }}" | |
| echo "Repository owner: ${{ github.repository_owner }}" | |
| echo "Branch: ${{ github.head_ref || github.ref_name }}" | |
| echo "Safe branch name: ${{ steps.cache-keys.outputs.SAFE_BRANCH_NAME }}" | |
| echo "Dependencies hash: ${{ steps.cache-keys.outputs.DEPS_HASH }}" | |
| echo "Is fork: ${{ env.IS_FORK }}" | |
| # Pre-check if branch cache exists (only for non-forks) | |
| - name: Check if branch cache exists | |
| id: branch-cache-exists | |
| if: ${{ env.IS_FORK != 'true' }} | |
| continue-on-error: true | |
| run: | | |
| BRANCH_CACHE_TAG="ghcr.io/${{ github.repository_owner }}/keep-backend:cache-${{ steps.cache-keys.outputs.SAFE_BRANCH_NAME }}" | |
| if docker buildx imagetools inspect "$BRANCH_CACHE_TAG" &>/dev/null; then | |
| echo "Branch cache exists: $BRANCH_CACHE_TAG" | |
| echo "cache_exists=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Branch cache does not exist: $BRANCH_CACHE_TAG" | |
| echo "cache_exists=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Log backend cache status | |
| if: ${{ env.IS_FORK != 'true' }} | |
| run: | | |
| if [ "${{ steps.branch-cache-exists.outputs.cache_exists }}" == "true" ]; then | |
| echo "BACKEND CACHE HIT ✅" | |
| echo "Cache tag: ghcr.io/${{ github.repository_owner }}/keep-backend:cache-${{ steps.cache-keys.outputs.SAFE_BRANCH_NAME }}" | |
| else | |
| echo "BACKEND CACHE MISS ❌" | |
| echo "Will attempt to use main branch cache and create a new branch cache" | |
| fi | |
| # For non-forks: Build and push to registry | |
| - name: Build and push backend image with registry cache | |
| if: ${{ env.IS_FORK != 'true' }} | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| file: ./docker/Dockerfile.api | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository_owner }}/keep-backend:${{ github.sha }} | |
| # Use registry-based caching with branch-specific tags | |
| cache-from: | | |
| type=registry,ref=ghcr.io/${{ github.repository_owner }}/keep-backend:cache-${{ steps.cache-keys.outputs.DEPS_HASH }} | |
| type=registry,ref=ghcr.io/${{ github.repository_owner }}/keep-backend:cache-${{ steps.cache-keys.outputs.SAFE_BRANCH_NAME }} | |
| cache-to: | | |
| type=registry,ref=ghcr.io/${{ github.repository_owner }}/keep-backend:cache-${{ steps.cache-keys.outputs.DEPS_HASH }},mode=max | |
| type=registry,ref=ghcr.io/${{ github.repository_owner }}/keep-backend:cache-${{ steps.cache-keys.outputs.SAFE_BRANCH_NAME }},mode=max | |
| # Add build args for better caching | |
| build-args: | | |
| BUILDKIT_INLINE_CACHE=1 | |
| # Verbose output | |
| outputs: type=image,push=true | |
| # Run tests with all services in one job | |
| run-mysql-with-redis: | |
| needs: [build-frontend, build-backend, prepare-test-environment] | |
| uses: ./.github/workflows/run-e2e-tests.yml | |
| with: | |
| db-type: mysql | |
| redis_enabled: true | |
| python-version: 3.11 | |
| is-fork: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork }} | |
| backend-image-name: ${{ needs.build-backend.outputs.image_name }} | |
| frontend-image-name: ${{ needs.build-frontend.outputs.image_name }} | |
| run-postgresql-without-redis: | |
| needs: [build-frontend, build-backend, prepare-test-environment] | |
| uses: ./.github/workflows/run-e2e-tests.yml | |
| with: | |
| db-type: postgres | |
| redis_enabled: false | |
| python-version: 3.11 | |
| is-fork: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork }} | |
| backend-image-name: ${{ needs.build-backend.outputs.image_name }} | |
| frontend-image-name: ${{ needs.build-frontend.outputs.image_name }} | |
| run-sqlite-without-redis: | |
| needs: [build-frontend, build-backend, prepare-test-environment] | |
| uses: ./.github/workflows/run-e2e-tests.yml | |
| with: | |
| db-type: sqlite | |
| redis_enabled: false | |
| python-version: 3.11 | |
| is-fork: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork }} | |
| backend-image-name: ${{ needs.build-backend.outputs.image_name }} | |
| frontend-image-name: ${{ needs.build-frontend.outputs.image_name }} |