Merge branch 'master' into embeddings_generation

Author: donhardman

.github/workflows/build_template.yml

@@ -27,6 +27,14 @@ on:
         required: false
         type: number
         default: 1
+      version:
+        required: false
+        type: string
+        default: ""
+      build_tag:
+        required: false
+        type: string
+        default: ""
       cmake_command:
         required: false
         type: string
@@ -89,13 +97,20 @@ jobs:
         HOMEBREW_PREFIX: ${{ inputs.HOMEBREW_PREFIX }}
         PACK_GALERA: ${{ inputs.PACK_GALERA }}
         COLUMNAR_LOCATOR: ${{ inputs.COLUMNAR_LOCATOR }}
+        BUILD_TAG: ${{ inputs.build_tag }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           set-safe-directory: true
 
+      - name: Patch version
+        if: inputs.version != ''
+        run: |
+          sed -i "s/0\.0\.0/${{ inputs.version }}/g" src/sphinxversion.h.in
+          cat src/sphinxversion.h.in
+
       - name: Initialization
         run: | # without adding the safe.directory cmake fails to do git log -1 --date=short --format=%cd
           bash /sysroot/root/entry_point.sh

.github/workflows/clt_nightly.yml

@@ -148,7 +148,7 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-      - uses: manticoresoftware/[email protected].3
+      - uses: manticoresoftware/[email protected].5
         with:
           test_prefix: ${{ matrix.test_prefix }}
           image: ${{ matrix.image }}
@@ -237,7 +237,7 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-      - uses: manticoresoftware/[email protected].3
+      - uses: manticoresoftware/[email protected].5
         with:
           test_prefix: ${{ matrix.test_prefix }}
           image: ${{ matrix.image }}

.github/workflows/clt_tests.yml

@@ -54,8 +54,10 @@ jobs:
             test_prefix: test/clt-tests/sharding/
           - name: Indexer
             test_prefix: test/clt-tests/indexer/
+          - name: Kibana
+            test_prefix: test/clt-tests/kibana/
     steps:
-      - uses: manticoresoftware/[email protected].3
+      - uses: manticoresoftware/[email protected].5
         with:
           artifact: ${{ inputs.artifact_name }}
           image: ${{ inputs.docker_image }}

.github/workflows/nightly_dumps.yml

@@ -31,7 +31,7 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - uses: manticoresoftware/[email protected].3
+      - uses: manticoresoftware/[email protected].5
         with:
           image: manticoresearch/dind:v1
           test_prefix: test/clt-tests/mysqldump/versions/
@@ -43,7 +43,7 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - uses: manticoresoftware/[email protected].3
+      - uses: manticoresoftware/[email protected].5
         with:
           image: manticoresearch/dind:v1
           test_prefix: test/clt-tests/migration-es-ms/

.github/workflows/nightly_memleaks.yml

@@ -1,5 +1,5 @@
-name: Nightly memleak tests
-run-name: 🌙 Nightly memleak tests ${{ github.sha }}
+name: Nightly valgrind checks
+run-name: 🌙 Nightly valgrind checks ${{ github.sha }}
 
 on:
   schedule:

.github/workflows/pack_publish.yml

@@ -21,6 +21,8 @@ on:
       - 'manual/**'
       - 'cmake/GetGALERA.cmake'
       - 'galera_packaging/**'
+    tags:
+      - 'release*'    
 
 # cancels the previous workflow run when a new one appears in the same branch (e.g. master or a PR's branch)
 concurrency:
@@ -64,6 +66,16 @@ jobs:
   pack:
     name: OK to pack?
     runs-on: ubuntu-22.04
+    permissions:
+      contents: write
+      pull-requests: read
+    outputs:
+      version: ${{ steps.semver-tagger.outputs.version }}
+      version_full: ${{ steps.semver-tagger.outputs.version_full }}
+      version_rpm: ${{ steps.semver-tagger.outputs.version_rpm }}
+      version_deb: ${{ steps.semver-tagger.outputs.version_deb }}
+      target: ${{ steps.semver-tagger.outputs.target }}
+      should_continue: ${{ steps.check-should-continue.outputs.should_continue }}
     if: |
       (
         github.event_name == 'pull_request'
@@ -95,18 +107,45 @@ jobs:
         )
       )
     steps:
-      - run: echo "All set to build packages"
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Update version
+        id: semver-tagger
+        uses: manticoresoftware/semver-tagger-action@main  
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          ignore_patterns: '\.md$|^test/|^manual/|\.clt|\.github|\.patterns|\.yml|\.gitignore'
+      - name: Check if we should continue packing
+        id: check-should-continue
+        run: |
+          # Continue if version was updated or if we have the "pack" label on PR
+          if [[ "${{ steps.semver-tagger.outputs.version_updated }}" == "true" ]]; then
+            echo "Continuing because version was updated"
+            echo "should_continue=true" >> $GITHUB_OUTPUT
+          elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'pack') }}" == "true" ]]; then
+            echo "Continuing because PR has 'pack' label"
+            echo "should_continue=true" >> $GITHUB_OUTPUT
+          else
+            echo "Skipping packing because version wasn't updated and there's no 'pack' label"
+            echo "should_continue=false" >> $GITHUB_OUTPUT
+          fi
       - run: |
-          echo '# Packing and publishing all for commit ${{ github.sha }}' >> $GITHUB_STEP_SUMMARY
-          echo '* Commit URL: [${{ github.sha }}](/${{ github.repository }}/commit/${{ github.sha }})' >> $GITHUB_STEP_SUMMARY
-          echo '* Initiated by: [@${{ github.actor }}](https://github.com/${{ github.actor }})' >> $GITHUB_STEP_SUMMARY
-          echo '* Ref: ${{ github.ref_type }} "${{ github.ref_name }}"' >> $GITHUB_STEP_SUMMARY
-          echo '* Attempt: ${{ github.run_attempt }}' >> $GITHUB_STEP_SUMMARY
+          echo "# Packing and publishing all for commit ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY
+          echo "* Commit URL: [${{ github.sha }}](/${{ github.repository }}/commit/${{ github.sha }})" >> $GITHUB_STEP_SUMMARY
+          echo "* Initiated by: [@${{ github.actor }}](https://github.com/${{ github.actor }})" >> $GITHUB_STEP_SUMMARY
+          echo "* Ref: ${{ github.ref_type }} \"${{ github.ref_name }}\"" >> $GITHUB_STEP_SUMMARY
+          echo "* Attempt: ${{ github.run_attempt }}" >> $GITHUB_STEP_SUMMARY
+          echo "* Version updated: ${{ steps.semver-tagger.outputs.version_updated }}" >> $GITHUB_STEP_SUMMARY
+          echo "* Should continue packing: ${{ steps.check-should-continue.outputs.should_continue }}" >> $GITHUB_STEP_SUMMARY
+
 
   pack_debian_ubuntu:
     name: Debian/Ubuntu packages
     uses: ./.github/workflows/build_template.yml
     needs: [pack, check_branch]
+    if: needs.pack.outputs.should_continue == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -124,11 +163,14 @@ jobs:
         cmake --build . --target package
       cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }}
       artifact_list: "build/manticore*deb"
+      version: ${{ needs.pack.outputs.version }}
+      build_tag: ${{ needs.pack.outputs.target == 'dev' && 'dev' || '' }}
 
   pack_rhel:
     name: RHEL packages
     uses: ./.github/workflows/build_template.yml
     needs: [pack, check_branch]
+    if: needs.pack.outputs.should_continue == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -149,11 +191,14 @@ jobs:
         cmake --build . --target package
       cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }}
       artifact_list: "build/manticore*rpm"
+      version: ${{ needs.pack.outputs.version }}
+      build_tag: ${{ needs.pack.outputs.target == 'dev' && 'dev' || '' }}
 
   pack_macos:
     name: MacOS packages
     uses: ./.github/workflows/build_template.yml
     needs: [pack, check_branch]
+    if: needs.pack.outputs.should_continue == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -172,11 +217,14 @@ jobs:
         cmake --build . --target package
       cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }}
       artifact_list: "build/manticore*tar.gz"
+      version: ${{ needs.pack.outputs.version }}
+      build_tag: ${{ needs.pack.outputs.target == 'dev' && 'dev' || '' }}
 
   pack_windows:
     name: Windows x64 package
     uses: ./.github/workflows/build_template.yml
     needs: [pack, check_branch]
+    if: needs.pack.outputs.should_continue == 'true'
     with:
       COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }}
       DISTR: windows
@@ -191,10 +239,13 @@ jobs:
         cmake --build . --target package
       cache_key: pack_windows_x64
       artifact_list: "build/manticore*exe build/manticore*zip"
-
+      version: ${{ needs.pack.outputs.version }}
+      build_tag: ${{ needs.pack.outputs.target == 'dev' && 'dev' || '' }}
+      
   build_nsis:
     name: Making Windows NSIS installer
-    needs: pack_windows
+    needs: [pack_windows, pack]
+    if: needs.pack.outputs.should_continue == 'true'
     runs-on: ubuntu-22.04
     container:
       image: manticoresearch/build_nsis:1.0.0
@@ -203,14 +254,17 @@ jobs:
     steps:
       - name: Checkout repository # We have to checkout to access .github/workflows/ in further steps
         uses: actions/checkout@v3
+      - name: Patch version
+        run: |
+          sed -i "s/0\.0\.0/${{ needs.pack.outputs.version }}/g" src/sphinxversion.h.in
       - name: Initialization
         # without adding the safe.directory the script fails to do git show ...
         run: git config --global --add safe.directory /__w/manticoresearch/manticoresearch
       - name: Make installer
         # TODO: remove the hardcoded paths /builds/ below
         run: |
           mkdir -p /builds/manticoresearch/dev/build/
-          /bin/bash dist/build_dockers/nsis/nsis_src_parser.sh
+          /bin/bash dist/build_dockers/nsis/nsis_src_parser.sh ${{ needs.pack.outputs.target }}
         shell: bash
       - run: mv /builds/manticoresearch/dev/build build
       - name: Upload artifact
@@ -236,7 +290,7 @@ jobs:
       - run: echo "Ready to publish"
 
   publish_debian_ubuntu:
-    needs: publish
+    needs: [publish, pack]
     strategy:
       fail-fast: true
       matrix:
@@ -253,9 +307,10 @@ jobs:
           artifact: build_${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }}
           type: deb
           delimiter: "-"
+          target: ${{ needs.pack.outputs.target }}
 
   publish_rhel:
-    needs: publish
+    needs: [publish, pack]
     strategy:
       fail-fast: true
       matrix:
@@ -272,10 +327,11 @@ jobs:
           artifact: build_rhel${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }}
           type: rpm
           delimiter: "_"
+          target: ${{ needs.pack.outputs.target }}
 
   publish_macos:
     name: Publishing MacOS
-    needs: publish
+    needs: [publish, pack]
     strategy:
       fail-fast: true
       matrix:
@@ -290,10 +346,11 @@ jobs:
           artifact: build_macos_RelWithDebInfo_${{ matrix.arch }}
           type: arc
           delimiter: "-"
+          target: ${{ needs.pack.outputs.target }}
 
   publish_windows:
     name: Publishing Windows packages to repo.manticoresearch.com
-    needs: publish
+    needs: [publish, pack]
     runs-on: ubuntu-22.04
     steps:
       - uses: manticoresoftware/publish_to_repo@main
@@ -304,10 +361,11 @@ jobs:
           artifact: build_windows_RelWithDebInfo_x64
           type: arc
           delimiter: "-"
+          target: ${{ needs.pack.outputs.target }}
 
   publish_nsis:
     name: Publishing Windows NSIS installer
-    needs: publish_windows
+    needs: [publish_windows, publish, pack]
     runs-on: ubuntu-22.04
     steps:
       - uses: manticoresoftware/publish_to_repo@main
@@ -318,10 +376,11 @@ jobs:
           artifact: win_installer
           type: arc
           delimiter: "-"
+          target: ${{ needs.pack.outputs.target }}
 
   build_docker:
     name: Building and pushing docker
-    needs: publish_debian_ubuntu
+    needs: [publish_debian_ubuntu, pack]
     runs-on: ubuntu-22.04
     env:
       DOCKER_USER: ${{ secrets.DOCKER_USER }}
@@ -331,6 +390,9 @@ jobs:
     steps:
       - name: Checkout repository # We have to checkout to access .github/workflows/ in further steps
         uses: actions/checkout@v3
+      - name: Patch version
+        run: |
+          sed -i "s/0\.0\.0/${{ needs.pack.outputs.version }}/g" src/sphinxversion.h.in
       - name: Calculate short commit hash
         id: sha
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
@@ -350,7 +412,7 @@ jobs:
         image: [ "centos:7", "almalinux:8", "almalinux:9", "oraclelinux:9", "amazonlinux:latest" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/[email protected].3
+      - uses: manticoresoftware/[email protected].5
         with:
           image: ${{ matrix.image }}
           test_prefix: test/clt-tests/installation/rhel-dev-
@@ -365,7 +427,7 @@ jobs:
         image: [ "ubuntu:bionic", "ubuntu:focal", "ubuntu:jammy", "debian:buster", "debian:bullseye", "debian:bookworm" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/[email protected].3
+      - uses: manticoresoftware/[email protected].5
         with:
           image: ${{ matrix.image }}
           test_prefix: test/clt-tests/installation/deb-dev-

.github/workflows/pack_publish_galera.yml

@@ -205,7 +205,7 @@ jobs:
         image: [ "centos:7", "almalinux:8", "almalinux:9", "oraclelinux:9", "amazonlinux:latest" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/[email protected].3
+      - uses: manticoresoftware/[email protected].5
         with:
           image: ${{ matrix.image }}
           test_prefix: test/clt-tests/installation/rhel-dev-
@@ -219,7 +219,7 @@ jobs:
         image: [ "ubuntu:bionic", "ubuntu:focal", "ubuntu:jammy", "debian:buster", "debian:bullseye", "debian:bookworm" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/[email protected].3
+      - uses: manticoresoftware/[email protected].5
         with:
           image: ${{ matrix.image }}
           test_prefix: test/clt-tests/installation/deb-dev-

.github/workflows/test_template.yml

@@ -130,4 +130,4 @@ jobs:
           files: build/junit*.xml
           compare_to_earlier_commit: false
           comment_mode: failures
-  
+  

.github/workflows/win_test_template.yml

@@ -77,7 +77,7 @@ jobs:
         id: test
         # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979
         # Add -VV to ctest to display extra debug info
-        run: ctest -V -S misc/ctest/gltest.cmake --no-compress-output --timeout 600
+        run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600
         continue-on-error: true
       - name: Remember status
         if: always()
@@ -96,4 +96,4 @@ jobs:
         uses: manticoresoftware/upload_artifact_with_retries@v4
         with:
           name: ${{ inputs.artifact_name }}
-          path: "build/xml* build/test_*/test_*/report* build/test_*/error*.txt build/test_*/*log build/status*"
+          path: "build/xml* build/test_*/test_*/report* build/test_*/error*.txt build/test_*/*log build/status*"