Merge branch 'main' into murphy_ref_exec_mem_limit

Signed-off-by: Murphy <[email protected]>

Author: murphyatwork

.github/PULL_REQUEST_TEMPLATE.md

@@ -32,9 +32,9 @@ If yes, please specify the type of change:
 - [ ] I have added test cases for my bug fix or my new feature
 - [ ] This pr needs user documentation (for new or modified features or behaviors)
   - [ ] I have added documentation for my new feature or new function
+- [ ] This is a backport pr
 
 ## Bugfix cherry-pick branch check:
-
 - [ ] I have checked the version labels which the pr will be auto-backported to the target branch
   - [ ] 3.2
   - [ ] 3.1

.github/workflows/ci-doc-checker.yml

@@ -86,7 +86,7 @@ jobs:
 
       - uses: articulate/[email protected]
         with:
-          config: "docs/.markdownlint.yaml"
+          config: "docs/.markdownlint.json"
           files: 'docs/**/*.md'
           ignore: node_modules
           version: 0.28.1

.github/workflows/inspection-pipeline.yml

@@ -7,10 +7,19 @@ on:
     - cron: "30 8 * * 1-5"
     - cron: "30 10 * * 1-5"
   workflow_dispatch:
-
-concurrency:
-  group: 'INSPECTION-MAIN'
-  cancel-in-progress: false
+    inputs:
+      BRANCH:
+        description: 'BRANCH'
+        required: true
+        type: string
+      COMMIT_ID:
+        description: 'COMMIT ID'
+        required: true
+        type: string
+      TAR_PATH:
+        description: 'TAR PATH'
+        required: true
+        type: string
 
 permissions:
   checks: write
@@ -30,7 +39,7 @@ jobs:
 
   be-ut:
     runs-on: [self-hosted, normal]
-    if: github.repository == 'StarRocks/starrocks'
+    if: github.event_name == 'schedule' && github.repository == 'StarRocks/starrocks'
     timeout-minutes: 180
     name: BE UT
     env:
@@ -68,7 +77,7 @@ jobs:
   fe-ut:
     runs-on: [self-hosted, normal]
     name: FE UT
-    if: github.repository == 'StarRocks/starrocks'
+    if: github.event_name == 'schedule' && github.repository == 'StarRocks/starrocks'
     env:
       BRANCH: 'main'
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -147,7 +156,7 @@ jobs:
   build:
     runs-on: [self-hosted, normal]
     name: BUILD
-    if: github.repository == 'StarRocks/starrocks'
+    if: github.event_name == 'schedule' && github.repository == 'StarRocks/starrocks'
     env:
       BRANCH: 'main'
     outputs:
@@ -187,7 +196,10 @@ jobs:
 
   deploy:
     runs-on: [self-hosted, normal]
-    if: always() && needs.build.result == 'success'
+    if: >
+      always() && 
+      ((github.event_name == 'schedule' && needs.build.result == 'success') ||
+      github.event_name != 'schedule')
     needs: build
     name: DEPLOY SR
     env:
@@ -198,12 +210,31 @@ jobs:
       fe: ${{steps.deploy_sr.outputs.fe}}
       be: ${{steps.deploy_sr.outputs.be}}
       deploy_conf_file: ${{steps.apply_resource.outputs.deploy_conf_file}}
+      pr_number: ${{steps.init_params.outputs.PR_NUMBER}}
+      base_version: ${{steps.init_params.outputs.BASE_VERSION}}
+      branch: ${{steps.init_params.outputs.BRANCH}}
     steps:
       - name: Clean Workspace
         uses: AutoModality/[email protected]
 
+      - name: init params
+        id: init_params
+        run: |
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "PR_NUMBER=${{ needs.build.outputs.pr_number }}" >> $GITHUB_OUTPUT
+            echo "BRANCH=main" >> $GITHUB_OUTPUT
+            echo "BASE_VERSION=${{ needs.build.outputs.base_version }}" >> $GITHUB_OUTPUT
+          else
+            echo "PR_NUMBER=${{ inputs.COMMIT_ID }}" >> $GITHUB_OUTPUT
+            echo "BRANCH=${{ inputs.BRANCH }}" >> $GITHUB_OUTPUT
+            echo "TAR_PATH=${{ inputs.TAR_PATH }}" >> $GITHUB_OUTPUT
+            echo "BASE_VERSION=${{ inputs.COMMIT_ID }}" >> $GITHUB_OUTPUT
+          fi
+
       - name: Apply for resources
         id: apply_resource
+        env:
+          PR_NUMBER: ${{ steps.init_params.outputs.PR_NUMBER }}
         run: |
           rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
           ./bin/elastic-cluster.sh --template ci-admit
@@ -212,13 +243,24 @@ jobs:
 
       - name: Deploy SR
         id: deploy_sr
+        env:
+          PR_NUMBER: ${{ steps.init_params.outputs.PR_NUMBER }}
+          BRANCH: ${{ steps.init_params.outputs.BRANCH }}
+          TAR_PATH: ${{ steps.init_params.outputs.TAR_PATH }}
         run: |
           cd ci-tool && source lib/init.sh
-          repo="${{ github.repository }}"
-          bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
-          tar_path="oss://${bucket_prefix}-ci-release/$BRANCH/Release/pr/${{needs.build.outputs.build_output_tar}}"
+          
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            repo="${{ github.repository }}"
+            bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
+            tar_path="oss://${bucket_prefix}-ci-release/$BRANCH/Release/pr/${{needs.build.outputs.build_output_tar}}"
+            with_cov="--with-coverage"
+          else
+            tar_path=${TAR_PATH}
+          fi
+          
           echo ${tar_path}
-          ./bin/deploy-cluster.sh -c ${CLUSTER_NAME} --output ${tar_path} --with-coverage
+          ./bin/deploy-cluster.sh -c ${CLUSTER_NAME} --output ${tar_path} ${with_cov}
 
       - run: |
           echo "FE: ${{ steps.deploy_sr.outputs.fe }}, BE: ${{ steps.deploy_sr.outputs.be }}"
@@ -231,16 +273,16 @@ jobs:
   SQL-Tester:
     runs-on: [self-hosted, normal]
     name: SQL-Tester
-    needs: [ build, deploy ]
+    needs: [ deploy ]
     if: always() && needs.deploy.result == 'success'
     timeout-minutes: 60
     env:
-      PR_NUMBER: ${{ needs.build.outputs.pr_number }}
+      PR_NUMBER: ${{ needs.deploy.outputs.pr_number }}
       CLUSTER_NAME: ci-admit
       FE_NODE: ${{ needs.deploy.outputs.fe }}
       BE_NODE: ${{ needs.deploy.outputs.be }}
-      BASE_VERSION: ${{needs.build.outputs.base_version}}
-      BRANCH: 'main'
+      BASE_VERSION: ${{needs.deploy.outputs.base_version}}
+      BRANCH: ${{needs.deploy.outputs.branch}}
     outputs:
       MYSQL_ECI_ID: ${{ steps.run_case.outputs.MYSQL_ECI_ID }}
     steps:
@@ -306,7 +348,7 @@ jobs:
     runs-on: [self-hosted, normal]
     name: ADMIT TEST
     needs: [ build, deploy, SQL-Tester ]
-    if: always() && needs.deploy.result == 'success'
+    if: always() && github.event_name == 'schedule' && needs.deploy.result == 'success'
     timeout-minutes: 40
     env:
       PR_NUMBER: ${{ needs.build.outputs.pr_number }}
@@ -390,7 +432,7 @@ jobs:
           ./bin/elastic-cluster.sh --renew 0.5
 
       - name: save unstable cases
-        if: always()
+        if: always() && github.event_name == 'schedule'
         run: |
           cd ci-tool && source lib/init.sh
           ./bin/save_unstable_cases.sh

.github/workflows/pr-checker.yml

@@ -42,16 +42,55 @@ jobs:
     steps:
       - run: echo "Normal PR."
 
+      - name: backport assign
+        if: startsWith(github.head_ref, 'mergify/bp/') && github.event.action == 'opened'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.number }}
+          REPO: ${{ github.repository }}
+        run: |
+          ORI_PR=$(echo "${{ github.event.pull_request.title }}" | grep -oP '\(backport #\K\d+' | tail -n 1)
+          author=$(gh pr view ${ORI_PR} -R ${REPO} --json author -q '.author.login')
+          if [[ ! "${author}" =~ "mergify" ]]; then
+            gh pr edit ${PR_NUMBER} -R ${REPO} --add-assignee ${author}
+          fi
+
   title-check:
     runs-on: ubuntu-latest
     needs: sync-checker
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      PR_NUMBER: ${{ github.event.number }}
+      REPO: ${{ github.repository }}
     steps:
       - uses: thehanimo/[email protected]
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           pass_on_octokit_error: false
           configuration_path: ".github/pr-title-checker-config.json"
 
+      - name: update body
+        if: always() && github.base_ref != 'main' && github.event.action == 'opened'
+        env:
+          GH_TOKEN: ${{ secrets.PAT }}
+        run: |
+          gh pr view ${PR_NUMBER} -R ${REPO} --json body -q .body > body.txt
+          ori_body=$(cat body.txt)
+          sed -i "s#\[ \] This is a backport pr#\[x\] This is a backport pr#g" body.txt
+          if [[ "${ori_body}" != $(cat body.txt) ]]; then
+            gh pr edit ${PR_NUMBER} -R ${REPO} -F body.txt
+          fi
+
+      - name: check backport pr's title
+        if: github.base_ref != 'main' && contains(toJson(github.event.pull_request.body), '[x] This is a backport pr')
+        run: |
+          PR_TITLE=$(echo "${{ github.event.pull_request.title }}")
+          count=$(echo $PR_TITLE | grep -E '\(backport #[0-9]+)$' | wc -l)
+          if [[ $count -le 0 ]]; then
+            echo "::error::Backport PR title is not valid. It should end with '(backport #[0-9]+)'"
+            exit 1
+          fi
+
   backport-check:
     needs: title-check
     runs-on: ubuntu-latest
@@ -134,7 +173,9 @@ jobs:
           (!contains(toJson(github.event.pull_request.body), '[ ] No, this PR will not result in a change in behavior') &&
           !contains(toJson(github.event.pull_request.body), '[x] No, this PR will not result in a change in behavior')) ||
           (contains(toJson(github.event.pull_request.body), '[x] Yes, this PR will result in a change in behavior') &&
-          contains(toJson(github.event.pull_request.body), '[x] No, this PR will not result in a change in behavior'))
+          contains(toJson(github.event.pull_request.body), '[x] No, this PR will not result in a change in behavior')) ||
+          (contains(toJson(github.event.pull_request.body), '[ ] Yes, this PR will result in a change in behavior') &&
+          contains(toJson(github.event.pull_request.body), '[ ] No, this PR will not result in a change in behavior'))
         run: |
           echo "::error::Please check the format of your pr body about the behavior change checkbox!"
           exit 1
@@ -193,7 +234,7 @@ jobs:
         with:
           token: ${{ secrets.PAT }}
           pull-request-number: ${{ github.event.pull_request.number }}
-          merge-method: rebase
+          merge-method: squash
 
   changelist-check:
     runs-on: [self-hosted, normal]

README.md

@@ -106,7 +106,7 @@ We welcome all kinds of contributions from the community, individuals and partne
 
 1. See [Contributing.md](https://github.com/StarRocks/starrocks/blob/main/CONTRIBUTING.md) to get started.
 2. Set up StarRocks development environment:
-* [IDE Setup](https://docs.starrocks.io/en-us/main/developers/development-environment/ide-setup) 
+* [IDE Setup](https://docs.starrocks.io/docs/developers/development-environment/ide-setup/) 
 3. Understand our [GitHub workflow](https://github.com/StarRocks/community/blob/main/Contributors/guide/workflow.md) for opening a pull request; use this [PR Template](https://github.com/StarRocks/starrocks/blob/main/.github/PULL_REQUEST_TEMPLATE.md) when submitting a pull request.
 4. Pick a [good first issue](https://github.com/StarRocks/starrocks/labels/good%20first%20issue) and start contributing. 
 
@@ -121,6 +121,6 @@ We welcome all kinds of contributions from the community, individuals and partne
 This project is used by the following companies. Learn more about their use cases:
 
 - [Airbnb](https://www.youtube.com/watch?v=AzDxEZuMBwM&ab_channel=StarRocks_labs)
+- [Tencent](https://www.starrocks.io/blog/tencent-unifies-their-gaming-analytics-with-starrocks)
 - [Trip.com](https://starrocks.medium.com/trip-com-starrocks-efficiently-supports-high-concurrent-queries-dramatically-reduces-labor-and-1e1921dd6bf8) 
-- [Zepp Health](https://starrocks.io/blog/zeppheath) 
-- [Lenovo](https://starrocks.io/blog/lenovo_en) 
+- [Didi](https://www.starrocks.io/blog/reduced-80-cost-didis-journey-from-multiple-olap-engines-to-starrocks) 

be/src/agent/CMakeLists.txt

@@ -23,7 +23,6 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/agent")
 
 add_library(Agent STATIC
         agent_server.cpp
-        client_cache.cpp
         heartbeat_server.cpp
         task_worker_pool.cpp
         utils.cpp

be/src/agent/agent_server.cpp

@@ -158,7 +158,7 @@ void AgentServer::Impl::init_or_die() {
 // But it seems that there's no limit for the number of tablets of a partition.
 // Since a large queue size brings a little overhead, a big one is chosen here.
 #ifdef BE_TEST
-        BUILD_DYNAMIC_TASK_THREAD_POOL("publish_version", 1, 1, DEFAULT_DYNAMIC_THREAD_POOL_QUEUE_SIZE,
+        BUILD_DYNAMIC_TASK_THREAD_POOL("publish_version", 1, 3, DEFAULT_DYNAMIC_THREAD_POOL_QUEUE_SIZE,
                                        _thread_pool_publish_version);
 #else
         int max_publish_version_worker_count = config::transaction_publish_version_worker_count;
@@ -177,9 +177,8 @@ void AgentServer::Impl::init_or_die() {
         BUILD_DYNAMIC_TASK_THREAD_POOL("drop", 1, config::drop_tablet_worker_count, std::numeric_limits<int>::max(),
                                        _thread_pool_drop);
 
-        BUILD_DYNAMIC_TASK_THREAD_POOL("create_tablet", config::create_tablet_worker_count,
-                                       config::create_tablet_worker_count, std::numeric_limits<int>::max(),
-                                       _thread_pool_create_tablet);
+        BUILD_DYNAMIC_TASK_THREAD_POOL("create_tablet", 1, config::create_tablet_worker_count,
+                                       std::numeric_limits<int>::max(), _thread_pool_create_tablet);
 
         BUILD_DYNAMIC_TASK_THREAD_POOL("alter_tablet", 0, config::alter_tablet_worker_count,
                                        std::numeric_limits<int>::max(), _thread_pool_alter_tablet);

be/src/agent/agent_task.cpp

@@ -19,6 +19,7 @@
 #include "agent/task_signatures_manager.h"
 #include "boost/lexical_cast.hpp"
 #include "common/status.h"
+#include "io/io_profiler.h"
 #include "runtime/current_thread.h"
 #include "runtime/snapshot_loader.h"
 #include "service/backend_options.h"
@@ -305,6 +306,8 @@ void run_clone_task(const std::shared_ptr<CloneAgentTaskRequest>& agent_task_req
     const TCloneReq& clone_req = agent_task_req->task_req;
     AgentStatus status = STARROCKS_SUCCESS;
 
+    auto scope = IOProfiler::scope(IOProfiler::TAG_CLONE, clone_req.tablet_id);
+
     // Return result to fe
     TStatus task_status;
     TFinishTaskRequest finish_task_request;
@@ -377,6 +380,9 @@ void run_clone_task(const std::shared_ptr<CloneAgentTaskRequest>& agent_task_req
 void run_storage_medium_migrate_task(const std::shared_ptr<StorageMediumMigrateTaskRequest>& agent_task_req,
                                      ExecEnv* exec_env) {
     const TStorageMediumMigrateReq& storage_medium_migrate_req = agent_task_req->task_req;
+
+    auto scope = IOProfiler::scope(IOProfiler::TAG_CLONE, storage_medium_migrate_req.tablet_id);
+
     TStatusCode::type status_code = TStatusCode::OK;
     std::vector<std::string> error_msgs;
     TStatus task_status;

be/src/agent/finish_task.cpp

@@ -14,10 +14,10 @@
 
 #include "finish_task.h"
 
-#include "agent/client_cache.h"
 #include "agent/status.h"
 #include "agent/utils.h"
 #include "common/logging.h"
+#include "runtime/exec_env.h"
 #include "util/starrocks_metrics.h"
 
 namespace starrocks {
@@ -32,7 +32,7 @@ void finish_task(const TFinishTaskRequest& finish_task_request) {
     int32_t sleep_seconds = 1;
     int32_t max_retry_times = TASK_FINISH_MAX_RETRY;
 
-    MasterServerClient client(&g_frontend_service_client_cache);
+    MasterServerClient client(ExecEnv::GetInstance()->frontend_client_cache());
 
     while (try_time < max_retry_times) {
         StarRocksMetrics::instance()->finish_task_requests_total.increment(1);

be/src/agent/report_task.cpp

@@ -14,13 +14,13 @@
 
 #include "agent/report_task.h"
 
-#include "agent/client_cache.h"
 #include "agent/utils.h"
+#include "runtime/exec_env.h"
 
 namespace starrocks {
 
 AgentStatus report_task(const TReportRequest& request, TMasterResult* result) {
-    MasterServerClient client(&g_frontend_service_client_cache);
+    MasterServerClient client(ExecEnv::GetInstance()->frontend_client_cache());
     return client.report(request, result);
 }
 

be/src/block_cache/cache_options.h

@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <string>
 #include <vector>
 

be/src/column/binary_column.cpp

@@ -291,7 +291,7 @@ void BinaryColumnBase<T>::append_value_multiple_times(const void* value, size_t
 template <typename T>
 void BinaryColumnBase<T>::_build_slices() const {
     if constexpr (std::is_same_v<T, uint32_t>) {
-        CHECK_LT(_bytes.size(), (size_t)UINT32_MAX) << "BinaryColumn size overflow";
+        DCHECK_LT(_bytes.size(), (size_t)UINT32_MAX) << "BinaryColumn size overflow";
     }
 
     DCHECK(_offsets.size() > 0);