Merge branch 'main' into cyx/sort-executor

Author: mergify[bot]

Cargo.lock

@@ -8,6 +8,7 @@ extend = [
   { path = "src/risedevtool/redis.toml" },
   { path = "src/risedevtool/risedev-components.toml" },
   { path = "src/frontend/planner_test/planner_test.toml" },
+  { path = "src/tests/compaction_test/Makefile.toml" },
 ]
 
 env_files = ["./risedev-components.user.env"]
@@ -447,15 +448,6 @@ cargo nextest run "$@"
 """
 description = "Run unit tests"
 
-[tasks.compaction-test]
-category = "RiseDev - Test"
-description = "Run hummock compaction e2e deterministic test"
-script = """
-#!/bin/bash
-set -e
-cargo run --bin compaction-test -- "$@"
-"""
-
 [tasks.sbuild]
 category = "RiseDev - Build in simulation mode"
 description = "Build in simulation mode"

Makefile.toml

@@ -6,4 +6,5 @@ set -euo pipefail
 source ci/scripts/common.env.sh
 export RUN_SQLSMITH=1; # fuzz tests
 export SQLSMITH_COUNT=1000; # fuzz with 1000 queries
+export RUN_COMPACTION=1;
 source ci/scripts/run-e2e-test.sh

ci/scripts/cron-e2e-test.sh

@@ -11,6 +11,9 @@ MATCHES="ci/scripts/pr.env.sh\
 CHANGED=$(git diff --name-only origin/main | grep "$MATCHES")
 set -e
 
+# Don't run e2e compaction test in PR build
+export RUN_COMPACTION=0;
+
 if [[ -n "$CHANGED" ]]; then
     echo "Changes to Sqlsmith source files detected.";
     export RUN_SQLSMITH=1;

ci/scripts/pr.env.sh

@@ -70,6 +70,35 @@ sqllogictest -p 4566 -d dev -e postgres-extended './e2e_test/extended_query/**/*
 echo "--- Kill cluster"
 cargo make ci-kill
 
+if [[ "$RUN_COMPACTION" -eq "1" ]]; then
+    echo "--- e2e, ci-compaction-test, nexmark_q7"
+    cargo make clean-data
+    cargo make ci-start ci-compaction-test
+    # Please make sure the regression is expected before increasing the timeout.
+    sqllogictest -p 4566 -d dev './e2e_test/compaction/ingest_rows.slt'
+
+    # We should ingest about 100 version deltas before the test
+    echo "--- Wait for data ingestion"
+    # Poll the current version id until we have around 100 version deltas
+    delta_log_cnt=0
+    while [ $delta_log_cnt -le 95 ]
+    do
+        delta_log_cnt="$(./risedev ctl hummock list-version | grep -w '^ *id:' | grep -o '[0-9]\+' | head -n 1)"
+        echo "Current version $delta_log_cnt"
+        sleep 5
+    done
+
+    echo "--- Pause source and disable commit new epochs"
+    ./risedev ctl meta pause
+    ./risedev ctl hummock disable-commit-epoch
+
+    echo "--- Start to run compaction test"
+    cargo run -r --bin compaction-test -- --ci-mode true --state-store hummock+minio://hummockadmin:[email protected]:9301/hummock001
+
+    echo "--- Kill cluster"
+    cargo make ci-kill
+fi
+
 if [[ "$RUN_SQLSMITH" -eq "1" ]]; then
     echo "--- e2e, ci-3cn-1fe, fuzzing"
     buildkite-agent artifact download sqlsmith-"$profile" target/debug/

ci/scripts/run-e2e-test.sh

@@ -21,15 +21,15 @@ auto-retry: &auto-retry
 
 steps:
   - label: "build"
-    command: "ci/scripts/build.sh -t ci-dev -p ci-dev"
+    command: "ci/scripts/build.sh -t ci-release -p ci-release"
     key: "build"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v3.9.0:
           run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
-    timeout_in_minutes: 10
+    timeout_in_minutes: 20
     retry: *auto-retry
 
   - label: "build (deterministic simulation)"
@@ -56,8 +56,8 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "end-to-end test"
-    command: "ci/scripts/cron-e2e-test.sh -p ci-dev"
+  - label: "end-to-end test (release mode)"
+    command: "ci/scripts/cron-e2e-test.sh -p ci-release"
     depends_on:
       - "build"
       - "docslt"
@@ -71,8 +71,8 @@ steps:
     timeout_in_minutes: 30
     retry: *auto-retry
 
-  - label: "end-to-end source test"
-    command: "ci/scripts/e2e-source-test.sh -p ci-dev"
+  - label: "end-to-end source test (release mode)"
+    command: "ci/scripts/e2e-source-test.sh -p ci-release"
     depends_on: "build"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache