Merge branch 'main' into zl_clnstate

Author: mergify[bot]

Cargo.lock

@@ -31,7 +31,6 @@ members = [
   "src/tests/compaction_test",
   "src/tests/regress",
   "src/tests/simulation",
-  "src/tests/simulation_scale",
   "src/tests/sqlsmith",
   "src/tracing",
   "src/utils/async_stack_trace",

Cargo.toml

@@ -582,7 +582,7 @@ script = """
 set -e
 
 cargo nextest run \
-  -p risingwave_simulation_scale \
+  -p risingwave_simulation \
   "$@"
 """
 
@@ -596,7 +596,7 @@ script = """
 set -e
 
 cargo nextest archive \
-  -p risingwave_simulation_scale \
+  -p risingwave_simulation \
   --archive-file scale-test.tar.zst \
   "$@"
 """

Makefile.toml

@@ -5,10 +5,14 @@ set -euo pipefail
 
 source ci/scripts/common.env.sh
 
+# Should set a stable version of connector node
+STABLE_VERSION=e31eb0bf6e4f708ceadce846538fc6bd55978c59
+
 echo "--- Build Java connector node"
-# clone a released version(tag)
-git clone --branch v0.0.2 --depth 1 https://"$GITHUB_TOKEN"@github.com/risingwavelabs/risingwave-connector-node.git
+git clone --depth 1 https://"$GITHUB_TOKEN"@github.com/risingwavelabs/risingwave-connector-node.git
 cd risingwave-connector-node
+# checkout a stable version
+git checkout $STABLE_VERSION
 mvn package -Dmaven.test.skip=true
 echo "--- Upload Java artifacts"
 cp service/target/service-*.jar ./connector-service.jar

ci/scripts/build-other.sh

@@ -5,9 +5,12 @@ set -euo pipefail
 
 source ci/scripts/common.env.sh
 
-echo "--- Run clippy check"
+echo "--- Run clippy check (dev, all features)"
 cargo clippy --all-targets --all-features --locked -- -D warnings
 
+echo "--- Run clippy check (release)"
+cargo clippy --release  --all-targets --features "static-link static-log-level" --locked -- -D warnings
+
 echo "--- Build documentation"
 cargo doc --document-private-items --no-deps
 

ci/scripts/pre-unit-test.sh renamed to ci/scripts/check.sh

@@ -24,7 +24,7 @@ echo "--- deterministic simulation e2e, ci-3cn-1fe, batch"
 seq 16 | parallel MADSIM_TEST_SEED={} './risingwave_simulation ./e2e_test/batch/\*\*/\*.slt > $LOGDIR/batch-{}.log && rm $LOGDIR/batch-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-1fe, kafka source"
-seq 16 | parallel MADSIM_TEST_SEED={} './risingwave_simulation ./e2e_test/source/kafka.slt > $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log'
+seq 16 | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kafka-datadir=./scripts/source/test_data ./e2e_test/source/kafka.slt > $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, streaming"
 seq 16 | parallel MADSIM_TEST_SEED={} './risingwave_simulation -j 16 ./e2e_test/streaming/\*\*/\*.slt > $LOGDIR/parallel-streaming-{}.log && rm $LOGDIR/parallel-streaming-{}.log'

ci/scripts/deterministic-e2e-test.sh

@@ -12,7 +12,7 @@ export LOGDIR=.risingwave/log
 mkdir -p $LOGDIR
 
 echo "--- deterministic simulation e2e, ci-3cn-1fe, recovery, streaming"
-seq 1 | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill-meta --kill-frontend --kill-compute --kill-compactor --kill-rate=${KILL_RATE} ./e2e_test/streaming/\*\*/\*.slt > $LOGDIR/recovery-streaming-{}.log && rm $LOGDIR/recovery-streaming-{}.log'
+seq 16 | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/streaming/\*\*/\*.slt > $LOGDIR/recovery-streaming-{}.log && rm $LOGDIR/recovery-streaming-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-1fe, recovery, batch"
-seq 1 | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill-meta --kill-frontend --kill-compute --kill-compactor --kill-rate=${KILL_RATE} ./e2e_test/batch/\*\*/\*.slt > $LOGDIR/recovery-batch-{}.log && rm $LOGDIR/recovery-batch-{}.log'
+seq 16 | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/batch/\*\*/\*.slt > $LOGDIR/recovery-batch-{}.log && rm $LOGDIR/recovery-batch-{}.log'

ci/scripts/deterministic-recovery-test.sh

@@ -14,17 +14,15 @@ docker build -f docker/Dockerfile -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" -
 echo "--- docker images"
 docker images
 
-if [ "$PUSH" = true ]; then
-  echo "--- ghcr login"
-  echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
+echo "--- ghcr login"
+echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
 
-  echo "--- dockerhub login"
-  echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
+echo "--- dockerhub login"
+echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
 
-  echo "--- docker push to ghcr"
-  docker push "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}"
+echo "--- docker push to ghcr"
+docker push "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}"
 
-  echo "--- docker push to dockerhub"
-  docker tag "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" "${dockerhubaddr}:${BUILDKITE_COMMIT}-${arch}"
-  docker push "${dockerhubaddr}:${BUILDKITE_COMMIT}-${arch}"
-fi
+echo "--- docker push to dockerhub"
+docker tag "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" "${dockerhubaddr}:${BUILDKITE_COMMIT}-${arch}"
+docker push "${dockerhubaddr}:${BUILDKITE_COMMIT}-${arch}"

ci/scripts/docker.sh

@@ -56,15 +56,32 @@ apt-get -y install mysql-client
 # import data to mysql
 mysql --host=mysql --port=3306 -u root -p123456 < ./e2e_test/source/cdc/mysql_cdc.sql
 # start risingwave cluster
-cargo make ci-start ci-1cn-1fe
+cargo make ci-start ci-1cn-1fe-with-recovery
 # start cdc connector node
 nohup java -jar ./connector-service.jar --port 60061 > .risingwave/log/connector-source.log 2>&1 &
 sleep 1
 sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.load.slt'
 # wait for cdc loading
-sleep 4
+sleep 10
 sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check.slt'
 
+# kill cluster
+cargo make kill
+# start cluster w/o clean-data
+cargo make dev ci-1cn-1fe-with-recovery
+echo "wait for recovery finish"
+sleep 10
+echo "check mviews after cluster recovery"
+# check snapshot
+sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check.slt'
+# insert new rows
+mysql --host=mysql --port=3306 -u root -p123456 < ./e2e_test/source/cdc/mysql_cdc_insert.sql
+# wait cdc ingesting
+sleep 10
+# check new results
+sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check_new_rows.slt'
+
+
 echo "--- Kill cluster"
 pkill -f connector-service.jar
 cargo make ci-kill

ci/scripts/e2e-source-test.sh

@@ -12,25 +12,38 @@ echo "--- Install protoc3"
 curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
 unzip -o protoc-3.15.8-linux-x86_64.zip -d /usr/local bin/protoc
 
-echo "--- Install gh cli"
-yum install -y dnf
-dnf install -y 'dnf-command(config-manager)'
-dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
-dnf install -y gh
-
 echo "--- Install lld"
 yum install -y centos-release-scl-rh
 yum install -y llvm-toolset-7.0-lld
 source /opt/rh/llvm-toolset-7.0/enable
 
-echo "--- Release create"
-gh release create "${BUILDKITE_TAG}" --generate-notes -d -p
+echo "--- Install aws cli"
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip -q awscliv2.zip && ./aws/install && mv /usr/local/bin/aws /bin/aws
 
-echo "--- Build release asset"
+echo "--- Build release binary"
 cargo build -p risingwave_cmd_all --features "static-link static-log-level" --profile release
-cd target/release
-chmod +x risingwave
-tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz risingwave
+cd target/release && chmod +x risingwave
+
+echo "--- Upload nightly binary to s3"
+tar -czvf risingwave-"$(date '+%Y%m%d')"-x86_64-unknown-linux.tar.gz risingwave
+aws s3 cp risingwave-"$(date '+%Y%m%d')"-x86_64-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary
+
+if [[ -n "${BUILDKITE_TAG+x}" ]]; then
+  echo "--- Install gh cli"
+  yum install -y dnf
+  dnf install -y 'dnf-command(config-manager)'
+  dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
+  dnf install -y gh
+
+  echo "--- Release create"
+  gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p
+
+  echo "--- Release upload asset"
+  tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz risingwave
+  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz
+fi
+
+
+
 
-echo "--- Release upload asset"
-gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz

ci/scripts/release.sh

@@ -1,14 +1,3 @@
-projects:
-  - label: docker-pr
-    path: docker/
-    skip:
-      - docker-build-push*
-      - multi-arch-image-create-push
-  - label: docker-schedule-ui
-    path: .
-    skip:
-      - docker-build-pr
-
 auto-retry: &auto-retry
   automatic:
     - exit_status: -1  # Agent was lost
@@ -17,21 +6,9 @@ auto-retry: &auto-retry
       limit: 2
 
 steps:
-  - label: "docker-build-pr"
-    command: "ci/scripts/docker.sh"
-    if: build.source != "schedule"
-    env:
-      BUILDPIPE_SCOPE: project
-      PUSH: false
-    retry: *auto-retry
-
   - label: "docker-build-push: amd64"
     command: "ci/scripts/docker.sh"
     key: "build-amd64"
-    if: build.source == "schedule" || build.source == "ui"
-    env:
-      BUILDPIPE_SCOPE: project
-      PUSH: true
     plugins:
       - seek-oss/aws-sm#v2.3.1:
           env:
@@ -43,10 +20,6 @@ steps:
   - label: "docker-build-push: aarch64"
     command: "ci/scripts/docker.sh"
     key: "build-aarch64"
-    if: build.source == "schedule" || build.source == "ui"
-    env:
-      BUILDPIPE_SCOPE: project
-      PUSH: true
     plugins:
       - seek-oss/aws-sm#v2.3.1:
           env:
@@ -59,16 +32,21 @@ steps:
 
   - label: "multi-arch-image-create-push"
     command: "ci/scripts/multi-arch-docker.sh"
-    if: build.source == "schedule" || build.source == "ui"
     depends_on:
       - "build-amd64"
       - "build-aarch64"
-    env:
-      BUILDPIPE_SCOPE: project
     plugins:
       - seek-oss/aws-sm#v2.3.1:
           env:
             GHCR_USERNAME: ghcr-username
             GHCR_TOKEN: ghcr-token
             DOCKER_TOKEN: docker-token
+    retry: *auto-retry
+
+  - label: "pre build binary"
+    command: "ci/scripts/release.sh"
+    plugins:
+      - docker-compose#v3.9.0:
+          run: release-env
+          config: ci/docker-compose.yml
     retry: *auto-retry