test: Add nexmark test, kafka test for backwards compatibility testing (risingwavelabs#10413)

Author: kwannoel

Makefile.toml

@@ -579,7 +579,53 @@ description = "Kill RisingWave dev cluster"
 script = '''
 #!/usr/bin/env bash
 
-tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d
+set -euo pipefail
+
+wait_kafka_exit() {
+  # Follow kafka-server-stop.sh
+  while [[ -n "$(ps ax | grep ' kafka\.Kafka ' | grep java | grep -v grep | awk '{print $1}')" ]]; do
+    echo "Waiting for kafka to exit"
+    sleep 1
+  done
+}
+
+wait_zookeeper_exit() {
+  # Follow zookeeper-server-stop.sh
+  while [[ -n "$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')" ]]; do
+    echo "Waiting for zookeeper to exit"
+    sleep 1
+  done
+}
+
+kill_kafka() {
+  ${PREFIX_BIN}/kafka/bin/kafka-server-stop.sh
+  wait_kafka_exit
+}
+
+kill_zookeeper() {
+  ${PREFIX_BIN}/kafka/bin/zookeeper-server-stop.sh
+  wait_zookeeper_exit
+}
+
+# Kill other components
+tmux list-windows -t risedev -F "#{window_name} #{pane_id}" \
+| grep -v 'kafka' \
+| grep -v 'zookeeper' \
+| awk '{ print $2 }' \
+| xargs -I {} tmux send-keys -t {} C-c C-d
+
+if [[ -n $(tmux list-windows -t risedev | grep kafka) ]];
+then
+  echo "kill kafka"
+  kill_kafka
+
+  echo "kill zookeeper"
+  kill_zookeeper
+
+  # Kill their tmux sessions
+  tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d
+fi
+
 tmux kill-session -t risedev
 test $? -eq 0 || { echo "Failed to stop all RiseDev components."; exit 1; }
 '''
@@ -1238,3 +1284,8 @@ cat << EOF > src/config/example.toml
 EOF
 cargo run -p risingwave_common --bin example-config >> src/config/example.toml
 '''
+
+[tasks.backwards-compat-test]
+category = "RiseDev - Backwards Compatibility Test"
+description = "Run backwards compatibility test"
+script = "./backwards-compat-tests/scripts/run_local.sh"

backwards-compat-tests/README.md

@@ -0,0 +1,18 @@
+# Backwards Compatibility Tests
+
+The backwards compatibility tests run in the following manner:
+1. Prepare old-cluster artifacts
+2. Configure the old-cluster.
+3. Start the old-cluster.
+4. Run DDL / DML / DQL.
+5. Stop the old-cluster.
+6. Prepare new-cluster artifacts.
+7. Configure the new-cluster.
+8. Start the new-cluster.
+9. Verify results of step 4.
+
+We currently cover the following:
+1. Basic mv
+2. Nexmark (on rw table not nexmark source)
+3. TPC-H
+4. Kafka Source

backwards-compat-tests/scripts/run_local.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ORIGINAL_BRANCH=$(git branch --show-current)
+
+on_exit() {
+  git checkout "$ORIGINAL_BRANCH"
+}
+
+trap on_exit EXIT
+
+source backwards-compat-tests/scripts/utils.sh
+
+configure_rw() {
+echo "--- Setting up cluster config"
+cat <<EOF > risedev-profiles.user.yml
+full-without-monitoring:
+  steps:
+    - use: minio
+    - use: etcd
+    - use: meta-node
+    - use: compute-node
+    - use: frontend
+    - use: compactor
+    - use: zookeeper
+    - use: kafka
+EOF
+
+cat <<EOF > risedev-components.user.env
+RISEDEV_CONFIGURED=false
+
+ENABLE_MINIO=true
+ENABLE_ETCD=true
+ENABLE_KAFKA=true
+
+# Fetch risingwave binary from release.
+ENABLE_BUILD_RUST=true
+
+# Ensure it will link the all-in-one binary from our release.
+ENABLE_ALL_IN_ONE=true
+
+# ENABLE_RELEASE_PROFILE=true
+EOF
+}
+
+setup_old_cluster() {
+  echo "--- Setting up old cluster"
+  git checkout "v${OLD_VERSION}-rc"
+}
+
+setup_new_cluster() {
+  echo "--- Setting up new cluster"
+  rm -r .risingwave/bin/risingwave
+  git checkout main
+}
+
+main() {
+  set -euo pipefail
+  get_rw_versions
+  setup_old_cluster
+  configure_rw
+  seed_old_cluster "$OLD_VERSION"
+
+  setup_new_cluster
+  configure_rw
+  validate_new_cluster "$NEW_VERSION"
+}
+
+main