Merge remote-tracking branch 'origin/main' into eric/unify_metrics_level

Author: Eric Fu

Cargo.lock

@@ -15,6 +15,7 @@ members = [
   "src/frontend",
   "src/frontend/planner_test",
   "src/java_binding",
+  "src/jni_core",
   "src/meta",
   "src/object_store",
   "src/prost",
@@ -113,6 +114,8 @@ risingwave_stream = { path = "./src/stream" }
 risingwave_test_runner = { path = "./src/test_runner" }
 risingwave_udf = { path = "./src/udf" }
 risingwave_variables = { path = "./src/utils/variables" }
+risingwave_java_binding = { path = "./src/java_binding" }
+risingwave_jni_core = { path = "src/jni_core" }
 
 [profile.dev]
 lto = 'off'

Cargo.toml

@@ -579,7 +579,53 @@ description = "Kill RisingWave dev cluster"
 script = '''
 #!/usr/bin/env bash
 
-tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d
+set -euo pipefail
+
+wait_kafka_exit() {
+  # Follow kafka-server-stop.sh
+  while [[ -n "$(ps ax | grep ' kafka\.Kafka ' | grep java | grep -v grep | awk '{print $1}')" ]]; do
+    echo "Waiting for kafka to exit"
+    sleep 1
+  done
+}
+
+wait_zookeeper_exit() {
+  # Follow zookeeper-server-stop.sh
+  while [[ -n "$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')" ]]; do
+    echo "Waiting for zookeeper to exit"
+    sleep 1
+  done
+}
+
+kill_kafka() {
+  ${PREFIX_BIN}/kafka/bin/kafka-server-stop.sh
+  wait_kafka_exit
+}
+
+kill_zookeeper() {
+  ${PREFIX_BIN}/kafka/bin/zookeeper-server-stop.sh
+  wait_zookeeper_exit
+}
+
+# Kill other components
+tmux list-windows -t risedev -F "#{window_name} #{pane_id}" \
+| grep -v 'kafka' \
+| grep -v 'zookeeper' \
+| awk '{ print $2 }' \
+| xargs -I {} tmux send-keys -t {} C-c C-d
+
+if [[ -n $(tmux list-windows -t risedev | grep kafka) ]];
+then
+  echo "kill kafka"
+  kill_kafka
+
+  echo "kill zookeeper"
+  kill_zookeeper
+
+  # Kill their tmux sessions
+  tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d
+fi
+
 tmux kill-session -t risedev
 test $? -eq 0 || { echo "Failed to stop all RiseDev components."; exit 1; }
 '''
@@ -1238,3 +1284,8 @@ cat << EOF > src/config/example.toml
 EOF
 cargo run -p risingwave_common --bin example-config >> src/config/example.toml
 '''
+
+[tasks.backwards-compat-test]
+category = "RiseDev - Backwards Compatibility Test"
+description = "Run backwards compatibility test"
+script = "./backwards-compat-tests/scripts/run_local.sh"

Makefile.toml

@@ -0,0 +1,18 @@
+# Backwards Compatibility Tests
+
+The backwards compatibility tests run in the following manner:
+1. Prepare old-cluster artifacts
+2. Configure the old-cluster.
+3. Start the old-cluster.
+4. Run DDL / DML / DQL.
+5. Stop the old-cluster.
+6. Prepare new-cluster artifacts.
+7. Configure the new-cluster.
+8. Start the new-cluster.
+9. Verify results of step 4.
+
+We currently cover the following:
+1. Basic mv
+2. Nexmark (on rw table not nexmark source)
+3. TPC-H
+4. Kafka Source