Merge branch 'main' into wangzheng/serving_ctl

Author: zwang28

.github/workflows/connector-node-integration.yml

@@ -41,7 +41,7 @@ jobs:
           
           echo "--- build connector node"
           cd ${RISINGWAVE_ROOT}/java
-          mvn --batch-mode --update-snapshots clean package
+          mvn --batch-mode --update-snapshots clean package -Dno-build-rust
           
           echo "--- install postgresql client"
           sudo apt install postgresql postgresql-contrib libpq-dev

.github/workflows/typo.yml

@@ -10,4 +10,4 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Check spelling of the entire repository
-      uses: crate-ci/typos@v1.14.12
+      uses: crate-ci/typos@v1.15.0

.typos.toml

@@ -1,13 +1,16 @@
 [default.extend-words]
-indexs = "indices"
-Stichting = "Stichting"
-fo = "fo"
-FPR = "FPR"
+indexs = "indices"      # Both are valid, just pick one.
+Stichting = "Stichting" # This is Dutch for "Foundation". From DuckDB.
+FPR = "FPR"             # False Positive Rate
+inout = "inout"         # This is a SQL keyword!
+numer = "numer"         # numerator
+nd = "nd"               # N-dimentional / 2nd
+steam = "stream"        # You played with Steam games too much.
+# Some weird short variable names
 ot = "ot"
-inout = "inout"
-numer = "numer"
-nd = "nd"
-steam = "stream"
+bui = "bui"
+
+[default.extend-identifiers]
 
 [files]
 extend-exclude = [

Cargo.lock

@@ -78,14 +78,16 @@ hashbrown = { version = "0.13.2", features = ["ahash", "inline-more", "nightly"]
 lto = 'off'
 
 [profile.release]
-debug = "line-tables-only"
+debug = 1
 lto = 'thin'
 
 # The profile used for CI in main branch.
 # This profile inherits from the release profile, but turns on some checks and assertions for us to
 # better catch bugs in CI.
 [profile.ci-release]
 inherits = "release"
+incremental = false
+debug = "line-tables-only"
 debug-assertions = true
 overflow-checks = true
 

Cargo.toml

@@ -910,7 +910,7 @@ if [ $# -gt 0 ]; then
   ARGS=("$@")
 
   echo "Applying clippy --fix for $@ (including dirty and staged files)"
-  cargo clippy ${ARGS[@]/#/--package risingwave_} ${RISINGWAVE_FEATURE_FLAGS} --fix --allow-dirty --allow-staged
+  cargo clippy ${ARGS[@]/#/--package risingwave_} --fix --allow-dirty --allow-staged
 else
   echo "Applying clippy --fix for all targets to all files (including dirty and staged files)"
   echo "Tip: run $(tput setaf 4)./risedev cf {package_names}$(tput sgr0) to only check-fix those packages (e.g. frontend, meta)."
@@ -922,7 +922,7 @@ fi
 private = true
 category = "RiseDev - Check"
 description = "Run cargo typos-cli check"
-install_crate = { min_version = "1.14.8", crate_name = "typos-cli", binary = "typos", test_arg = [
+install_crate = { min_version = "1.15.0", crate_name = "typos-cli", binary = "typos", test_arg = [
   "--help",
 ], install_command = "binstall" }
 script = """

Makefile.toml

@@ -13,6 +13,6 @@ cd ..
 
 echo "--- Upload Java artifacts"
 cp java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz ./risingwave-connector.tar.gz
-cp java/udf/target/risingwave-udf-example.jar ./risingwave-udf-example.jar
+cp java/udf-example/target/risingwave-udf-example.jar ./risingwave-udf-example.jar
 buildkite-agent artifact upload ./risingwave-connector.tar.gz
 buildkite-agent artifact upload ./risingwave-udf-example.jar

ci/scripts/build-other.sh

@@ -52,13 +52,11 @@ cargo build \
     -p risingwave_sqlsmith \
     -p risingwave_compaction_test \
     -p risingwave_backup_cmd \
-    -p risingwave_java_binding \
     -p risingwave_e2e_extended_mode_test \
     $RISINGWAVE_FEATURE_FLAGS \
     --profile "$profile"
 
-# the file name suffix of artifact for risingwave_java_binding is so only for linux. It is dylib for MacOS
-artifacts=(risingwave sqlsmith compaction-test backup-restore risingwave_regress_test risingwave_e2e_extended_mode_test risedev-dev delete-range-test librisingwave_java_binding.so)
+artifacts=(risingwave sqlsmith compaction-test backup-restore risingwave_regress_test risingwave_e2e_extended_mode_test risedev-dev delete-range-test)
 
 echo "--- Show link info"
 ldd target/"$profile"/risingwave

ci/scripts/build.sh

@@ -86,23 +86,3 @@ function download_and_prepare_rw() {
   cargo make pre-start-dev
   cargo make --allow-private link-all-in-one-binaries
 }
-
-# Arguments:
-#   $1: cargo build `profile` of the binaries
-function download_java_binding() {
-  echo "--- Download java binding"
-  if [ -z "$1" ]; then
-    echo "download_java_binding: missing argument profile"
-    exit 1
-  fi
-
-  profile=$1
-
-  echo -e "\033[33mDownload artifacts\033[0m"
-
-  mkdir -p target/debug
-  download-and-decompress-artifact librisingwave_java_binding.so-"$profile" target/debug
-  mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so
-  
-  export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug
-}

ci/scripts/common.sh

@@ -23,10 +23,6 @@ shift $((OPTIND -1))
 
 download_and_prepare_rw "$profile" source
 
-download_java_binding "$profile"
-
-export RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT=stream_chunk
-
 echo "--- Download connector node package"
 buildkite-agent artifact download risingwave-connector.tar.gz ./
 mkdir ./connector-node

ci/scripts/e2e-iceberg-sink-test.sh

@@ -23,11 +23,6 @@ shift $((OPTIND -1))
 
 download_and_prepare_rw "$profile" source
 
-download_java_binding "$profile"
-
-# TODO: Switch to stream_chunk encoding once it's completed, and then remove json encoding as well as this env var.
-export RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT=stream_chunk
-
 # Change process number limit
 echo "--- os limits"
 ulimit -a

ci/scripts/e2e-sink-test.sh

@@ -26,8 +26,6 @@ shift $((OPTIND -1))
 
 download_and_prepare_rw "$profile" source
 
-download_java_binding "$profile"
-
 echo "--- Download connector node package"
 buildkite-agent artifact download risingwave-connector.tar.gz ./
 mkdir ./connector-node

ci/scripts/e2e-source-test.sh

@@ -23,13 +23,11 @@ shift $((OPTIND -1))
 
 download_and_prepare_rw "$profile" source
 
-download_java_binding "$profile"
-
 echo "--- starting risingwave cluster"
 cargo make ci-start java-binding-demo
 
 echo "--- Build java binding demo"
-cargo make build-java-binding-java
+cargo make build-java-binding
 
 echo "--- ingest data and run java binding"
 cargo make ingest-data-and-run-java-binding

ci/scripts/java-binding-test.sh

@@ -83,7 +83,7 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then
   gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz
 
   echo "--- Release build and upload risingwave connector node jar asset"
-  cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true
+  cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true
   cd connector-node/assembly/target && mv risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz
   gh release upload "${BUILDKITE_TAG}" risingwave-connector-"${BUILDKITE_TAG}".tar.gz
 fi

ci/scripts/release.sh

@@ -34,12 +34,11 @@ RUN rustup self update \
 
 RUN cargo fetch
 
-RUN cargo build -p risingwave_cmd_all -p risingwave_java_binding --release --features "rw-static-link" && \
+RUN cargo build -p risingwave_cmd_all --release --features "rw-static-link" && \
   mkdir -p /risingwave/bin && mv /risingwave/target/release/risingwave /risingwave/bin/ && \
-  mkdir -p /risingwave/lib && mv /risingwave/target/release/librisingwave_java_binding.so /risingwave/lib && \
-  cargo clean
+  mkdir -p /risingwave/lib && cargo clean
 
-RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true && \
+RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true && \
     mkdir -p /risingwave/bin/connector-node && \
     tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
 
@@ -52,9 +51,6 @@ RUN mkdir -p /risingwave/bin/connector-node && mkdir -p /risingwave/lib
 COPY --from=builder /risingwave/bin/risingwave /risingwave/bin/risingwave
 COPY --from=builder /risingwave/bin/connector-node /risingwave/bin/connector-node
 COPY --from=builder /risingwave/ui /risingwave/ui
-COPY --from=builder /risingwave/lib/librisingwave_java_binding.so /risingwave/lib/librisingwave_java_binding.so
-# Set java.library.path env to /risingwave/lib
-ENV RW_JAVA_BINDING_LIB_PATH /risingwave/lib
 # Set default playground mode to docker-playground profile
 ENV PLAYGROUND_PROFILE docker-playground
 # Set default dashboard UI to local path instead of github proxy

docker/Dockerfile

@@ -39,12 +39,11 @@ ARG JAVA_HOME_PATH
 ENV JAVA_HOME ${JAVA_HOME_PATH}
 ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH}
 
-RUN cargo build -p risingwave_cmd_all -p risingwave_java_binding --release --features "static-link static-log-level" && \
+RUN cargo build -p risingwave_cmd_all --release --features "static-link static-log-level" && \
   mkdir -p /risingwave/bin && mv /risingwave/target/release/risingwave /risingwave/bin/ && \
-  mkdir -p /risingwave/lib && mv /risingwave/target/release/librisingwave_java_binding.so /risingwave/lib && \
-  cargo clean
+  mkdir -p /risingwave/lib && cargo clean
 
-RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true && \
+RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true && \
     mkdir -p /risingwave/bin/connector-node && \
     tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
 
@@ -57,7 +56,6 @@ RUN mkdir -p /risingwave/bin/connector-node && mkdir -p /risingwave/lib
 COPY --from=builder /risingwave/bin/risingwave /risingwave/bin/risingwave
 COPY --from=builder /risingwave/bin/connector-node /risingwave/bin/connector-node
 COPY --from=builder /risingwave/ui /risingwave/ui
-COPY --from=builder /risingwave/lib/librisingwave_java_binding.so /risingwave/lib/librisingwave_java_binding.so
 
 # hadoop
 RUN wget https://rw-yufan.s3.ap-southeast-1.amazonaws.com/hadoop-2.7.3.tar.gz -P /root/
@@ -75,8 +73,6 @@ ARG JAVA_HOME_PATH
 ENV JAVA_HOME ${JAVA_HOME_PATH}
 ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH}
 
-# Set java.library.path env to /risingwave/lib
-ENV RW_JAVA_BINDING_LIB_PATH /risingwave/lib
 # Set default playground mode to docker-playground profile
 ENV PLAYGROUND_PROFILE docker-playground
 # Set default dashboard UI to local path instead of github proxy

docker/Dockerfile.hdfs

@@ -0,0 +1,33 @@
+statement error
+select p, percentile_cont(p) within group (order by x::float8)
+from generate_series(1,5) x,
+     (values (0::float8),(0.1),(0.25),(0.4),(0.5),(0.6),(0.75),(0.9),(1)) v(p)
+group by p order by p;
+
+statement error
+select percentile_cont(array[0,1,0.25,0.75,0.5,1,0.3,0.32,0.35,0.38,0.4]) within group (order by x)
+from generate_series(1,6) x;
+
+statement error
+select percentile_disc(array[0.25,0.5,0.75]) within group (order by x)
+from unnest('{fred,jim,fred,jack,jill,fred,jill,jim,jim,sheila,jim,sheila}'::text[]) u(x);
+
+statement error
+select pg_collation_for(percentile_disc(1) within group (order by x collate "POSIX"))
+  from (values ('fred'),('jim')) v(x);
+
+query RR
+select
+  percentile_cont(0.5) within group (order by a),
+  percentile_disc(0.5) within group (order by a)
+from (values(1::float8),(3),(5),(7)) t(a);
+----
+4 3
+
+query RR
+select
+  percentile_cont(0.25) within group (order by a),
+  percentile_disc(0.5) within group (order by a)
+from (values(1::float8),(3),(5),(7)) t(a);
+----
+2.5 3

docker/dashboards/risingwave-dev-dashboard.json

@@ -0,0 +1,14 @@
+query I
+select abs.abs from abs(-1);
+----
+1
+
+query I
+select alias.alias from abs(-1) alias;
+----
+1
+
+query I
+select alias.col from abs(-1) alias(col);
+----
+1