feat(udf): support embedded python udf

Cargo.toml

@@ -136,6 +136,7 @@ arrow-ord = "50"
 arrow-row = "50"
 arrow-udf-js = "0.1"
 arrow-udf-wasm = { version = "0.1.2", features = ["build"] }
+arrow-udf-python = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "0479980" }
 arrow-array-deltalake = { package = "arrow-array", version = "48.0.1" }
 arrow-buffer-deltalake = { package = "arrow-buffer", version = "48.0.1" }
 arrow-cast-deltalake = { package = "arrow-cast", version = "48.0.1" }

ci/Dockerfile

@@ -11,10 +11,16 @@ ENV LANG en_US.utf8
 # Use AWS ubuntu mirror
 RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://us-east-2.ec2.archive.ubuntu.com/ubuntu/|g' /etc/apt/sources.list
 RUN apt-get update -yy && \
-    DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel python3 python3-pip \
+    DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel python3 python3-pip software-properties-common \
     openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld postgresql-client kafkacat netcat mysql-client \
     maven zstd libzstd-dev locales -yy \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/
+# Install Python 3.12
+RUN add-apt-repository ppa:deadsnakes/ppa -y && \
+    apt-get update -yy && \
+    DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy && \
+    rm -rf /var/lib/{apt,dpkg,cache,log}/
+ENV PYO3_PYTHON=python3.12
 
 SHELL ["/bin/bash", "-c"]
 

ci/build-ci-image.sh

@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240213
+export BUILD_ENV_VERSION=v20240223
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223
     depends_on:
       - mysql
       - db
@@ -81,7 +81,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223
     depends_on:
       - mysql
       - db
@@ -93,12 +93,12 @@ services:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -109,7 +109,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223
     depends_on:
       db:
         condition: service_healthy

ci/scripts/run-e2e-test.sh

@@ -82,25 +82,27 @@ sqllogictest -p 4566 -d test './e2e_test/database/test.slt'
 echo "--- e2e, $mode, Apache Superset"
 sqllogictest -p 4566 -d dev './e2e_test/superset/*.slt' --junit "batch-${profile}"
 
-echo "--- e2e, $mode, python udf"
+echo "--- e2e, $mode, external python udf"
 python3 e2e_test/udf/test.py &
 sleep 1
-sqllogictest -p 4566 -d dev './e2e_test/udf/udf.slt'
+sqllogictest -p 4566 -d dev './e2e_test/udf/external_udf.slt'
 pkill python3
 
 sqllogictest -p 4566 -d dev './e2e_test/udf/alter_function.slt'
 sqllogictest -p 4566 -d dev './e2e_test/udf/graceful_shutdown_python.slt'
 # FIXME: flaky test
 # sqllogictest -p 4566 -d dev './e2e_test/udf/retry_python.slt'
 
-echo "--- e2e, $mode, java udf"
+echo "--- e2e, $mode, external java udf"
 java -jar risingwave-udf-example.jar &
 sleep 1
-sqllogictest -p 4566 -d dev './e2e_test/udf/udf.slt'
+sqllogictest -p 4566 -d dev './e2e_test/udf/external_udf.slt'
 pkill java
 
-echo "--- e2e, $mode, wasm udf"
+echo "--- e2e, $mode, embedded udf"
 sqllogictest -p 4566 -d dev './e2e_test/udf/wasm_udf.slt'
+sqllogictest -p 4566 -d dev './e2e_test/udf/js_udf.slt'
+sqllogictest -p 4566 -d dev './e2e_test/udf/python_udf.slt'
 
 echo "--- Kill cluster"
 cluster_stop