feat(test): test opendal fs engine on main cron with large dataset (risingwavelabs#8637)

Author: wcy-fdu

ci/scripts/s3-source-test-for-opendal-fs-engine.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source ci/scripts/common.env.sh
+
+while getopts 'p:s:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        s )
+            script=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+
+
+echo "--- Download artifacts"
+mkdir -p target/debug
+buildkite-agent artifact download risingwave-"$profile" target/debug/
+buildkite-agent artifact download risedev-dev-"$profile" target/debug/
+
+mv target/debug/risingwave-"$profile" target/debug/risingwave
+mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
+
+echo "--- Adjust permission"
+chmod +x ./target/debug/risingwave
+chmod +x ./target/debug/risedev-dev
+
+echo "--- Generate RiseDev CI config"
+cp ci/risedev-components.ci.env risedev-components.user.env
+
+echo "--- Prepare RiseDev dev cluster"
+cargo make pre-start-dev
+cargo make link-all-in-one-binaries
+
+echo "--- starting risingwave cluster with connector node"
+cargo make ci-start ci-3cn-3fe-opendal-fs-backend
+
+echo "--- Run test"
+python3 -m pip install minio psycopg2-binary
+python3 e2e_test/s3/$script.py
+
+echo "--- Kill cluster"
+rm -rf /tmp/rw_ci
+cargo make ci-kill

ci/workflows/main-cron.yml

@@ -215,6 +215,7 @@ steps:
             - S3_SOURCE_TEST_CONF
     timeout_in_minutes: 20
     retry: *auto-retry  
+
   - label: "S3 source check on AWS (csv parser)"
     command: "ci/scripts/s3-source-test.sh -p ci-release -s run_csv"
     depends_on: build
@@ -229,4 +230,20 @@ steps:
           environment:
             - S3_SOURCE_TEST_CONF
     timeout_in_minutes: 20
+    retry: *auto-retry  
+
+  - label: "S3 source on OpenDAL fs engine"
+    command: "ci/scripts/s3-source-test-for-opendal-fs-engine.sh -p ci-release -s run"
+    depends_on: build
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - S3_SOURCE_TEST_CONF
+    timeout_in_minutes: 20
     retry: *auto-retry  

risedev.yml

@@ -502,8 +502,8 @@ profile:
   ci-3cn-3fe-opendal-fs-backend:
     config-path: src/config/ci.toml
     steps:
-      # - use: etcd
-      #   unsafe-no-fsync: true
+      - use: etcd
+        unsafe-no-fsync: true
       - use: meta-node
       - use: opendal
         engine: fs