refactor integration test & add e2e test

StrikeW · StrikeW · commit 3863a2398b10 · 2024-02-05T17:26:16.000+08:00
diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
@@ -210,3 +210,43 @@ services:
       interval: 5s
       timeout: 5s
       retries: 5
+
+  mongodb:
+    image: mongo:4.4
+    container_name: mongodb
+    ports:
+      - "27017:27017"
+    command: --replSet rs0 --oplogSize 128
+    restart: always
+    healthcheck:
+      test: "echo 'db.runCommand({ping: 1})' | mongo"
+      interval: 5s
+      timeout: 10s
+      retries: 3
+
+  mongodb-setup:
+    image: mongo:4.4
+    container_name: mongodb-setup
+    depends_on:
+      - mongodb
+    entrypoint:
+      [
+        "bash",
+        "-c",
+        "sleep 10 && mongo --host mongodb:27017 /config-replica.js && sleep 10"
+      ]
+    restart: "no"
+    volumes:
+      - ./mongodb/config-replica.js:/config-replica.js
+
+  mongo_data_generator:
+      build:
+        context: .
+        dockerfile: ./mongodb/Dockerfile.generator
+      container_name: mongo_data_generator
+      depends_on:
+        - mongodb
+      environment:
+        MONGO_HOST: mongodb
+        MONGO_PORT: 27017
+        MONGO_DB_NAME: random_data
diff --git a/ci/mongodb/Dockerfile.generator b/ci/mongodb/Dockerfile.generator
@@ -0,0 +1,23 @@
+# Use an official Python runtime as a parent image
+FROM python:3.8-slim
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy the requirements file into the container at /app
+COPY ./mongodb/requirements.txt /app
+COPY ./mongodb/app.py /app
+
+# Install any needed packages specified in requirements.txt
+RUN pip install -r requirements.txt
+
+# Make port 5000 available to the world outside this container
+EXPOSE 5000
+
+# Define environment variables
+ENV MONGO_HOST mongodb
+ENV MONGO_PORT 27017
+ENV MONGO_DB_NAME random_data
+
+# Run app.py when the container launches
+CMD ["python", "app.py"]
diff --git a/ci/mongodb/app.py b/ci/mongodb/app.py
diff --git a/ci/mongodb/config-replica.js b/ci/mongodb/config-replica.js
@@ -0,0 +1,6 @@
+rsconf = {
+    _id: "rs0",
+    members: [{ _id: 0, host: "mongodb:27017", priority: 1.0 }],
+};
+rs.initiate(rsconf);
+rs.status();
diff --git a/ci/mongodb/requirements.txt b/ci/mongodb/requirements.txt
diff --git a/e2e_test/source/cdc/mongodb/mongo_basic.slt b/e2e_test/source/cdc/mongodb/mongo_basic.slt
@@ -0,0 +1,25 @@
+# CDC source basic test
+control substitution on
+
+statement ok
+CREATE TABLE users (_id JSONB PRIMARY KEY, payload JSONB) WITH (
+  connector = 'mongodb-cdc',
+  mongodb.url = 'mongodb://mongodb:27017/?replicaSet=rs0',
+  collection.name = 'random_data.*'
+);
+
+statement ok
+CREATE MATERIALIZED VIEW normalized_users AS
+SELECT
+    payload ->> 'name' as name,
+    payload ->> 'email' as email,
+    payload ->> 'address' as address
+FROM
+    users;
+
+sleep 5s;
+
+query I
+select count(*) from normalized_users;
+----
+1000
diff --git a/integration_tests/debezium-mongo/docker-compose.yaml b/integration_tests/debezium-mongo/docker-compose.yaml
@@ -28,32 +28,12 @@ services:
       service: message_queue
 
   mongodb:
-    image: mongo:4.4
-    container_name: mongodb
-    ports:
-      - "27017:27017"
-    command: --replSet rs0 --oplogSize 128
-    restart: always
-    healthcheck:
-      test: "echo 'db.runCommand({ping: 1})' | mongo"
-      interval: 5s
-      timeout: 10s
-      retries: 3
+    extends: ../mongodb/docker-compose.yaml
+    service: mongodb
 
   mongodb-setup:
-    image: mongo:4.4
-    container_name: mongodb-setup
-    depends_on:
-      - mongodb
-    entrypoint:
-      [
-        "bash",
-        "-c",
-        "sleep 10 && mongo --host mongodb:27017 /config-replica.js && sleep 10"
-      ]
-    restart: "no"
-    volumes:
-      - ./config-replica.js:/config-replica.js
+    extends: ../mongodb/docker-compose.yaml
+    service: mongodb-setup
 
   debezium:
     image: debezium/connect:1.9
@@ -78,16 +58,9 @@ services:
       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://message_queue:8081
 
   random_data_generator:
-    build:
-      context: .
-      dockerfile: Dockerfile.generator
-    container_name: random_data_generator
-    depends_on:
-      - mongodb
-    environment:
-      MONGO_HOST: mongodb
-      MONGO_PORT: 27017
-      MONGO_DB_NAME: random_data
+    extends: ../mongodb/docker-compose.yaml
+    service: random_data_generator
+
 
   register-mongodb-connector:
     image: curlimages/curl:7.79.1
diff --git a/integration_tests/mongodb-cdc/docker-compose.yaml b/integration_tests/mongodb-cdc/docker-compose.yaml
@@ -2,41 +2,14 @@ version: '3.1'
 
 services:
   mongodb:
-    image: mongodb/mongodb-community-server:4.4.23-ubi8
-    container_name: mongodb
-    ports:
-      - "27017:27017"
-    command: --replSet rs0 --oplogSize 128
-    restart: always
-    healthcheck:
-      test: "echo 'db.runCommand({ping: 1})' | mongo"
-      interval: 5s
-      timeout: 10s
-      retries: 3
+    extends: ../mongodb/docker-compose.yaml
+    service: mongodb
 
   mongodb-setup:
-    image: mongodb/mongodb-community-server:4.4.23-ubi8
-    container_name: mongodb-setup
-    depends_on:
-      - mongodb
-    entrypoint:
-      [
-        "bash",
-        "-c",
-        "sleep 10 && mongo --host mongodb:27017 /config-replica.js && sleep 10"
-      ]
-    restart: "no"
-    volumes:
-      - ./config-replica.js:/config-replica.js
+    extends: ../mongodb/docker-compose.yaml
+    service: mongodb-setup
 
   random_data_generator:
-    build:
-      context: .
-      dockerfile: Dockerfile.generator
-    container_name: random_data_generator
-    depends_on:
-      - mongodb
-    environment:
-      MONGO_HOST: mongodb
-      MONGO_PORT: 27017
-      MONGO_DB_NAME: random_data
+    extends: ../mongodb/docker-compose.yaml
+    service: random_data_generator
+
diff --git a/integration_tests/mongodb/Dockerfile.generator b/integration_tests/mongodb/Dockerfile.generator
diff --git a/integration_tests/mongodb/app.py b/integration_tests/mongodb/app.py
@@ -0,0 +1,41 @@
+import os
+import pymongo
+from faker import Faker
+
+# To check the data through mongosh or mongo, run the following command:
+#  > mongosh mongodb://admin:admin123@127.0.0.1:27017
+#  > rs0 [direct: primary] test> use random_data
+#  > rs0 [direct: primary] random_data> db.users.find()
+#  > rs0 [direct: primary] random_data> db.users.count()
+
+# Connect to MongoDB
+mongo_host = os.environ["MONGO_HOST"]
+mongo_port = os.environ["MONGO_PORT"]
+mongo_db_name = os.environ["MONGO_DB_NAME"]
+
+url = f"mongodb://{mongo_host}:{mongo_port}"
+client = pymongo.MongoClient(url)
+db = client[mongo_db_name]
+
+# Generate random data
+fake = Faker()
+collection = db["users"]
+
+for _ in range(1000):
+    user_data = {
+        "name": fake.name(),
+        "address": fake.address(),
+        "email": fake.email(),
+    }
+    collection.insert_one(user_data)
+
+# Count the number of records in the collection
+total_records = collection.count_documents({})
+
+# Close the MongoDB connection
+client.close()
+print(f"Random data generated and inserted into MongoDB: {url}")
+
+# Print insertion summary
+print("Insertion summary:")
+print(f"Total records in the collection: {total_records}")
diff --git a/integration_tests/mongodb/config-replica.js b/integration_tests/mongodb/config-replica.js
diff --git a/integration_tests/mongodb/docker-compose.yaml b/integration_tests/mongodb/docker-compose.yaml
@@ -0,0 +1,42 @@
+version: '3.1'
+
+services:
+  mongodb:
+    image: mongo:4.4
+    container_name: mongodb
+    ports:
+      - "27017:27017"
+    command: --replSet rs0 --oplogSize 128
+    restart: always
+    healthcheck:
+      test: "echo 'db.runCommand({ping: 1})' | mongo"
+      interval: 5s
+      timeout: 10s
+      retries: 3
+
+  mongodb-setup:
+    image: mongo:4.4
+    container_name: mongodb-setup
+    depends_on:
+      - mongodb
+    entrypoint:
+      [
+        "bash",
+        "-c",
+        "sleep 10 && mongo --host mongodb:27017 /config-replica.js && sleep 10"
+      ]
+    restart: "no"
+    volumes:
+      - ./config-replica.js:/config-replica.js
+
+  random_data_generator:
+    build:
+      context: .
+      dockerfile: Dockerfile.generator
+    container_name: random_data_generator
+    depends_on:
+      - mongodb
+    environment:
+      MONGO_HOST: mongodb
+      MONGO_PORT: 27017
+      MONGO_DB_NAME: random_data
diff --git a/integration_tests/mongodb/requirements.txt b/integration_tests/mongodb/requirements.txt
@@ -0,0 +1,2 @@
+pymongo
+Faker
diff --git a/java/connector-node/risingwave-source-test/src/test/java/com/risingwave/connector/source/MongoDbSourceTest.java b/java/connector-node/risingwave-source-test/src/test/java/com/risingwave/connector/source/MongoDbSourceTest.java
@@ -14,31 +14,18 @@
 
 package com.risingwave.connector.source;
 
-import com.mongodb.client.MongoClient;
-import com.mongodb.client.MongoClients;
-import com.mongodb.client.MongoCollection;
-import com.mongodb.client.MongoDatabase;
-import com.mongodb.client.model.InsertManyOptions;
 import com.risingwave.connector.ConnectorServiceImpl;
 import com.risingwave.proto.ConnectorServiceProto;
 import io.grpc.*;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Executors;
-import org.bson.Document;
-import org.bson.types.ObjectId;
 import org.junit.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.MongoDBContainer;
-
-import static java.lang.Thread.sleep;
 
 public class MongoDbSourceTest {
     private static final Logger LOG = LoggerFactory.getLogger(MongoDbSourceTest.class.getName());
