refactor integration test & add e2e test

Author: StrikeW

ci/docker-compose.yml

@@ -210,3 +210,43 @@ services:
       interval: 5s
       timeout: 5s
       retries: 5
+
+  mongodb:
+    image: mongo:4.4
+    container_name: mongodb
+    ports:
+      - "27017:27017"
+    command: --replSet rs0 --oplogSize 128
+    restart: always
+    healthcheck:
+      test: "echo 'db.runCommand({ping: 1})' | mongo"
+      interval: 5s
+      timeout: 10s
+      retries: 3
+
+  mongodb-setup:
+    image: mongo:4.4
+    container_name: mongodb-setup
+    depends_on:
+      - mongodb
+    entrypoint:
+      [
+        "bash",
+        "-c",
+        "sleep 10 && mongo --host mongodb:27017 /config-replica.js && sleep 10"
+      ]
+    restart: "no"
+    volumes:
+      - ./mongodb/config-replica.js:/config-replica.js
+
+  mongo_data_generator:
+      build:
+        context: .
+        dockerfile: ./mongodb/Dockerfile.generator
+      container_name: mongo_data_generator
+      depends_on:
+        - mongodb
+      environment:
+        MONGO_HOST: mongodb
+        MONGO_PORT: 27017
+        MONGO_DB_NAME: random_data

ci/mongodb/Dockerfile.generator

@@ -0,0 +1,23 @@
+# Use an official Python runtime as a parent image
+FROM python:3.8-slim
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy the requirements file into the container at /app
+COPY ./mongodb/requirements.txt /app
+COPY ./mongodb/app.py /app
+
+# Install any needed packages specified in requirements.txt
+RUN pip install -r requirements.txt
+
+# Make port 5000 available to the world outside this container
+EXPOSE 5000
+
+# Define environment variables
+ENV MONGO_HOST mongodb
+ENV MONGO_PORT 27017
+ENV MONGO_DB_NAME random_data
+
+# Run app.py when the container launches
+CMD ["python", "app.py"]

integration_tests/mongodb-cdc/app.py renamed to ci/mongodb/app.py

@@ -0,0 +1,6 @@
+rsconf = {
+    _id: "rs0",
+    members: [{ _id: 0, host: "mongodb:27017", priority: 1.0 }],
+};
+rs.initiate(rsconf);
+rs.status();

ci/mongodb/config-replica.js

@@ -0,0 +1,25 @@
+# CDC source basic test
+control substitution on
+
+statement ok
+CREATE TABLE users (_id JSONB PRIMARY KEY, payload JSONB) WITH (
+  connector = 'mongodb-cdc',
+  mongodb.url = 'mongodb://mongodb:27017/?replicaSet=rs0',
+  collection.name = 'random_data.*'
+);
+
+statement ok
+CREATE MATERIALIZED VIEW normalized_users AS
+SELECT
+    payload ->> 'name' as name,
+    payload ->> 'email' as email,
+    payload ->> 'address' as address
+FROM
+    users;
+
+sleep 5s;
+
+query I
+select count(*) from normalized_users;
+----
+1000

integration_tests/mongodb-cdc/requirements.txt renamed to ci/mongodb/requirements.txt

@@ -28,32 +28,12 @@ services:
       service: message_queue
 
   mongodb:
-    image: mongo:4.4
-    container_name: mongodb
-    ports:
-      - "27017:27017"
-    command: --replSet rs0 --oplogSize 128
-    restart: always
-    healthcheck:
-      test: "echo 'db.runCommand({ping: 1})' | mongo"
-      interval: 5s
-      timeout: 10s
-      retries: 3
+    extends: ../mongodb/docker-compose.yaml
+    service: mongodb
 
   mongodb-setup:
-    image: mongo:4.4
-    container_name: mongodb-setup
-    depends_on:
-      - mongodb
-    entrypoint:
-      [
-        "bash",
-        "-c",
-        "sleep 10 && mongo --host mongodb:27017 /config-replica.js && sleep 10"
-      ]
-    restart: "no"
-    volumes:
-      - ./config-replica.js:/config-replica.js
+    extends: ../mongodb/docker-compose.yaml
+    service: mongodb-setup
 
   debezium:
     image: debezium/connect:1.9
@@ -78,16 +58,9 @@ services:
       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://message_queue:8081
 
   random_data_generator:
-    build:
-      context: .
-      dockerfile: Dockerfile.generator
-    container_name: random_data_generator
-    depends_on:
-      - mongodb
-    environment:
-      MONGO_HOST: mongodb
-      MONGO_PORT: 27017
-      MONGO_DB_NAME: random_data
+    extends: ../mongodb/docker-compose.yaml
+    service: random_data_generator
+
 
   register-mongodb-connector:
     image: curlimages/curl:7.79.1

e2e_test/source/cdc/mongodb/mongo_basic.slt

@@ -2,41 +2,14 @@ version: '3.1'
 
 services:
   mongodb:
-    image: mongodb/mongodb-community-server:4.4.23-ubi8
-    container_name: mongodb
-    ports:
-      - "27017:27017"
-    command: --replSet rs0 --oplogSize 128
-    restart: always
-    healthcheck:
-      test: "echo 'db.runCommand({ping: 1})' | mongo"
-      interval: 5s
-      timeout: 10s
-      retries: 3
+    extends: ../mongodb/docker-compose.yaml
+    service: mongodb
 
   mongodb-setup:
-    image: mongodb/mongodb-community-server:4.4.23-ubi8
-    container_name: mongodb-setup
-    depends_on:
-      - mongodb
-    entrypoint:
-      [
-        "bash",
-        "-c",
-        "sleep 10 && mongo --host mongodb:27017 /config-replica.js && sleep 10"
-      ]
-    restart: "no"
-    volumes:
-      - ./config-replica.js:/config-replica.js
+    extends: ../mongodb/docker-compose.yaml
+    service: mongodb-setup
 
   random_data_generator:
-    build:
-      context: .
-      dockerfile: Dockerfile.generator
-    container_name: random_data_generator
-    depends_on:
-      - mongodb
-    environment:
-      MONGO_HOST: mongodb
-      MONGO_PORT: 27017
-      MONGO_DB_NAME: random_data
+    extends: ../mongodb/docker-compose.yaml
+    service: random_data_generator
+

integration_tests/debezium-mongo/docker-compose.yaml

@@ -0,0 +1,41 @@
+import os
+import pymongo
+from faker import Faker
+
+# To check the data through mongosh or mongo, run the following command:
+#  > mongosh mongodb://admin:[email protected]:27017
+#  > rs0 [direct: primary] test> use random_data
+#  > rs0 [direct: primary] random_data> db.users.find()
+#  > rs0 [direct: primary] random_data> db.users.count()
+
+# Connect to MongoDB
+mongo_host = os.environ["MONGO_HOST"]
+mongo_port = os.environ["MONGO_PORT"]
+mongo_db_name = os.environ["MONGO_DB_NAME"]
+
+url = f"mongodb://{mongo_host}:{mongo_port}"
+client = pymongo.MongoClient(url)
+db = client[mongo_db_name]
+
+# Generate random data
+fake = Faker()
+collection = db["users"]
+
+for _ in range(1000):
+    user_data = {
+        "name": fake.name(),
+        "address": fake.address(),
+        "email": fake.email(),
+    }
+    collection.insert_one(user_data)
+
+# Count the number of records in the collection
+total_records = collection.count_documents({})
+
+# Close the MongoDB connection
+client.close()
+print(f"Random data generated and inserted into MongoDB: {url}")
+
+# Print insertion summary
+print("Insertion summary:")
+print(f"Total records in the collection: {total_records}")

integration_tests/mongodb-cdc/docker-compose.yaml

@@ -0,0 +1,42 @@
+version: '3.1'
+
+services:
+  mongodb:
+    image: mongo:4.4
+    container_name: mongodb
+    ports:
+      - "27017:27017"
+    command: --replSet rs0 --oplogSize 128
+    restart: always
+    healthcheck:
+      test: "echo 'db.runCommand({ping: 1})' | mongo"
+      interval: 5s
+      timeout: 10s
+      retries: 3
+
+  mongodb-setup:
+    image: mongo:4.4
+    container_name: mongodb-setup
+    depends_on:
+      - mongodb
+    entrypoint:
+      [
+        "bash",
+        "-c",
+        "sleep 10 && mongo --host mongodb:27017 /config-replica.js && sleep 10"
+      ]
+    restart: "no"
+    volumes:
+      - ./config-replica.js:/config-replica.js
+
+  random_data_generator:
+    build:
+      context: .
+      dockerfile: Dockerfile.generator
+    container_name: random_data_generator
+    depends_on:
+      - mongodb
+    environment:
+      MONGO_HOST: mongodb
+      MONGO_PORT: 27017
+      MONGO_DB_NAME: random_data

integration_tests/mongodb-cdc/Dockerfile.generator renamed to integration_tests/mongodb/Dockerfile.generator

@@ -0,0 +1,2 @@
+pymongo
+Faker