Merge remote-tracking branch 'origin/main' into jinser/connector-ssl

Cargo.toml

@@ -7,6 +7,7 @@ members = [
   "src/cmd_all",
   "src/common",
   "src/common/common_service",
+  "src/common/estimate_size",
   "src/common/fields-derive",
   "src/common/heap_profiling",
   "src/common/metrics",
@@ -148,7 +149,7 @@ arrow-schema-deltalake = { package = "arrow-schema", version = "48.0.1" }
 deltalake = { git = "https://github.com/risingwavelabs/delta-rs", rev = "5c2dccd4640490202ffe98adbd13b09cef8e007b", features = [
   "s3-no-concurrent-write",
 ] }
-lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "95f347b" }
+lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "2682b85" }
 parquet = "50"
 thiserror-ext = "0.0.11"
 tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" }
@@ -174,8 +175,10 @@ risingwave_batch = { path = "./src/batch" }
 risingwave_cmd = { path = "./src/cmd" }
 risingwave_common = { path = "./src/common" }
 risingwave_common_service = { path = "./src/common/common_service" }
+risingwave_common_estimate_size = { path = "./src/common/estimate_size" }
 risingwave_common_heap_profiling = { path = "./src/common/heap_profiling" }
 risingwave_common_metrics = { path = "./src/common/metrics" }
+risingwave_common_proc_macro = { path = "./src/common/proc_macro" }
 risingwave_compactor = { path = "./src/storage/compactor" }
 risingwave_compute = { path = "./src/compute" }
 risingwave_ctl = { path = "./src/ctl" }

ci/docker-compose.yml

@@ -77,6 +77,9 @@ services:
       - db
       - message_queue
       - schemaregistry
+      - mongodb
+      - mongodb-setup
+      - mongo_data_generator
     volumes:
       - ..:/risingwave
 
@@ -122,23 +125,23 @@ services:
       - ..:/risingwave
 
   release-env-x86:
-    # build binaries on a earlier Linux distribution (therefore with earlier version GLIBC)
-    # See https://github.com/risingwavelabs/risingwave/issues/4556 for more details.
+    # Build binaries on a earlier Linux distribution (therefore with earlier version GLIBC)
+    # `manylinux_2_28` is based on AlmaLinux 8 with GLIBC 2.28.
     #
     # GLIBC versions on some systems:
-    # Amazon Linux 2: 2.26 (EOL 2025-06-30) (We will definitely want to support this)
-    # AL2023: 2.34
-    # Ubuntu 18.04: 2.27 (Already EOL 2023-05-31)
-    # Ubuntu 20.04: 2.31
+    # - Amazon Linux 2023 (AL2023): 2.34
+    # - Ubuntu 20.04: 2.31
     #
-    # manylinux2014: CentOS 7 (EOL 2024-06-30), GLIBC 2.17
-    image: quay.io/pypa/manylinux2014_x86_64
+    # Systems that we don't provide support for:
+    # - Ubuntu 18.04: 2.27 (Already EOL 2023-05-31)
+    # - Amazon Linux 2: 2.26 (Originally EOL 2023-06-30, superseded by AL2023)
+    image: quay.io/pypa/manylinux_2_28_x86_64
     working_dir: /mnt
     volumes:
       - ..:/mnt
 
   release-env-arm:
-    image: quay.io/pypa/manylinux2014_aarch64
+    image: quay.io/pypa/manylinux_2_28_aarch64
     working_dir: /mnt
     volumes:
       - ..:/mnt
@@ -274,3 +277,42 @@ services:
       interval: 5s
       timeout: 5s
       retries: 5
+
+  mongodb:
+    image: mongo:4.4
+    ports:
+      - "27017"
+    command: --replSet rs0 --oplogSize 128
+    restart: always
+    healthcheck:
+      test: "echo 'db.runCommand({ping: 1})' | mongo"
+      interval: 5s
+      timeout: 10s
+      retries: 3
+
+  mongodb-setup:
+    image: mongo:4.4
+    container_name: mongodb-setup
+    depends_on:
+      - mongodb
+    entrypoint:
+      [
+        "bash",
+        "-c",
+        "sleep 10 && mongo --host mongodb:27017 /config-replica.js && sleep 10"
+      ]
+    restart: "no"
+    volumes:
+      - ./mongodb/config-replica.js:/config-replica.js
+
+  mongo_data_generator:
+      build:
+        context: .
+        dockerfile: ./mongodb/Dockerfile.generator
+      container_name: mongo_data_generator
+      depends_on:
+        - mongodb
+      environment:
+        MONGO_HOST: mongodb
+        MONGO_PORT: 27017
+        MONGO_DB_NAME: random_data

ci/mongodb/Dockerfile.generator

@@ -0,0 +1,23 @@
+# Use an official Python runtime as a parent image
+FROM python:3.8-slim
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy the requirements file into the container at /app
+COPY ./mongodb/requirements.txt /app
+COPY ./mongodb/app.py /app
+
+# Install any needed packages specified in requirements.txt
+RUN pip install -r requirements.txt
+
+# Make port 5000 available to the world outside this container
+EXPOSE 5000
+
+# Define environment variables
+ENV MONGO_HOST mongodb
+ENV MONGO_PORT 27017
+ENV MONGO_DB_NAME random_data
+
+# Run app.py when the container launches
+CMD ["python", "app.py"]

ci/mongodb/app.py

@@ -0,0 +1,41 @@
+import os
+import pymongo
+from faker import Faker
+
+# To check the data through mongosh or mongo, run the following command:
+#  > mongosh mongodb://admin:[email protected]:27017
+#  > rs0 [direct: primary] test> use random_data
+#  > rs0 [direct: primary] random_data> db.users.find()
+#  > rs0 [direct: primary] random_data> db.users.count()
+
+# Connect to MongoDB
+mongo_host = os.environ["MONGO_HOST"]
+mongo_port = os.environ["MONGO_PORT"]
+mongo_db_name = os.environ["MONGO_DB_NAME"]
+
+url = f"mongodb://{mongo_host}:{mongo_port}"
+client = pymongo.MongoClient(url)
+db = client[mongo_db_name]
+
+# Generate random data
+fake = Faker()
+collection = db["users"]
+
+for _ in range(55):
+    user_data = {
+        "name": fake.name(),
+        "address": fake.address(),
+        "email": fake.email(),
+    }
+    collection.insert_one(user_data)
+
+# Count the number of records in the collection
+total_records = collection.count_documents({})
+
+# Close the MongoDB connection
+client.close()
+print(f"Random data generated and inserted into MongoDB: {url}")
+
+# Print insertion summary
+print("Insertion summary:")
+print(f"Total records in the collection: {total_records}")