DO NOT MERGE: Notes about the RW + TiDB presentation in London

integration_tests/tidb-cdc-sink/docker-compose.yml

@@ -1,26 +1,230 @@
 ---
-version: '3'
+version: "3"
+x-image: &image
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.8.1}
 services:
   risingwave-standalone:
-    extends:
-      file: ../../docker/docker-compose.yml
-      service: risingwave-standalone
+    <<: *image
+    command: "standalone --meta-opts=\" \
+                    --listen-addr 0.0.0.0:5690 \
+                    --advertise-addr 0.0.0.0:5690 \
+                    --dashboard-host 0.0.0.0:5691 \
+                    --prometheus-host 0.0.0.0:1250 \
+                    --prometheus-endpoint http://prometheus-0:9500 \
+                    --backend etcd \
+                    --etcd-endpoints etcd-0:2388 \
+                    --state-store hummock+minio://hummockadmin:hummockadmin@minio-0:9301/hummock001 \
+                    --data-directory hummock_001 \
+                    --config-path /risingwave.toml\" \
+                 --compute-opts=\" \
+                    --config-path /risingwave.toml \
+                    --listen-addr 0.0.0.0:5688 \
+                    --prometheus-listener-addr 0.0.0.0:1250 \
+                    --advertise-addr 0.0.0.0:5688 \
+                    --async-stack-trace verbose \
+                    #--parallelism 4 \
+                    #--total-memory-bytes 8589934592 \
+                    --role both \
+                    --meta-address http://0.0.0.0:5690\" \
+                 --frontend-opts=\" \
+                   --config-path /risingwave.toml \
+                   --listen-addr 0.0.0.0:4566 \
+                   --advertise-addr 0.0.0.0:4566 \
+                   --prometheus-listener-addr 0.0.0.0:1250 \
+                   --health-check-listener-addr 0.0.0.0:6786 \
+                   --meta-addr http://0.0.0.0:5690\" \
+                 --compactor-opts=\" \
+                   --listen-addr 0.0.0.0:6660 \
+                   --prometheus-listener-addr 0.0.0.0:1250 \
+                   --advertise-addr 0.0.0.0:6660 \
+                   --meta-address http://0.0.0.0:5690\""
+    expose:
+      - "6660"
+      - "4566"
+      - "5688"
+      - "5690"
+      - "1250"
+      - "5691"
+    ports:
+      - "4566:4566"
+      - "5690:5690"
+      - "5691:5691"
+      - "1250:1250"
+    depends_on:
+      - etcd-0
+      - minio-0
+    volumes:
+      - "./risingwave.toml:/risingwave.toml"
+    environment:
+      RUST_BACKTRACE: "1"
+      # If ENABLE_TELEMETRY is not set, telemetry will start by default
+      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
+    container_name: risingwave-standalone
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/6660; exit $$?;'
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/5688; exit $$?;'
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/4566; exit $$?;'
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/5690; exit $$?;'
+      interval: 1s
+      timeout: 5s
+    restart: always
+    deploy:
+      resources:
+        limits:
+          memory: 28G
+        reservations:
+          memory: 28G
+
   etcd-0:
-    extends:
-      file: ../../docker/docker-compose.yml
-      service: etcd-0
+    image: "quay.io/coreos/etcd:v3.5.10"
+    command:
+      - /usr/local/bin/etcd
+      - "--listen-client-urls"
+      - "http://0.0.0.0:2388"
+      - "--advertise-client-urls"
+      - "http://etcd-0:2388"
+      - "--listen-peer-urls"
+      - "http://0.0.0.0:2389"
+      - "--initial-advertise-peer-urls"
+      - "http://etcd-0:2389"
+      - "--listen-metrics-urls"
+      - "http://0.0.0.0:2379"
+      - "--name"
+      - risedev-meta
+      - "--max-txn-ops"
+      - "999999"
+      - "--max-request-bytes"
+      - "10485760"
+      - "--auto-compaction-mode"
+      - periodic
+      - "--auto-compaction-retention"
+      - 1m
+      - "--snapshot-count"
+      - "10000"
+      - "--data-dir"
+      - /etcd-data
+    expose:
+      - "2388"
+    ports:
+      - "2388:2388"
+      - "2389:2389"
+    depends_on: []
+    volumes:
+      - "etcd-0:/etcd-data"
+    environment: {}
+    container_name: etcd-0
+    healthcheck:
+      test:
+        - CMD
+        - etcdctl
+        - --endpoints=http://localhost:2388
+        - endpoint
+        - health
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+
   grafana-0:
-    extends:
-      file: ../../docker/docker-compose.yml
-      service: grafana-0
+    image: "grafana/grafana-oss:latest"
+    command: []
+    expose:
+      - "3001"
+    ports:
+      - "3001:3001"
+    depends_on: []
+    volumes:
+      - "grafana-0:/var/lib/grafana"
+      - "./grafana.ini:/etc/grafana/grafana.ini"
+      - "./grafana-risedev-datasource.yml:/etc/grafana/provisioning/datasources/grafana-risedev-datasource.yml"
+      - "./grafana-risedev-dashboard.yml:/etc/grafana/provisioning/dashboards/grafana-risedev-dashboard.yml"
+      - "./dashboards:/dashboards"
+    environment: {}
+    container_name: grafana-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/3001; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+
   minio-0:
-    extends:
-      file: ../../docker/docker-compose.yml
-      service: minio-0
+    image: "quay.io/minio/minio:latest"
+    command:
+      - server
+      - "--address"
+      - "0.0.0.0:9301"
+      - "--console-address"
+      - "0.0.0.0:9400"
+      - /data
+    expose:
+      - "9301"
+      - "9400"
+    ports:
+      - "9301:9301"
+      - "9400:9400"
+    depends_on: []
+    volumes:
+      - "minio-0:/data"
+    entrypoint: "
+
+      /bin/sh -c '
+
+      set -e
+
+      mkdir -p \"/data/hummock001\"
+
+      /usr/bin/docker-entrypoint.sh \"$$0\" \"$$@\"
+
+      '"
+    environment:
+      MINIO_CI_CD: "1"
+      MINIO_PROMETHEUS_AUTH_TYPE: public
+      MINIO_PROMETHEUS_URL: "http://prometheus-0:9500"
+      MINIO_ROOT_PASSWORD: hummockadmin
+      MINIO_ROOT_USER: hummockadmin
+      MINIO_DOMAIN: "minio-0"
+    container_name: minio-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/9301; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+
   prometheus-0:
-    extends:
-      file: ../../docker/docker-compose.yml
-      service: prometheus-0
+    image: "prom/prometheus:latest"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--web.listen-address=0.0.0.0:9500"
+      - "--storage.tsdb.retention.time=30d"
+    expose:
+      - "9500"
+    ports:
+      - "9500:9500"
+    depends_on: []
+    volumes:
+      - "prometheus-0:/prometheus"
+      - "./prometheus.yaml:/etc/prometheus/prometheus.yml"
+    environment: {}
+    container_name: prometheus-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - sh -c 'printf "GET /-/healthy HTTP/1.0\n\n" | nc localhost 9500; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
 
   #=================== TiDB & TiCDC components ==================
   ticdc-controller:
@@ -163,7 +367,7 @@ services:
       # Exposes 9092 for external connections to the broker
       # Use kafka:29092 for connections internal on the docker network
       # See https://rmoff.net/2018/08/02/kafka-listeners-explained/ for details
-      - 9092:9092
+      - 29092:29092
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

integration_tests/tidb-cdc-sink/grafana-risedev-dashboard.yml

@@ -0,0 +1,15 @@
+# --- THIS FILE IS AUTO GENERATED BY RISEDEV ---
+apiVersion: 1
+
+providers:
+  - name: 'risingwave-grafana'
+    orgId: 1
+    folder: ''
+    folderUid: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 1
+    allowUiUpdates: true
+    options:
+      path: /dashboards
+      foldersFromFilesStructure: false

integration_tests/tidb-cdc-sink/grafana-risedev-datasource.yml

@@ -0,0 +1,15 @@
+# --- THIS FILE IS AUTO GENERATED BY RISEDEV ---
+apiVersion: 1
+deleteDatasources:
+  - name: risedev-prometheus
+datasources:
+  - name: risedev-prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus-0:9500
+    withCredentials: false
+    isDefault: true
+    tlsAuth: false
+    tlsAuthWithCACert: false
+    version: 1
+    editable: true

integration_tests/tidb-cdc-sink/grafana.ini

@@ -0,0 +1,11 @@
+# --- THIS FILE IS AUTO GENERATED BY RISEDEV ---
+[server]
+http_addr = 0.0.0.0
+http_port = 3001
+
+[users]
+default_theme = light
+
+[auth.anonymous]
+enabled = true
+org_role = Admin

integration_tests/tidb-cdc-sink/prometheus.yaml

@@ -0,0 +1,42 @@
+# --- THIS FILE IS AUTO GENERATED BY RISEDEV ---
+global:
+  scrape_interval: 15s
+  evaluation_interval: 60s
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ["prometheus-0:9500"]
+
+  - job_name: compute
+    static_configs:
+      - targets: ["compute-node-0:1222"]
+
+  - job_name: meta
+    static_configs:
+      - targets: ["meta-node-0:1250"]
+
+  - job_name: minio
+    metrics_path: /minio/v2/metrics/cluster
+    static_configs:
+      - targets: ["minio-0:9301"]
+
+  - job_name: compactor
+    static_configs:
+      - targets: ["compactor-0:1260"]
+
+  - job_name: etcd
+    static_configs:
+      - targets: ["etcd-0:2379"]
+
+  - job_name: frontend
+    static_configs:
+      - targets: ["frontend-node-0:2222"]
+
+  - job_name: redpanda
+    static_configs:
+      - targets: ["message_queue:9644"]
+
+  - job_name: standalone
+    static_configs:
+      - targets: ["risingwave-standalone:1250"]

integration_tests/tidb-cdc-sink/risingwave.toml

@@ -0,0 +1,2 @@
+# RisingWave config file to be mounted into the Docker containers.
+# See src/config/example.toml for example

presentation_london/docker-compose-debezium.yml

@@ -0,0 +1,37 @@
+version: '2'
+services:
+  zookeeper:
+    image: quay.io/debezium/zookeeper:latest
+    ports:
+     - 2182:2181
+     - 2888:2888
+     - 3888:3888
+  kafka:
+    image: quay.io/debezium/kafka:latest
+    ports:
+     - 9093:9092  
+    links:
+     - zookeeper
+    environment:
+     - ZOOKEEPER_CONNECT=zookeeper:2181
+  mysql:
+    image: quay.io/debezium/example-mysql:latest
+    ports:
+     - 3306:3306
+    environment:
+     - MYSQL_ROOT_PASSWORD=debezium
+     - MYSQL_USER=mysqluser
+     - MYSQL_PASSWORD=mysqlpw
+  connect:
+    image: quay.io/debezium/connect:latest
+    ports:
+     - 8083:8083
+    links:
+     - kafka
+     - mysql
+    environment:
+     - BOOTSTRAP_SERVERS=kafka:9092
+     - GROUP_ID=1
+     - CONFIG_STORAGE_TOPIC=my_connect_configs
+     - OFFSET_STORAGE_TOPIC=my_connect_offsets
+     - STATUS_STORAGE_TOPIC=my_connect_statuses