From 67bdcd76e5068d4c26849320440b736650ae3315 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Wed, 20 Nov 2024 13:46:29 +0800 Subject: [PATCH 1/9] chore: log of frontend memory --- src/compute/src/server.rs | 3 +-- src/frontend/src/lib.rs | 2 +- src/frontend/src/session.rs | 12 ++++++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs index da3328b0b2ce..bd0988f7b94f 100644 --- a/src/compute/src/server.rs +++ b/src/compute/src/server.rs @@ -542,8 +542,7 @@ fn print_memory_config( reserved_memory_bytes: usize, ) { let memory_config = format!( - "\n\ - Memory outline:\n\ + "Memory outline:\n\ > total_memory: {}\n\ > storage_memory: {}\n\ > block_cache_capacity: {}\n\ diff --git a/src/frontend/src/lib.rs b/src/frontend/src/lib.rs index bf03edb6a87d..6424da42a151 100644 --- a/src/frontend/src/lib.rs +++ b/src/frontend/src/lib.rs @@ -162,7 +162,7 @@ pub struct FrontendOpts { )] pub temp_secret_file_dir: String, - /// Total available memory for the frontend node in bytes. Used by both computing and storage. + /// Total available memory for the frontend node in bytes. Used for batch computing. #[clap(long, env = "RW_FRONTEND_TOTAL_MEMORY_BYTES", default_value_t = default_frontend_total_memory_bytes())] pub frontend_total_memory_bytes: usize, } diff --git a/src/frontend/src/session.rs b/src/frontend/src/session.rs index fa4836e73e95..e53dc3ce6818 100644 --- a/src/frontend/src/session.rs +++ b/src/frontend/src/session.rs @@ -62,6 +62,7 @@ use risingwave_common::util::addr::HostAddr; use risingwave_common::util::cluster_limit; use risingwave_common::util::cluster_limit::ActorCountPerParallelism; use risingwave_common::util::iter_util::ZipEqFast; +use risingwave_common::util::pretty_bytes::convert; use risingwave_common::util::runtime::BackgroundShutdownRuntime; use risingwave_common::{GIT_SHA, RW_VERSION}; use risingwave_common_heap_profiling::HeapProfiler; @@ -450,9 +451,16 @@ impl FrontendEnv { // Run a background heap profiler heap_profiler.start(); - let mem_context = risingwave_common::memory::MemoryContext::root( + let batch_memory_limit = (total_memory_bytes as f64 * FRONTEND_BATCH_MEMORY_PROPORTION); + let mem_context = MemoryContext::root( frontend_metrics.batch_total_mem.clone(), - (total_memory_bytes as f64 * FRONTEND_BATCH_MEMORY_PROPORTION) as u64, + batch_memory_limit as u64, + ); + + format!( + "Frontend total_memory: {} batch_memory: {}", + convert(total_memory_bytes as _), + convert(batch_memory_limit as _), ); Ok(( From 2ca8c888d4d432c34d01f02145f39498da42ae24 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Wed, 20 Nov 2024 14:13:12 +0800 Subject: [PATCH 2/9] rename `total_memory` -> `target_memory` --- src/compute/src/memory/controller.rs | 6 +++--- src/compute/src/memory/manager.rs | 4 +++- src/compute/src/server.rs | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/compute/src/memory/controller.rs b/src/compute/src/memory/controller.rs index 2857b0e6b0d6..878471a7bedb 100644 --- a/src/compute/src/memory/controller.rs +++ b/src/compute/src/memory/controller.rs @@ -85,10 +85,10 @@ pub struct LruWatermarkController { impl LruWatermarkController { pub fn new(config: &MemoryManagerConfig) -> Self { - let threshold_stable = (config.total_memory as f64 * config.threshold_stable) as usize; - let threshold_graceful = (config.total_memory as f64 * config.threshold_graceful) as usize; + let threshold_stable = (config.target_memory as f64 * config.threshold_stable) as usize; + let threshold_graceful = (config.target_memory as f64 * config.threshold_graceful) as usize; let threshold_aggressive = - (config.total_memory as f64 * config.threshold_aggressive) as usize; + (config.target_memory as f64 * config.threshold_aggressive) as usize; Self { metrics: config.metrics.clone(), diff --git a/src/compute/src/memory/manager.rs b/src/compute/src/memory/manager.rs index b90624193c70..71b98100cb63 100644 --- a/src/compute/src/memory/manager.rs +++ b/src/compute/src/memory/manager.rs @@ -24,7 +24,9 @@ use risingwave_stream::executor::monitor::StreamingMetrics; use super::controller::LruWatermarkController; pub struct MemoryManagerConfig { - pub total_memory: usize, + /// [`MemoryManager`] will try to control the jemalloc-reported memory usage + /// to be lower than this + pub target_memory: usize, pub threshold_aggressive: f64, pub threshold_graceful: f64, diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs index bd0988f7b94f..52b671811577 100644 --- a/src/compute/src/server.rs +++ b/src/compute/src/server.rs @@ -296,7 +296,7 @@ pub async fn compute_node_serve( // - https://github.com/risingwavelabs/risingwave/issues/8696 // - https://github.com/risingwavelabs/risingwave/issues/8822 let memory_mgr = MemoryManager::new(MemoryManagerConfig { - total_memory: compute_memory_bytes + storage_memory_bytes, + target_memory: compute_memory_bytes + storage_memory_bytes, threshold_aggressive: config .streaming .developer From 20ddc77133938f7bf863b2c976e43807c02efd6d Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Wed, 20 Nov 2024 14:40:16 +0800 Subject: [PATCH 3/9] allow override memory_manager_target_bytes --- src/cmd_all/src/standalone.rs | 1 + src/compute/src/lib.rs | 7 +++++++ src/compute/src/server.rs | 15 +++++++-------- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/cmd_all/src/standalone.rs b/src/cmd_all/src/standalone.rs index 6eb62492999a..34d9749ff0e7 100644 --- a/src/cmd_all/src/standalone.rs +++ b/src/cmd_all/src/standalone.rs @@ -509,6 +509,7 @@ mod test { enable_barrier_read: None, temp_secret_file_dir: "./frontend/secrets/", frontend_total_memory_bytes: 34359738368, + memory_manager_target_bytes: None, }, ), compactor_opts: None, diff --git a/src/compute/src/lib.rs b/src/compute/src/lib.rs index 1336a84980ce..ef4b5c5e32d3 100644 --- a/src/compute/src/lib.rs +++ b/src/compute/src/lib.rs @@ -92,6 +92,13 @@ pub struct ComputeNodeOpts { #[clap(long, env = "RW_RESERVED_MEMORY_BYTES")] pub reserved_memory_bytes: Option, + /// Target memory usage for Memory Manager. + /// If not set, the default value is `total_memory_bytes` - `reserved_memory_bytes` + /// + /// It's strongly recommended to set it for standalone deployment. + #[clap(long, env = "RW_MEMORY_MANAGER_TARGET_BYTES")] + pub memory_manager_target_bytes: Option, + /// The parallelism that the compute node will register to the scheduler of the meta service. #[clap(long, env = "RW_PARALLELISM", default_value_t = default_parallelism())] #[override_opts(if_absent, path = streaming.actor_runtime_worker_threads_num)] diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs index 52b671811577..8de74cd45645 100644 --- a/src/compute/src/server.rs +++ b/src/compute/src/server.rs @@ -288,15 +288,14 @@ pub async fn compute_node_serve( batch_mem_limit(compute_memory_bytes, opts.role.for_serving()), )); - // NOTE: Due to some limits, we use `compute_memory_bytes + storage_memory_bytes` as - // `total_compute_memory_bytes` for memory control. This is just a workaround for some - // memory control issues and should be modified as soon as we figure out a better solution. - // - // Related issues: - // - https://github.com/risingwavelabs/risingwave/issues/8696 - // - https://github.com/risingwavelabs/risingwave/issues/8822 + let target_memory = if let Some(v) = opts.memory_manager_target_bytes { + v + } else { + compute_memory_bytes + storage_memory_bytes + }; + let memory_mgr = MemoryManager::new(MemoryManagerConfig { - target_memory: compute_memory_bytes + storage_memory_bytes, + target_memory, threshold_aggressive: config .streaming .developer From cf1582c4eb1ccce2886ce5194d3df5122834cc0f Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Wed, 20 Nov 2024 15:00:33 +0800 Subject: [PATCH 4/9] update docker-compose file --- docker/docker-compose-with-azblob.yml | 13 ++++++++----- docker/docker-compose-with-gcs.yml | 13 ++++++++----- docker/docker-compose-with-local-fs.yml | 12 ++++++++---- docker/docker-compose-with-obs.yml | 13 ++++++++----- docker/docker-compose-with-oss.yml | 13 ++++++++----- docker/docker-compose-with-s3.yml | 13 ++++++++----- docker/docker-compose-with-sqlite.yml | 13 ++++++++----- docker/docker-compose.yml | 13 ++++++++----- 8 files changed, 64 insertions(+), 39 deletions(-) diff --git a/docker/docker-compose-with-azblob.yml b/docker/docker-compose-with-azblob.yml index 99889d846dcb..d479f9464b5f 100644 --- a/docker/docker-compose-with-azblob.yml +++ b/docker/docker-compose-with-azblob.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-gcs.yml b/docker/docker-compose-with-gcs.yml index 80466c7cccab..6380b91a01a9 100644 --- a/docker/docker-compose-with-gcs.yml +++ b/docker/docker-compose-with-gcs.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-local-fs.yml b/docker/docker-compose-with-local-fs.yml index 68483796ac80..abfea87b5a68 100644 --- a/docker/docker-compose-with-local-fs.yml +++ b/docker/docker-compose-with-local-fs.yml @@ -21,21 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - # --parallelism 4 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-obs.yml b/docker/docker-compose-with-obs.yml index f4bf8dc0e74c..690da92d8b96 100644 --- a/docker/docker-compose-with-obs.yml +++ b/docker/docker-compose-with-obs.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-oss.yml b/docker/docker-compose-with-oss.yml index 7d9563473182..dab8af159299 100644 --- a/docker/docker-compose-with-oss.yml +++ b/docker/docker-compose-with-oss.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-s3.yml b/docker/docker-compose-with-s3.yml index d7dc75aa556a..c9d839220c94 100644 --- a/docker/docker-compose-with-s3.yml +++ b/docker/docker-compose-with-s3.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-sqlite.yml b/docker/docker-compose-with-sqlite.yml index d4081b592c2a..98d88a415d49 100644 --- a/docker/docker-compose-with-sqlite.yml +++ b/docker/docker-compose-with-sqlite.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e315878c98b7..b7b29313547b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" From 4a0fbeb488859cbbb7ab0f54905c74c35f8fae03 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Wed, 20 Nov 2024 15:29:54 +0800 Subject: [PATCH 5/9] update Readme --- docker/README.md | 99 ++++++++++++++++++++++-------------------------- 1 file changed, 46 insertions(+), 53 deletions(-) diff --git a/docker/README.md b/docker/README.md index 865f7c64c518..8a3d51778b73 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,43 +1,21 @@ -# Docker Images +# RisingWave Docker Images -## Published images - -- `latest` on GHCR (latest nightly build): `ghcr.io/risingwavelabs/risingwave:latest` -- `latest` on Docker Hub (latest release): `risingwavelabs/risingwave:latest` -- Other tags available on both GHCR and Docker Hub: - - `nightly-yyyyMMdd`, e.g., `nightly-20230108` - - `vX.Y.Z`, e.g., `v0.1.15` - -## Build the images - -The docker images for x86_64 are built with AVX2 SIMD extensions, while the images for aarch64 are built with NEON SIMD extensions. These must be available on your machine. If your machine does not support these extensions, you must build the docker image with the build-arg `simd_disabled=true`. - -To build the images, simply run: - -``` -docker build . -f docker/Dockerfile -``` - -from the project root. - -To build the images without SIMD vector extensions, run - -``` -docker build . -f docker/Dockerfile --build-arg simd_disabled=true -``` +## Run RisingWave with Docker Compose -from the project root and run any subsequent docker commands on the resultant image. +Docker Compose allows you to easily launch a RisingWave instance on a single node. If you are using more than one server, please refer to [Deploy RisingWave on Kubernetes](https://docs.risingwave.com/deploy/risingwave-kubernetes). -## Use the images -To ensure you are using the latest version of RisingWave image, +Ensure you are using the latest version of RisingWave image: ``` # Ensure risingwave image is of latest version docker pull ghcr.io/risingwavelabs/risingwave:latest ``` -### playground +### Playground + +Playground mode does not persist any data. **Never** use it for production purpose. + To start a RisingWave playground, run ``` @@ -45,25 +23,26 @@ To start a RisingWave playground, run docker run -it --pull=always -p 4566:4566 -p 5691:5691 ghcr.io/risingwavelabs/risingwave:latest playground ``` -### standalone minio -To start a RisingWave standalone mode with minio backend, run +### Standalone (MinIO backend) + +To start a RisingWave standalone instance with MinIO backend, run ``` # Start all components docker-compose up ``` -### distributed cluster minio -To start a RisingWave cluster with minio backend, run +**⚠️ Important Notes: Memory is crucial for RisingWave!** Inappropriate memory configuration may lead to OOM (out-of-memory) errors or poor performance. Before deploying Docker Compose, ensure that the correct memory settings are configured in the `docker-compose.yaml` file. Here are examples of some typical settings. -``` -# Start all components -docker-compose -f docker-compose-distributed.yml up -``` +| Memory for RW container (`resource.limits.memory`) | 8 GiB | 14 GiB | 28 GiB | 58 GiB | +|----------------------------------------------------|---|---|---|---| +| `compute-opts.memory-manager-target-bytes` | 6 GiB | 10 GiB | 20 GiB | 46 GiB | +| `frontend-opts.frontend-total-memory-bytes` | 1 GiB | 2 GiB | 4 GiB | 6 GiB | +| `compactor-opts.compactor-total-memory-bytes` | 1 GiB | 2 GiB | 4 GiB | 6 GiB | +| `compute-opts.memory-manager-target-bytes` | 5.6 GiB | 9.8 GiB | 20.8 GiB | 44.8 GiB | -It will start a minio, a meta node, a compute node, a frontend, a compactor, a prometheus and a redpanda instance. +### Standalone (S3 backend) -### s3 and other s3-compatible storage backend To start a RisingWave cluster with s3 backend, configure the aws credit in [aws.env](https://github.com/risingwavelabs/risingwave/blob/main/docker/aws.env). If you want to use some s3 compatible storage like Tencent Cloud COS, just configure one more [endpoint](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/aws.env#L7). After configuring the environment and fill in your [bucket name](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-s3.yml#L196), run @@ -75,7 +54,8 @@ docker-compose -f docker-compose-with-s3.yml up It will run with s3 (compatible) object storage with a meta node, a compute node, a frontend, a compactor, a prometheus and a redpanda instance. -### Start with other storage products of public cloud vendors +### Standalone (other backend) + To start a RisingWave cluster with other storage backend, like Google Cloud Storage, Alicloud OSS or Azure Blob Storage, configure the authentication information in [multiple_object_storage.env](https://github.com/risingwavelabs/risingwave/blob/main/docker/multiple_object_storage.env), fill in your [bucket name](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-gcs.yml#L196). and run @@ -86,27 +66,39 @@ docker-compose -f docker-compose-with-xxx.yml up It will run RisingWave with corresponding (object) storage products. -### Start with HDFS backend -To start a RisingWave cluster with HDFS, mount your `HADDOP_HOME` in [compactor node volumes](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L28), [compute node volumes](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L112) [compute node volumes](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L218), fill in the [cluster_name/namenode](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L202), -and run +> [!NOTE] +> +> For RisingWave kernel hackers, we always recommend using [risedev](../src/risedevtool/README.md) to start the full cluster, instead of using docker images. +> See [CONTRIBUTING](../CONTRIBUTING.md) for more information. + + +## Published images + +- `latest` on GHCR (latest nightly build): `ghcr.io/risingwavelabs/risingwave:latest` +- `latest` on Docker Hub (latest release): `risingwavelabs/risingwave:latest` +- Other tags available on both GHCR and Docker Hub: + - `nightly-yyyyMMdd`, e.g., `nightly-20230108` + - `vX.Y.Z`, e.g., `v0.1.15` + +## Build the images + +The docker images for x86_64 are built with AVX2 SIMD extensions, while the images for aarch64 are built with NEON SIMD extensions. These must be available on your machine. If your machine does not support these extensions, you must build the docker image with the build-arg `simd_disabled=true`. + +To build the images, simply run: ``` -# Start all components -docker-compose -f docker-compose-with-hdfs.yml up +docker build . -f docker/Dockerfile ``` -It will run RisingWave with HDFS. +from the project root. -To clean all data, run: +To build the images without SIMD vector extensions, run ``` -docker-compose down -v +docker build . -f docker/Dockerfile --build-arg simd_disabled=true ``` -> [!NOTE] -> -> For RisingWave kernel hackers, we always recommend using [risedev](../src/risedevtool/README.md) to start the full cluster, instead of using docker images. -> See [CONTRIBUTING](../CONTRIBUTING.md) for more information. +from the project root and run any subsequent docker commands on the resultant image. ## Generate docker-compose.yml @@ -122,4 +114,5 @@ Error { code: "XMinioStorageFull", message: "Storage backend has reached its min ``` Solution: + This usually happens on MacOS with Docker Desktop. The Docker Deskup runs in the macOS Hypervisor. All the data, including logs, images, volumes, and so on, is stored in this hypervisor and the hypervisor has a default disk capacity limit. So when this message emerges, simply cleaning up the unused container or image can help mitigate. You can also increase capacity limit by clicking the Docker Desktop icon in the menu bar, then clicking Preferences > Resources > `Increase Disk image size`. From 4d87710aa5ce69b88857c57906dfea9b52b082c5 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Thu, 21 Nov 2024 15:59:45 +0800 Subject: [PATCH 6/9] minor fix --- docker/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 8a3d51778b73..13782a0447bb 100644 --- a/docker/README.md +++ b/docker/README.md @@ -36,11 +36,12 @@ docker-compose up | Memory for RW container (`resource.limits.memory`) | 8 GiB | 14 GiB | 28 GiB | 58 GiB | |----------------------------------------------------|---|---|---|---| -| `compute-opts.memory-manager-target-bytes` | 6 GiB | 10 GiB | 20 GiB | 46 GiB | +| `compute-opts.total-memory-bytes` | 6 GiB | 10 GiB | 20 GiB | 46 GiB | | `frontend-opts.frontend-total-memory-bytes` | 1 GiB | 2 GiB | 4 GiB | 6 GiB | | `compactor-opts.compactor-total-memory-bytes` | 1 GiB | 2 GiB | 4 GiB | 6 GiB | | `compute-opts.memory-manager-target-bytes` | 5.6 GiB | 9.8 GiB | 20.8 GiB | 44.8 GiB | + ### Standalone (S3 backend) To start a RisingWave cluster with s3 backend, configure the aws credit in [aws.env](https://github.com/risingwavelabs/risingwave/blob/main/docker/aws.env). From ce05e4d89c9a08008491989496b85c5455fee53d Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Fri, 22 Nov 2024 10:45:21 +0800 Subject: [PATCH 7/9] fix risedev check --- src/frontend/src/session.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/frontend/src/session.rs b/src/frontend/src/session.rs index e53dc3ce6818..a920dc32653f 100644 --- a/src/frontend/src/session.rs +++ b/src/frontend/src/session.rs @@ -451,13 +451,13 @@ impl FrontendEnv { // Run a background heap profiler heap_profiler.start(); - let batch_memory_limit = (total_memory_bytes as f64 * FRONTEND_BATCH_MEMORY_PROPORTION); + let batch_memory_limit = total_memory_bytes as f64 * FRONTEND_BATCH_MEMORY_PROPORTION; let mem_context = MemoryContext::root( frontend_metrics.batch_total_mem.clone(), batch_memory_limit as u64, ); - format!( + info!( "Frontend total_memory: {} batch_memory: {}", convert(total_memory_bytes as _), convert(batch_memory_limit as _), From e30f448305f79459a0f3366059a10e1facd2de3c Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Fri, 22 Nov 2024 15:19:20 +0800 Subject: [PATCH 8/9] fix the test case... --- src/cmd_all/src/standalone.rs | 126 ++++++++++------------------------ 1 file changed, 37 insertions(+), 89 deletions(-) diff --git a/src/cmd_all/src/standalone.rs b/src/cmd_all/src/standalone.rs index 34d9749ff0e7..22960daeb996 100644 --- a/src/cmd_all/src/standalone.rs +++ b/src/cmd_all/src/standalone.rs @@ -426,94 +426,42 @@ mod test { // Test parsing into node-level opts. let actual = parse_standalone_opt_args(&opts); - check( - actual, - expect![[r#" - ParsedStandaloneOpts { - meta_opts: Some( - MetaNodeOpts { - listen_addr: "127.0.0.1:8001", - advertise_addr: "127.0.0.1:9999", - dashboard_host: None, - prometheus_listener_addr: Some( - "127.0.0.1:1234", - ), - sql_endpoint: None, - sql_username: "", - sql_password: [REDACTED alloc::string::String], - sql_database: "", - prometheus_endpoint: None, - prometheus_selector: None, - privatelink_endpoint_default_tags: None, - vpc_id: None, - security_group_id: None, - config_path: "src/config/test.toml", - backend: None, - barrier_interval_ms: None, - sstable_size_mb: None, - block_size_kb: None, - bloom_false_positive: None, - state_store: None, - data_directory: Some( - "some path with spaces", - ), - do_not_config_object_storage_lifecycle: None, - backup_storage_url: None, - backup_storage_directory: None, - heap_profiling_dir: None, - dangerous_max_idle_secs: None, - connector_rpc_endpoint: None, - license_key: None, - license_key_path: None, - temp_secret_file_dir: "./meta/secrets/", - }, - ), - compute_opts: Some( - ComputeNodeOpts { - listen_addr: "127.0.0.1:8000", - advertise_addr: None, - prometheus_listener_addr: "127.0.0.1:1234", - meta_address: List( - [ - http://127.0.0.1:5690/, - ], - ), - config_path: "src/config/test.toml", - total_memory_bytes: 34359738368, - reserved_memory_bytes: None, - parallelism: 10, - role: Both, - metrics_level: None, - data_file_cache_dir: None, - meta_file_cache_dir: None, - async_stack_trace: None, - heap_profiling_dir: None, - connector_rpc_endpoint: None, - temp_secret_file_dir: "./compute/secrets/", - }, - ), - frontend_opts: Some( - FrontendOpts { - listen_addr: "0.0.0.0:4566", - tcp_keepalive_idle_secs: 300, - advertise_addr: None, - meta_addr: List( - [ - http://127.0.0.1:5690/, - ], - ), - prometheus_listener_addr: "127.0.0.1:1234", - frontend_rpc_listener_addr: "127.0.0.1:6786", - config_path: "src/config/test.toml", - metrics_level: None, - enable_barrier_read: None, - temp_secret_file_dir: "./frontend/secrets/", - frontend_total_memory_bytes: 34359738368, - memory_manager_target_bytes: None, - }, - ), - compactor_opts: None, - }"#]], - ); + if let Some(compute_opts) = &actual.compute_opts { + assert_eq!(compute_opts.listen_addr, "127.0.0.1:8000"); + assert_eq!(compute_opts.total_memory_bytes, 34359738368); + assert_eq!(compute_opts.parallelism, 10); + assert_eq!(compute_opts.temp_secret_file_dir, "./compute/secrets/"); + assert_eq!(compute_opts.prometheus_listener_addr, "127.0.0.1:1234"); + assert_eq!(compute_opts.config_path, "src/config/test.toml"); + } else { + assert!(false); + } + if let Some(meta_opts) = &actual.meta_opts { + assert_eq!(meta_opts.listen_addr, "127.0.0.1:8001"); + assert_eq!(meta_opts.advertise_addr, "127.0.0.1:9999"); + assert_eq!( + meta_opts.data_directory, + Some("some path with spaces".to_string()) + ); + assert_eq!(meta_opts.temp_secret_file_dir, "./meta/secrets/"); + assert_eq!( + meta_opts.prometheus_listener_addr, + Some("127.0.0.1:1234".to_string()) + ); + assert_eq!(meta_opts.config_path, "src/config/test.toml"); + } else { + assert!(false); + } + + if let Some(frontend_opts) = &actual.frontend_opts { + assert_eq!(frontend_opts.config_path, "src/config/test.toml"); + assert_eq!(frontend_opts.temp_secret_file_dir, "./frontend/secrets/"); + assert_eq!(frontend_opts.frontend_total_memory_bytes, 34359738368); + assert_eq!(frontend_opts.prometheus_listener_addr, "127.0.0.1:1234"); + } else { + assert!(false); + } + + assert!(actual.compactor_opts.is_none()); } } From 2c9432696bb86378238c6f21381e827c2f16daf4 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Fri, 22 Nov 2024 16:20:53 +0800 Subject: [PATCH 9/9] fix cargo clippy --- src/cmd_all/src/standalone.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/cmd_all/src/standalone.rs b/src/cmd_all/src/standalone.rs index 22960daeb996..b5730d8d7845 100644 --- a/src/cmd_all/src/standalone.rs +++ b/src/cmd_all/src/standalone.rs @@ -391,17 +391,9 @@ It SHOULD NEVER be used in benchmarks and production environment!!!" #[cfg(test)] mod test { - use std::fmt::Debug; - - use expect_test::{expect, Expect}; - use super::*; - fn check(actual: impl Debug, expect: Expect) { - let actual = format!("{:#?}", actual); - expect.assert_eq(&actual); - } - + #[allow(clippy::assertions_on_constants)] #[test] fn test_parse_opt_args() { // Test parsing into standalone-level opts.