diff --git a/docker/README.md b/docker/README.md index 865f7c64c518..13782a0447bb 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,43 +1,21 @@ -# Docker Images +# RisingWave Docker Images -## Published images +## Run RisingWave with Docker Compose -- `latest` on GHCR (latest nightly build): `ghcr.io/risingwavelabs/risingwave:latest` -- `latest` on Docker Hub (latest release): `risingwavelabs/risingwave:latest` -- Other tags available on both GHCR and Docker Hub: - - `nightly-yyyyMMdd`, e.g., `nightly-20230108` - - `vX.Y.Z`, e.g., `v0.1.15` +Docker Compose allows you to easily launch a RisingWave instance on a single node. If you are using more than one server, please refer to [Deploy RisingWave on Kubernetes](https://docs.risingwave.com/deploy/risingwave-kubernetes). -## Build the images -The docker images for x86_64 are built with AVX2 SIMD extensions, while the images for aarch64 are built with NEON SIMD extensions. These must be available on your machine. If your machine does not support these extensions, you must build the docker image with the build-arg `simd_disabled=true`. - -To build the images, simply run: - -``` -docker build . -f docker/Dockerfile -``` - -from the project root. - -To build the images without SIMD vector extensions, run - -``` -docker build . -f docker/Dockerfile --build-arg simd_disabled=true -``` - -from the project root and run any subsequent docker commands on the resultant image. - -## Use the images - -To ensure you are using the latest version of RisingWave image, +Ensure you are using the latest version of RisingWave image: ``` # Ensure risingwave image is of latest version docker pull ghcr.io/risingwavelabs/risingwave:latest ``` -### playground +### Playground + +Playground mode does not persist any data. **Never** use it for production purpose. + To start a RisingWave playground, run ``` @@ -45,25 +23,27 @@ To start a RisingWave playground, run docker run -it --pull=always -p 4566:4566 -p 5691:5691 ghcr.io/risingwavelabs/risingwave:latest playground ``` -### standalone minio -To start a RisingWave standalone mode with minio backend, run +### Standalone (MinIO backend) + +To start a RisingWave standalone instance with MinIO backend, run ``` # Start all components docker-compose up ``` -### distributed cluster minio -To start a RisingWave cluster with minio backend, run +**⚠️ Important Notes: Memory is crucial for RisingWave!** Inappropriate memory configuration may lead to OOM (out-of-memory) errors or poor performance. Before deploying Docker Compose, ensure that the correct memory settings are configured in the `docker-compose.yaml` file. Here are examples of some typical settings. + +| Memory for RW container (`resource.limits.memory`) | 8 GiB | 14 GiB | 28 GiB | 58 GiB | +|----------------------------------------------------|---|---|---|---| +| `compute-opts.total-memory-bytes` | 6 GiB | 10 GiB | 20 GiB | 46 GiB | +| `frontend-opts.frontend-total-memory-bytes` | 1 GiB | 2 GiB | 4 GiB | 6 GiB | +| `compactor-opts.compactor-total-memory-bytes` | 1 GiB | 2 GiB | 4 GiB | 6 GiB | +| `compute-opts.memory-manager-target-bytes` | 5.6 GiB | 9.8 GiB | 20.8 GiB | 44.8 GiB | -``` -# Start all components -docker-compose -f docker-compose-distributed.yml up -``` -It will start a minio, a meta node, a compute node, a frontend, a compactor, a prometheus and a redpanda instance. +### Standalone (S3 backend) -### s3 and other s3-compatible storage backend To start a RisingWave cluster with s3 backend, configure the aws credit in [aws.env](https://github.com/risingwavelabs/risingwave/blob/main/docker/aws.env). If you want to use some s3 compatible storage like Tencent Cloud COS, just configure one more [endpoint](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/aws.env#L7). After configuring the environment and fill in your [bucket name](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-s3.yml#L196), run @@ -75,7 +55,8 @@ docker-compose -f docker-compose-with-s3.yml up It will run with s3 (compatible) object storage with a meta node, a compute node, a frontend, a compactor, a prometheus and a redpanda instance. -### Start with other storage products of public cloud vendors +### Standalone (other backend) + To start a RisingWave cluster with other storage backend, like Google Cloud Storage, Alicloud OSS or Azure Blob Storage, configure the authentication information in [multiple_object_storage.env](https://github.com/risingwavelabs/risingwave/blob/main/docker/multiple_object_storage.env), fill in your [bucket name](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-gcs.yml#L196). and run @@ -86,27 +67,39 @@ docker-compose -f docker-compose-with-xxx.yml up It will run RisingWave with corresponding (object) storage products. -### Start with HDFS backend -To start a RisingWave cluster with HDFS, mount your `HADDOP_HOME` in [compactor node volumes](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L28), [compute node volumes](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L112) [compute node volumes](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L218), fill in the [cluster_name/namenode](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L202), -and run +> [!NOTE] +> +> For RisingWave kernel hackers, we always recommend using [risedev](../src/risedevtool/README.md) to start the full cluster, instead of using docker images. +> See [CONTRIBUTING](../CONTRIBUTING.md) for more information. + + +## Published images + +- `latest` on GHCR (latest nightly build): `ghcr.io/risingwavelabs/risingwave:latest` +- `latest` on Docker Hub (latest release): `risingwavelabs/risingwave:latest` +- Other tags available on both GHCR and Docker Hub: + - `nightly-yyyyMMdd`, e.g., `nightly-20230108` + - `vX.Y.Z`, e.g., `v0.1.15` + +## Build the images + +The docker images for x86_64 are built with AVX2 SIMD extensions, while the images for aarch64 are built with NEON SIMD extensions. These must be available on your machine. If your machine does not support these extensions, you must build the docker image with the build-arg `simd_disabled=true`. + +To build the images, simply run: ``` -# Start all components -docker-compose -f docker-compose-with-hdfs.yml up +docker build . -f docker/Dockerfile ``` -It will run RisingWave with HDFS. +from the project root. -To clean all data, run: +To build the images without SIMD vector extensions, run ``` -docker-compose down -v +docker build . -f docker/Dockerfile --build-arg simd_disabled=true ``` -> [!NOTE] -> -> For RisingWave kernel hackers, we always recommend using [risedev](../src/risedevtool/README.md) to start the full cluster, instead of using docker images. -> See [CONTRIBUTING](../CONTRIBUTING.md) for more information. +from the project root and run any subsequent docker commands on the resultant image. ## Generate docker-compose.yml @@ -122,4 +115,5 @@ Error { code: "XMinioStorageFull", message: "Storage backend has reached its min ``` Solution: + This usually happens on MacOS with Docker Desktop. The Docker Deskup runs in the macOS Hypervisor. All the data, including logs, images, volumes, and so on, is stored in this hypervisor and the hypervisor has a default disk capacity limit. So when this message emerges, simply cleaning up the unused container or image can help mitigate. You can also increase capacity limit by clicking the Docker Desktop icon in the menu bar, then clicking Preferences > Resources > `Increase Disk image size`. diff --git a/docker/docker-compose-with-azblob.yml b/docker/docker-compose-with-azblob.yml index 99889d846dcb..d479f9464b5f 100644 --- a/docker/docker-compose-with-azblob.yml +++ b/docker/docker-compose-with-azblob.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-gcs.yml b/docker/docker-compose-with-gcs.yml index 80466c7cccab..6380b91a01a9 100644 --- a/docker/docker-compose-with-gcs.yml +++ b/docker/docker-compose-with-gcs.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-local-fs.yml b/docker/docker-compose-with-local-fs.yml index 68483796ac80..abfea87b5a68 100644 --- a/docker/docker-compose-with-local-fs.yml +++ b/docker/docker-compose-with-local-fs.yml @@ -21,21 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - # --parallelism 4 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-obs.yml b/docker/docker-compose-with-obs.yml index f4bf8dc0e74c..690da92d8b96 100644 --- a/docker/docker-compose-with-obs.yml +++ b/docker/docker-compose-with-obs.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-oss.yml b/docker/docker-compose-with-oss.yml index 7d9563473182..dab8af159299 100644 --- a/docker/docker-compose-with-oss.yml +++ b/docker/docker-compose-with-oss.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-s3.yml b/docker/docker-compose-with-s3.yml index d7dc75aa556a..c9d839220c94 100644 --- a/docker/docker-compose-with-s3.yml +++ b/docker/docker-compose-with-s3.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose-with-sqlite.yml b/docker/docker-compose-with-sqlite.yml index d4081b592c2a..98d88a415d49 100644 --- a/docker/docker-compose-with-sqlite.yml +++ b/docker/docker-compose-with-sqlite.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e315878c98b7..b7b29313547b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -21,22 +21,25 @@ services: --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ - #--parallelism 4 \ - #--total-memory-bytes 8589934592 \ + --parallelism 8 \ + --total-memory-bytes 21474836480 \ --role both \ - --meta-address http://0.0.0.0:5690\" \ + --meta-address http://0.0.0.0:5690 \ + --memory-manager-target-bytes 22333829939 \" \ --frontend-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ - --meta-addr http://0.0.0.0:5690\" \ + --meta-addr http://0.0.0.0:5690 \ + --frontend-total-memory-bytes=4294967296\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ - --meta-address http://0.0.0.0:5690\"" + --meta-address http://0.0.0.0:5690 \ + --compactor-total-memory-bytes=4294967296\"" expose: - "6660" - "4566" diff --git a/src/cmd_all/src/standalone.rs b/src/cmd_all/src/standalone.rs index 6eb62492999a..b5730d8d7845 100644 --- a/src/cmd_all/src/standalone.rs +++ b/src/cmd_all/src/standalone.rs @@ -391,17 +391,9 @@ It SHOULD NEVER be used in benchmarks and production environment!!!" #[cfg(test)] mod test { - use std::fmt::Debug; - - use expect_test::{expect, Expect}; - use super::*; - fn check(actual: impl Debug, expect: Expect) { - let actual = format!("{:#?}", actual); - expect.assert_eq(&actual); - } - + #[allow(clippy::assertions_on_constants)] #[test] fn test_parse_opt_args() { // Test parsing into standalone-level opts. @@ -426,93 +418,42 @@ mod test { // Test parsing into node-level opts. let actual = parse_standalone_opt_args(&opts); - check( - actual, - expect![[r#" - ParsedStandaloneOpts { - meta_opts: Some( - MetaNodeOpts { - listen_addr: "127.0.0.1:8001", - advertise_addr: "127.0.0.1:9999", - dashboard_host: None, - prometheus_listener_addr: Some( - "127.0.0.1:1234", - ), - sql_endpoint: None, - sql_username: "", - sql_password: [REDACTED alloc::string::String], - sql_database: "", - prometheus_endpoint: None, - prometheus_selector: None, - privatelink_endpoint_default_tags: None, - vpc_id: None, - security_group_id: None, - config_path: "src/config/test.toml", - backend: None, - barrier_interval_ms: None, - sstable_size_mb: None, - block_size_kb: None, - bloom_false_positive: None, - state_store: None, - data_directory: Some( - "some path with spaces", - ), - do_not_config_object_storage_lifecycle: None, - backup_storage_url: None, - backup_storage_directory: None, - heap_profiling_dir: None, - dangerous_max_idle_secs: None, - connector_rpc_endpoint: None, - license_key: None, - license_key_path: None, - temp_secret_file_dir: "./meta/secrets/", - }, - ), - compute_opts: Some( - ComputeNodeOpts { - listen_addr: "127.0.0.1:8000", - advertise_addr: None, - prometheus_listener_addr: "127.0.0.1:1234", - meta_address: List( - [ - http://127.0.0.1:5690/, - ], - ), - config_path: "src/config/test.toml", - total_memory_bytes: 34359738368, - reserved_memory_bytes: None, - parallelism: 10, - role: Both, - metrics_level: None, - data_file_cache_dir: None, - meta_file_cache_dir: None, - async_stack_trace: None, - heap_profiling_dir: None, - connector_rpc_endpoint: None, - temp_secret_file_dir: "./compute/secrets/", - }, - ), - frontend_opts: Some( - FrontendOpts { - listen_addr: "0.0.0.0:4566", - tcp_keepalive_idle_secs: 300, - advertise_addr: None, - meta_addr: List( - [ - http://127.0.0.1:5690/, - ], - ), - prometheus_listener_addr: "127.0.0.1:1234", - frontend_rpc_listener_addr: "127.0.0.1:6786", - config_path: "src/config/test.toml", - metrics_level: None, - enable_barrier_read: None, - temp_secret_file_dir: "./frontend/secrets/", - frontend_total_memory_bytes: 34359738368, - }, - ), - compactor_opts: None, - }"#]], - ); + if let Some(compute_opts) = &actual.compute_opts { + assert_eq!(compute_opts.listen_addr, "127.0.0.1:8000"); + assert_eq!(compute_opts.total_memory_bytes, 34359738368); + assert_eq!(compute_opts.parallelism, 10); + assert_eq!(compute_opts.temp_secret_file_dir, "./compute/secrets/"); + assert_eq!(compute_opts.prometheus_listener_addr, "127.0.0.1:1234"); + assert_eq!(compute_opts.config_path, "src/config/test.toml"); + } else { + assert!(false); + } + if let Some(meta_opts) = &actual.meta_opts { + assert_eq!(meta_opts.listen_addr, "127.0.0.1:8001"); + assert_eq!(meta_opts.advertise_addr, "127.0.0.1:9999"); + assert_eq!( + meta_opts.data_directory, + Some("some path with spaces".to_string()) + ); + assert_eq!(meta_opts.temp_secret_file_dir, "./meta/secrets/"); + assert_eq!( + meta_opts.prometheus_listener_addr, + Some("127.0.0.1:1234".to_string()) + ); + assert_eq!(meta_opts.config_path, "src/config/test.toml"); + } else { + assert!(false); + } + + if let Some(frontend_opts) = &actual.frontend_opts { + assert_eq!(frontend_opts.config_path, "src/config/test.toml"); + assert_eq!(frontend_opts.temp_secret_file_dir, "./frontend/secrets/"); + assert_eq!(frontend_opts.frontend_total_memory_bytes, 34359738368); + assert_eq!(frontend_opts.prometheus_listener_addr, "127.0.0.1:1234"); + } else { + assert!(false); + } + + assert!(actual.compactor_opts.is_none()); } } diff --git a/src/compute/src/lib.rs b/src/compute/src/lib.rs index 1336a84980ce..ef4b5c5e32d3 100644 --- a/src/compute/src/lib.rs +++ b/src/compute/src/lib.rs @@ -92,6 +92,13 @@ pub struct ComputeNodeOpts { #[clap(long, env = "RW_RESERVED_MEMORY_BYTES")] pub reserved_memory_bytes: Option, + /// Target memory usage for Memory Manager. + /// If not set, the default value is `total_memory_bytes` - `reserved_memory_bytes` + /// + /// It's strongly recommended to set it for standalone deployment. + #[clap(long, env = "RW_MEMORY_MANAGER_TARGET_BYTES")] + pub memory_manager_target_bytes: Option, + /// The parallelism that the compute node will register to the scheduler of the meta service. #[clap(long, env = "RW_PARALLELISM", default_value_t = default_parallelism())] #[override_opts(if_absent, path = streaming.actor_runtime_worker_threads_num)] diff --git a/src/compute/src/memory/controller.rs b/src/compute/src/memory/controller.rs index 2857b0e6b0d6..878471a7bedb 100644 --- a/src/compute/src/memory/controller.rs +++ b/src/compute/src/memory/controller.rs @@ -85,10 +85,10 @@ pub struct LruWatermarkController { impl LruWatermarkController { pub fn new(config: &MemoryManagerConfig) -> Self { - let threshold_stable = (config.total_memory as f64 * config.threshold_stable) as usize; - let threshold_graceful = (config.total_memory as f64 * config.threshold_graceful) as usize; + let threshold_stable = (config.target_memory as f64 * config.threshold_stable) as usize; + let threshold_graceful = (config.target_memory as f64 * config.threshold_graceful) as usize; let threshold_aggressive = - (config.total_memory as f64 * config.threshold_aggressive) as usize; + (config.target_memory as f64 * config.threshold_aggressive) as usize; Self { metrics: config.metrics.clone(), diff --git a/src/compute/src/memory/manager.rs b/src/compute/src/memory/manager.rs index b90624193c70..71b98100cb63 100644 --- a/src/compute/src/memory/manager.rs +++ b/src/compute/src/memory/manager.rs @@ -24,7 +24,9 @@ use risingwave_stream::executor::monitor::StreamingMetrics; use super::controller::LruWatermarkController; pub struct MemoryManagerConfig { - pub total_memory: usize, + /// [`MemoryManager`] will try to control the jemalloc-reported memory usage + /// to be lower than this + pub target_memory: usize, pub threshold_aggressive: f64, pub threshold_graceful: f64, diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs index da3328b0b2ce..8de74cd45645 100644 --- a/src/compute/src/server.rs +++ b/src/compute/src/server.rs @@ -288,15 +288,14 @@ pub async fn compute_node_serve( batch_mem_limit(compute_memory_bytes, opts.role.for_serving()), )); - // NOTE: Due to some limits, we use `compute_memory_bytes + storage_memory_bytes` as - // `total_compute_memory_bytes` for memory control. This is just a workaround for some - // memory control issues and should be modified as soon as we figure out a better solution. - // - // Related issues: - // - https://github.com/risingwavelabs/risingwave/issues/8696 - // - https://github.com/risingwavelabs/risingwave/issues/8822 + let target_memory = if let Some(v) = opts.memory_manager_target_bytes { + v + } else { + compute_memory_bytes + storage_memory_bytes + }; + let memory_mgr = MemoryManager::new(MemoryManagerConfig { - total_memory: compute_memory_bytes + storage_memory_bytes, + target_memory, threshold_aggressive: config .streaming .developer @@ -542,8 +541,7 @@ fn print_memory_config( reserved_memory_bytes: usize, ) { let memory_config = format!( - "\n\ - Memory outline:\n\ + "Memory outline:\n\ > total_memory: {}\n\ > storage_memory: {}\n\ > block_cache_capacity: {}\n\ diff --git a/src/frontend/src/lib.rs b/src/frontend/src/lib.rs index bf03edb6a87d..6424da42a151 100644 --- a/src/frontend/src/lib.rs +++ b/src/frontend/src/lib.rs @@ -162,7 +162,7 @@ pub struct FrontendOpts { )] pub temp_secret_file_dir: String, - /// Total available memory for the frontend node in bytes. Used by both computing and storage. + /// Total available memory for the frontend node in bytes. Used for batch computing. #[clap(long, env = "RW_FRONTEND_TOTAL_MEMORY_BYTES", default_value_t = default_frontend_total_memory_bytes())] pub frontend_total_memory_bytes: usize, } diff --git a/src/frontend/src/session.rs b/src/frontend/src/session.rs index fa4836e73e95..a920dc32653f 100644 --- a/src/frontend/src/session.rs +++ b/src/frontend/src/session.rs @@ -62,6 +62,7 @@ use risingwave_common::util::addr::HostAddr; use risingwave_common::util::cluster_limit; use risingwave_common::util::cluster_limit::ActorCountPerParallelism; use risingwave_common::util::iter_util::ZipEqFast; +use risingwave_common::util::pretty_bytes::convert; use risingwave_common::util::runtime::BackgroundShutdownRuntime; use risingwave_common::{GIT_SHA, RW_VERSION}; use risingwave_common_heap_profiling::HeapProfiler; @@ -450,9 +451,16 @@ impl FrontendEnv { // Run a background heap profiler heap_profiler.start(); - let mem_context = risingwave_common::memory::MemoryContext::root( + let batch_memory_limit = total_memory_bytes as f64 * FRONTEND_BATCH_MEMORY_PROPORTION; + let mem_context = MemoryContext::root( frontend_metrics.batch_total_mem.clone(), - (total_memory_bytes as f64 * FRONTEND_BATCH_MEMORY_PROPORTION) as u64, + batch_memory_limit as u64, + ); + + info!( + "Frontend total_memory: {} batch_memory: {}", + convert(total_memory_bytes as _), + convert(batch_memory_limit as _), ); Ok((