diff --git a/ci/scripts/run-e2e-test.sh b/ci/scripts/run-e2e-test.sh index 29f6c8fe2133d..c9a084396b296 100755 --- a/ci/scripts/run-e2e-test.sh +++ b/ci/scripts/run-e2e-test.sh @@ -27,7 +27,6 @@ if [[ $mode == "standalone" ]]; then fi if [[ $mode == "single-node" ]]; then - export RUST_MIN_STACK=4194304 source ci/scripts/single-node-utils.sh fi @@ -37,7 +36,7 @@ cluster_start() { risedev clean-data risedev pre-start-dev start_standalone "$PREFIX_LOG"/standalone.log & - risedev dev standalone-minio-etcd + risedev dev standalone-minio-sqlite elif [[ $mode == "single-node" ]]; then mkdir -p "$PREFIX_LOG" risedev clean-data @@ -257,7 +256,7 @@ if [[ "$mode" == "standalone" ]]; then risedev clean-data risedev pre-start-dev start_standalone_without_compactor "$PREFIX_LOG"/standalone.log & - risedev dev standalone-minio-etcd-compactor + risedev dev standalone-minio-sqlite-compactor wait_standalone if compactor_is_online then @@ -274,7 +273,7 @@ if [[ "$mode" == "standalone" ]]; then risedev clean-data risedev pre-start-dev start_standalone "$PREFIX_LOG"/standalone.log & - risedev dev standalone-minio-etcd + risedev dev standalone-minio-sqlite wait_standalone if ! compactor_is_online then diff --git a/ci/scripts/run-meta-backup-test.sh b/ci/scripts/run-meta-backup-test.sh index 098e4dfcd286c..69314b6958035 100755 --- a/ci/scripts/run-meta-backup-test.sh +++ b/ci/scripts/run-meta-backup-test.sh @@ -38,7 +38,7 @@ cluster_start() { risedev clean-data risedev pre-start-dev start_standalone "$PREFIX_LOG"/standalone.log & - risedev dev standalone-minio-etcd + risedev dev standalone-minio-sqlite elif [[ $mode == "single-node" ]]; then mkdir -p "$PREFIX_LOG" risedev clean-data diff --git a/ci/scripts/standalone-utils.sh b/ci/scripts/standalone-utils.sh index 059d11aada83f..63324039be652 100755 --- a/ci/scripts/standalone-utils.sh +++ b/ci/scripts/standalone-utils.sh @@ -5,6 +5,8 @@ set -euo pipefail export RW_PREFIX=$PWD/.risingwave export PREFIX_BIN=$RW_PREFIX/bin export PREFIX_LOG=$RW_PREFIX/log +export PREFIX_DATA=$RW_PREFIX/data +export RW_SQLITE_DB=$PREFIX_DATA/metadata.db # NOTE(kwannoel): Compared to start_standalone below, we omitted the compactor-opts, # so it should not start. @@ -16,8 +18,8 @@ start_standalone_without_compactor() { --advertise-addr 127.0.0.1:5690 \ --dashboard-host 127.0.0.1:5691 \ --prometheus-host 127.0.0.1:1250 \ - --backend etcd \ - --etcd-endpoints 127.0.0.1:2388 \ + --backend sqlite \ + --sql-endpoint sqlite://${RW_SQLITE_DB} \ --state-store hummock+minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001 \ --data-directory hummock_001" \ --compute-opts=" \ @@ -39,7 +41,6 @@ start_standalone_without_compactor() { # You can fill up this section by consulting # .risingwave/log/risedev.log, after calling `risedev d full`. -# It is expected that minio, etcd will be started after this is called. start_standalone() { RUST_BACKTRACE=1 \ "$PREFIX_BIN"/risingwave/standalone \ @@ -48,8 +49,8 @@ start_standalone() { --advertise-addr 127.0.0.1:5690 \ --dashboard-host 127.0.0.1:5691 \ --prometheus-host 127.0.0.1:1250 \ - --backend etcd \ - --etcd-endpoints 127.0.0.1:2388 \ + --backend sqlite \ + --sql-endpoint sqlite://${RW_SQLITE_DB} \ --state-store hummock+minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001 \ --data-directory hummock_001" \ --compute-opts=" \ diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml index 140b43ed5ca21..a2f75bc17b8ec 100644 --- a/ci/workflows/main-cron.yml +++ b/ci/workflows/main-cron.yml @@ -79,7 +79,7 @@ steps: config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 15 + timeout_in_minutes: 25 retry: *auto-retry - label: "slow end-to-end test (release)" diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml index ddba0f82f95fd..81659354f9b9c 100644 --- a/ci/workflows/pull-request.yml +++ b/ci/workflows/pull-request.yml @@ -99,7 +99,7 @@ steps: config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 23 + timeout_in_minutes: 30 retry: *auto-retry - label: "slow end-to-end test" @@ -154,7 +154,7 @@ steps: config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 17 + timeout_in_minutes: 25 retry: *auto-retry - label: "end-to-end test for opendal (parallel)" diff --git a/docs/dev/src/connector/intro.md b/docs/dev/src/connector/intro.md index e9903175c5f61..d5460eae7b0b0 100644 --- a/docs/dev/src/connector/intro.md +++ b/docs/dev/src/connector/intro.md @@ -1,6 +1,7 @@ # Develop Connectors This page describes the development workflow to develop connectors. For design docs, see + - [Source](./source.md) RisingWave supports a lot of connectors (sources and sinks). @@ -11,16 +12,21 @@ However, developing connectors is tricky because it involves external systems: - The test relies on the configuration of the setup. e.g., the test needs to know the port of your Kafka in order to - We need to do the setup for both CI and local development. -Our solution is: we resort to RiseDev, our all-in-one development tool, to help manage external systems and solve these problems. +Our solution is: we resort to RiseDev, our all-in-one development tool, to help manage external systems and solve these +problems. Before going to the specific methods described in the sections below, the principles we should follow are: -- *environment-independent*: It should easy to start cluster and run tests on your own machine, other developers' machines, and CI. + +- *environment-independent*: It should easy to start cluster and run tests on your own machine, other developers' + machines, and CI. * Don't use hard-coded configurations (e.g., `localhost:9092` for Kafka). * Don't write too many logic in `ci/scripts`. Let CI scripts be thin wrappers. -- *self-contained* tests: It should be straightforward to run one test case, without worrying about where is the script to prepare the test. +- *self-contained* tests: It should be straightforward to run one test case, without worrying about where is the script + to prepare the test. * Don't put setup logic, running logic and verification logic of a test in different places. -Reference: for the full explanations of the difficulies and the design of our solution, see [here](https://github.com/risingwavelabs/risingwave/issues/12451#issuecomment-2051861048). +Reference: for the full explanations of the difficulies and the design of our solution, +see [here](https://github.com/risingwavelabs/risingwave/issues/12451#issuecomment-2051861048). The following sections first walk you through what is the development workflow for existing connectors, and finally explain how to extend the development framework to support a new connector. @@ -29,18 +35,20 @@ existing connectors, and finally explain how to extend the development framework ## Set up the development environment -RiseDev supports starting external connector systems (e.g., Kafka, MySQL) just like how it starts the RisingWave cluster, and other standard systems used as part of the RisingWave Cluster (e.g., MinIO, etcd, Grafana). +RiseDev supports starting external connector systems (e.g., Kafka, MySQL) just like how it starts the RisingWave +cluster, and other standard systems used as part of the RisingWave Cluster (e.g., MinIO, etcd, Grafana). -You write the profile in `risedev.yml` (Or `risedev-profiles.user.yml` ), e.g., the following config includes Kafka and MySQL, which will be used to test sources. +You write the profile in `risedev.yml` (Or `risedev-profiles.user.yml` ), e.g., the following config includes Kafka and +MySQL, which will be used to test sources. ```yml my-cool-profile: steps: # RisingWave cluster - use: minio - - use: etcd + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node - use: frontend - use: compactor @@ -66,9 +74,12 @@ For all config options of supported systems, check the comments in `template` se ### Escape hatch: `user-managed` mode -`user-managed` is a special config. When set to `true` , you will need to start the system by yourself. You may wonder why bother to add it to the RiseDev profile if you start it by yourself. In this case, the config will still be loaded by RiseDev, which will be useful in tests. See chapters below for more details. +`user-managed` is a special config. When set to `true` , you will need to start the system by yourself. You may wonder +why bother to add it to the RiseDev profile if you start it by yourself. In this case, the config will still be loaded +by RiseDev, which will be useful in tests. See chapters below for more details. -The `user-managed` mode can be used as a workaround to start a system that is not yet supported by RiseDev, or is buggy. It's also used to config the CI profile. (In CI, all services are pre-started by `ci/docker-compose.yml` ) +The `user-managed` mode can be used as a workaround to start a system that is not yet supported by RiseDev, or is buggy. +It's also used to config the CI profile. (In CI, all services are pre-started by `ci/docker-compose.yml` ) Example of the config: @@ -82,18 +93,23 @@ Example of the config: ## End-to-end tests The e2e tests are written in `slt` files. There are 2 main points to note: + 1. Use `system ok` to run `bash` commands to interact with external systems. Use this to prepare the test data, and verify the results. The whole lifecycle of a test case should be written in the same `slt` file. -2. Use `control substitution on` and then use environment variables to specify the config of the external systems, e.g., the port of Kafka. +2. Use `control substitution on` and then use environment variables to specify the config of the external systems, e.g., + the port of Kafka. -Refer to the [sqllogictest-rs documentation](https://github.com/risinglightdb/sqllogictest-rs#extension-run-external-shell-commands) for the details of `system` and `substitution` . +Refer to +the [sqllogictest-rs documentation](https://github.com/risinglightdb/sqllogictest-rs#extension-run-external-shell-commands) +for the details of `system` and `substitution` . --- Take Kafka as an example about how to the tests are written: -When you use `risedev d` to start the external services, related environment variables for Kafka will be available when you run `risedev slt`: +When you use `risedev d` to start the external services, related environment variables for Kafka will be available when +you run `risedev slt`: ```sh RISEDEV_KAFKA_BOOTSTRAP_SERVERS="127.0.0.1:9092" @@ -129,16 +145,20 @@ create source s0 (v1 int, v2 varchar) with ( See `src/risedevtool/src/risedev_env.rs` for variables supported for each service. -> Note again: You need to use `risedev d` to start the cluster, and then use `risedev slt` to run the tests. It doesn't work if you start the cluster by yourself without telling RiseDev, or you use raw `sqllogictest` binary directly. +> Note again: You need to use `risedev d` to start the cluster, and then use `risedev slt` to run the tests. It doesn't +> work if you start the cluster by yourself without telling RiseDev, or you use raw `sqllogictest` binary directly. > > How it works: `risedev d` will write env vars to `.risingwave/config/risedev-env`, > and `risedev slt` will load env vars from this file. ### Tips for writing `system` commands -Refer to the [sqllogictest-rs documentation](https://github.com/risinglightdb/sqllogictest-rs#extension-run-external-shell-commands) for the syntax. +Refer to +the [sqllogictest-rs documentation](https://github.com/risinglightdb/sqllogictest-rs#extension-run-external-shell-commands) +for the syntax. For simple cases, you can directly write a bash command, e.g., + ``` system ok mysql -e " @@ -156,13 +176,23 @@ cat << EOF | rpk topic produce my_source -f "%p %v\n" -p 0 EOF ``` -For more complex cases, you can write a test script, and invoke it in `slt`. Scripts can be written in any language you like, but kindly write a `README.md` to help other developers get started more easily. +For more complex cases, you can write a test script, and invoke it in `slt`. Scripts can be written in any language you +like, but kindly write a `README.md` to help other developers get started more easily. + - For ad-hoc scripts (only used for one test), it's better to put next to the test file. - e.g., [`e2e_test/source_inline/kafka/consumer_group.mjs`](https://github.com/risingwavelabs/risingwave/blob/c22c4265052c2a4f2876132a10a0b522ec7c03c9/e2e_test/source_inline/kafka/consumer_group.mjs), which is invoked by [`consumer_group.slt`](https://github.com/risingwavelabs/risingwave/blob/c22c4265052c2a4f2876132a10a0b522ec7c03c9/e2e_test/source_inline/kafka/consumer_group.slt) next to it. -- For general scripts that can be used under many situations, put it in `e2e_test/commands/`. This directory will be loaded in `PATH` by `risedev slt`, and thus function as kind of "built-in" commands. + e.g., [`e2e_test/source_inline/kafka/consumer_group.mjs`](https://github.com/risingwavelabs/risingwave/blob/c22c4265052c2a4f2876132a10a0b522ec7c03c9/e2e_test/source_inline/kafka/consumer_group.mjs), + which is invoked + by [`consumer_group.slt`](https://github.com/risingwavelabs/risingwave/blob/c22c4265052c2a4f2876132a10a0b522ec7c03c9/e2e_test/source_inline/kafka/consumer_group.slt) + next to it. +- For general scripts that can be used under many situations, put it in `e2e_test/commands/`. This directory will be + loaded in `PATH` by `risedev slt`, and thus function as kind of "built-in" commands. - A common scenario is when a CLI tool does not accept envvars as arguments. In such cases, instead of manually specifying the arguments each time invoking it in `slt`, you can create a wrapper to handle this implicitly, making it more concise. [`e2e_test/commands/mysql`](https://github.com/risingwavelabs/risingwave/blob/c22c4265052c2a4f2876132a10a0b522ec7c03c9/e2e_test/commands/mysql) is a good demonstration. + A common scenario is when a CLI tool does not accept envvars as arguments. In such cases, instead of manually + specifying the arguments each time invoking it in `slt`, you can create a wrapper to handle this implicitly, making it + more + concise. [`e2e_test/commands/mysql`](https://github.com/risingwavelabs/risingwave/blob/c22c4265052c2a4f2876132a10a0b522ec7c03c9/e2e_test/commands/mysql) + is a good demonstration. --- Tips for debugging: @@ -176,14 +206,16 @@ Tips for debugging: placeholder ``` - Then running `risedev slt` will return error "result mismatch", and shows what's the output - of the `echo` command, i.e., the value of `PGPORT`. + Then running `risedev slt` will return error "result mismatch", and shows what's the output + of the `echo` command, i.e., the value of `PGPORT`. -- Use `risedev show-risedev-env` to see the environment variables available for `risedev slt`, after you starting the cluster with `risedev d`. +- Use `risedev show-risedev-env` to see the environment variables available for `risedev slt`, after you starting the + cluster with `risedev d`. ## Adding a new connector to the development framework -Refer to [#16449](https://github.com/risingwavelabs/risingwave/pull/16449) ( `user-managed` only MySQL), and [#16514](https://github.com/risingwavelabs/risingwave/pull/16514) (Docker based MySQL) as examples. +Refer to [#16449](https://github.com/risingwavelabs/risingwave/pull/16449) ( `user-managed` only MySQL), +and [#16514](https://github.com/risingwavelabs/risingwave/pull/16514) (Docker based MySQL) as examples. 1. Add a new service in `template` section of `risedev.yml`. And add corresponding config in `src/risedevtool/src/service_config.rs` . diff --git a/e2e_test/ddl/secret.slt b/e2e_test/ddl/secret.slt index 62a5a09add668..6522565986a18 100644 --- a/e2e_test/ddl/secret.slt +++ b/e2e_test/ddl/secret.slt @@ -1,6 +1,8 @@ statement ok ALTER SYSTEM SET license_key TO ''; +sleep 1s + statement error create secret secret_1 with ( backend = 'fake-backend' diff --git a/risedev.yml b/risedev.yml index 78b835dad973b..b021737449c1f 100644 --- a/risedev.yml +++ b/risedev.yml @@ -154,13 +154,13 @@ profile: - use: kafka persist-data: true - standalone-minio-etcd: + standalone-minio-sqlite: steps: - use: minio - - use: etcd + - use: sqlite - use: meta-node user-managed: true - meta-backend: etcd + meta-backend: sqlite - use: compute-node user-managed: true - use: frontend @@ -168,13 +168,13 @@ profile: - use: compactor user-managed: true - standalone-minio-etcd-compactor: + standalone-minio-sqlite-compactor: steps: - use: minio - - use: etcd + - use: sqlite - use: meta-node user-managed: true - meta-backend: etcd + meta-backend: sqlite - use: compute-node user-managed: true - use: frontend @@ -277,9 +277,9 @@ profile: full-benchmark: steps: - use: minio - - use: etcd + - use: postgres - use: meta-node - meta-backend: etcd + meta-backend: postgres - use: compute-node - use: frontend - use: compactor @@ -291,78 +291,6 @@ profile: - use: kafka persist-data: true - 3etcd-3meta: - steps: - - use: minio - - use: etcd - unsafe-no-fsync: true - port: 2388 - peer-port: 2389 - exporter-port: 2379 - - use: etcd - unsafe-no-fsync: true - port: 12388 - peer-port: 12389 - exporter-port: 12379 - - use: etcd - unsafe-no-fsync: true - port: 22388 - peer-port: 22389 - exporter-port: 22379 - - use: meta-node - port: 5690 - dashboard-port: 5691 - exporter-port: 1250 - meta-backend: etcd - - use: meta-node - port: 15690 - dashboard-port: 15691 - exporter-port: 11250 - meta-backend: etcd - - use: meta-node - port: 25690 - dashboard-port: 25691 - exporter-port: 21250 - meta-backend: etcd - - use: compactor - - 3etcd-3meta-1cn-1fe: - steps: - - use: minio - - use: etcd - unsafe-no-fsync: true - port: 2388 - peer-port: 2389 - exporter-port: 2379 - - use: etcd - unsafe-no-fsync: true - port: 12388 - peer-port: 12389 - exporter-port: 12379 - - use: etcd - unsafe-no-fsync: true - port: 22388 - peer-port: 22389 - exporter-port: 22379 - - use: meta-node - port: 5690 - dashboard-port: 5691 - exporter-port: 1250 - meta-backend: etcd - - use: meta-node - port: 15690 - dashboard-port: 15691 - exporter-port: 11250 - meta-backend: etcd - - use: meta-node - port: 25690 - dashboard-port: 25691 - exporter-port: 21250 - meta-backend: etcd - - use: compactor - - use: compute-node - - use: frontend - meta-1cn-1fe-sqlite: steps: - use: minio @@ -492,9 +420,9 @@ profile: # NOTE(kwannoel): We do not use aws-s3 here, to avoid # contention over s3 bucket when multiple benchmarks at run at once. - use: minio - - use: etcd + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node parallelism: 8 - use: frontend @@ -650,10 +578,9 @@ profile: config-path: src/config/ci.toml steps: - use: minio - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node enable-tiered-cache: true - use: frontend @@ -663,10 +590,9 @@ profile: config-path: src/config/ci.toml steps: - use: minio - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node port: 5687 exporter-port: 1222 @@ -709,10 +635,9 @@ profile: config-path: src/config/ci-longer-streaming-upload-timeout.toml steps: - use: minio - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node port: 5687 exporter-port: 1222 @@ -736,10 +661,9 @@ profile: - use: minio api-requests-max: 30 api-requests-deadline: 3s - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node port: 5687 exporter-port: 1222 @@ -761,10 +685,9 @@ profile: - use: minio api-requests-max: 30 api-requests-deadline: 2s - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node port: 5687 exporter-port: 1222 @@ -786,10 +709,9 @@ profile: config-path: src/config/ci.toml steps: - use: minio - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node port: 5687 exporter-port: 1222 @@ -846,10 +768,9 @@ profile: ci-3cn-3fe-opendal-fs-backend: config-path: src/config/ci.toml steps: - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: opendal engine: fs bucket: "/tmp/rw_ci" @@ -880,10 +801,9 @@ profile: config-path: src/config/ci.toml steps: - use: minio - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node port: 5687 exporter-port: 1222 @@ -932,10 +852,9 @@ profile: config-path: src/config/ci.toml steps: - use: minio - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node enable-tiered-cache: true - use: frontend @@ -953,10 +872,9 @@ profile: config-path: src/config/ci-recovery.toml steps: - use: minio - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node enable-tiered-cache: true - use: frontend @@ -982,10 +900,9 @@ profile: config-path: src/config/ci.toml steps: - use: minio - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node enable-tiered-cache: true - use: frontend @@ -996,10 +913,9 @@ profile: config-path: src/config/ci-compaction-test.toml steps: - use: minio - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node enable-tiered-cache: true total-memory-bytes: 17179869184 @@ -1010,9 +926,9 @@ profile: config-path: src/config/ci-recovery.toml steps: - use: minio - - use: etcd + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node enable-tiered-cache: true - use: frontend @@ -1022,10 +938,9 @@ profile: config-path: src/config/ci-recovery.toml steps: - use: minio - - use: etcd - unsafe-no-fsync: true + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node port: 5687 exporter-port: 1222 @@ -1045,9 +960,9 @@ profile: config-path: src/config/ci-recovery.toml steps: - use: minio - - use: etcd + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node enable-tiered-cache: true - use: frontend @@ -1129,9 +1044,9 @@ profile: config-path: "src/config/ci-backfill.toml" steps: - use: minio - - use: etcd + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node - use: frontend - use: compactor @@ -1141,9 +1056,9 @@ profile: steps: - use: aws-s3 bucket: renjie-iceberg-bench - - use: etcd + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node - use: frontend - use: compactor @@ -1154,9 +1069,9 @@ profile: config-path: src/config/full-with-batch-query-limit.toml steps: - use: minio - - use: etcd + - use: sqlite - use: meta-node - meta-backend: etcd + meta-backend: sqlite - use: compute-node - use: frontend - use: compactor diff --git a/src/cmd_all/scripts/standalone-demo-full.sh b/src/cmd_all/scripts/standalone-demo-full.sh index e555dd9ced82d..5f4966ac7aee2 100755 --- a/src/cmd_all/scripts/standalone-demo-full.sh +++ b/src/cmd_all/scripts/standalone-demo-full.sh @@ -5,6 +5,8 @@ set -euo pipefail RW_PREFIX=$PWD/.risingwave PREFIX_BIN=$RW_PREFIX/bin PREFIX_LOG=$RW_PREFIX/log +PREFIX_DATA=$RW_PREFIX/data +RW_SQLITE_DB=$PREFIX_DATA/metadata.db start_standalone() { RUST_BACKTRACE=1 \ @@ -17,8 +19,8 @@ start_standalone() { --advertise-addr 127.0.0.1:5690 \ --dashboard-host 127.0.0.1:5691 \ --prometheus-host 127.0.0.1:1250 \ - --backend etcd \ - --etcd-endpoints 127.0.0.1:2388 \ + --backend sqlite \ + --sql-endpoint sqlite://${RW_SQLITE_DB} \ --state-store hummock+minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001 \ --data-directory hummock_001 \ --config-path src/config/standalone-example.toml" \ diff --git a/src/meta/node/src/server.rs b/src/meta/node/src/server.rs index 6d190db3b2efc..965ccb5a6727a 100644 --- a/src/meta/node/src/server.rs +++ b/src/meta/node/src/server.rs @@ -201,19 +201,20 @@ pub async fn rpc_serve( .await } MetaStoreBackend::Sql { endpoint } => { - let max_connection = if DbBackend::Sqlite.is_prefix_of(&endpoint) { - // Due to the fact that Sqlite is prone to the error "(code: 5) database is locked" under concurrent access, - // here we forcibly specify the number of connections as 1. - 1 - } else { - 10 - }; - + let is_sqlite = DbBackend::Sqlite.is_prefix_of(&endpoint); let mut options = sea_orm::ConnectOptions::new(endpoint); options - .max_connections(max_connection) + .max_connections(10) .connect_timeout(Duration::from_secs(10)) - .idle_timeout(Duration::from_secs(30)); + .idle_timeout(Duration::from_secs(30)) + .acquire_timeout(Duration::from_secs(30)); + + if is_sqlite { + // Since Sqlite is prone to the error "(code: 5) database is locked" under concurrent access, + // here we forcibly specify the number of connections as 1. + options.max_connections(1); + } + let conn = sea_orm::Database::connect(options).await?; let meta_store_sql = SqlMetaStore::new(conn);