Merge branch 'yiming/uploader-no-global-epoch' into yiming/dag-uploader

.cargo/config.toml

@@ -32,3 +32,11 @@ rustflags = [
 # Flags for all targets.
 [target.'cfg(all())']
 rustflags = ["--cfg", "tokio_unstable"]
+
+# We have large git dependencies. This can make cloning faster.
+# https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#git
+# Actually we also want to prevent submodule cloning completely
+# https://github.com/rust-lang/cargo/issues/4247
+[unstable.git]
+shallow_index = true
+shallow_deps = true

.github/workflows/doc.yml

@@ -25,28 +25,28 @@ jobs:
           remove-docker-images: 'true'
           root-reserve-mb: 10240
           temp-reserve-mb: 10240
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Rust toolchain
         run: rustup show
-      - name: Install dependencies
+      - name: Install dependencies for compiling RisingWave
         run: sudo apt-get update && sudo apt-get install -y make build-essential cmake protobuf-compiler curl openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config postgresql-client tmux lld
       - name: Run sccache-cache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].5
         with:
           version: "v0.5.2"
       - name: build rustdocs
         run: |
           cargo doc --workspace --no-deps --document-private-items
           cp docs/rustdoc/rust.css target/doc/rust.css
 
-          mkdir artifact
-          cp -R target/doc/* artifact
+          mkdir -p artifact/rustdoc
+          cp -R target/doc/* artifact/rustdoc
       - name: Show available storage
         run: df -h
-      - name: Install cargo-docset
+      - name: Install tools for building docs
         uses: taiki-e/install-action@v2
         with:
-          tool: cargo-docset
+          tool: cargo-docset,mdbook,mdbook-toc,mdbook-linkcheck
       - name: build docsets
         run: |
           cargo docset --no-clean --docset-name RisingWave
@@ -58,8 +58,13 @@ jobs:
           </entry>" > RisingWave.xml
 
           cp -t artifact "risingwave.docset.tgz" "RisingWave.xml"
+      - name: build developer doc
+        run: |
+          cd docs/dev
+          mdbook build
+          cp -R book/html/* ../../artifact
       - name: Upload artifacts
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: artifact
       - name: Show available storage
@@ -76,4 +81,4 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v4

.gitignore

@@ -81,3 +81,8 @@ e2e_test/iceberg/spark-*-bin*
 **/poetry.lock
 
 *.slt.temp
+
+.direnv/
+
+# mdbook
+book

.typos.toml

@@ -25,6 +25,7 @@ extend-exclude = [
     "src/sqlparser/tests/testdata/",
     "src/frontend/planner_test/tests/testdata",
     "src/tests/sqlsmith/tests/freeze",
+    "src/license/src/manager.rs",
     "Cargo.lock",
     "**/Cargo.toml",
     "**/go.mod",

CONTRIBUTING.md

@@ -1,72 +1,16 @@
-# Contribution guidelines
+# Contributing to RisingWave
 
-Thanks for your interest in contributing to RisingWave! We welcome and appreciate contributions.
+Contributing documentation has moved to the **[RisingWave Developer Guide](https://risingwavelabs.github.io/risingwave/)**.
 
-This document describes how to submit your code changes. To learn about the development process, see the [developer guide](docs/developer-guide.md). To understand the design and implementation of RisingWave, refer to the design docs listed in [docs/README.md](docs/README.md).
 
-If you have questions, you can search for existing discussions or start a new discussion in the [Discussions forum of RisingWave](https://github.com/risingwavelabs/risingwave/discussions), or ask in the RisingWave Community channel on Slack. Please use the [invitation link](https://risingwave.com/slack) to join the channel.
+## Before hacking on RisingWave
 
-To report bugs, create a [GitHub issue](https://github.com/risingwavelabs/risingwave/issues/new/choose).
+We encourage people to discuss their design before hacking on code. Typically,
+you [file an issue] or start a thread on the [Community Slack] before submitting
+a pull request.
 
+Please read [the process] of how to submit your change to RisingWave.
 
-## Table of contents
-
-- [Contribution guidelines](#contribution-guidelines)
-  - [Table of contents](#table-of-contents)
-  - [Tests and miscellaneous checks](#tests-and-miscellaneous-checks)
-  - [Submit a PR](#submit-a-pr)
-    - [Pull Request title](#pull-request-title)
-    - [Pull Request description](#pull-request-description)
-    - [Sign the CLA](#sign-the-cla)
-    - [Cherry pick the commit to release candidate branch](#cherry-pick-the-commit-to-release-candidate-branch)
-
-## Tests and miscellaneous checks
-
-Before submitting your code changes, ensure you fully test them and perform necessary checks. The testing instructions and necessary checks are detailed in the [developer guide](docs/developer-guide.md#test-your-code-changes).
-
-## Submit a PR
-
-### Pull Request title
-
-As described in [here](https://github.com/commitizen/conventional-commit-types/blob/master/index.json), a valid PR title should begin with one of the following prefixes:
-
-- `feat`: A new feature
-- `fix`: A bug fix
-- `doc`: Documentation only changes
-- `refactor`: A code change that neither fixes a bug nor adds a feature
-- `style`: A refactoring that improves code style
-- `perf`: A code change that improves performance
-- `test`: Adding missing tests or correcting existing tests
-- `build`: Changes that affect the build system or external dependencies (example scopes: `.config`, `.cargo`, `Cargo.toml`)
-- `ci`: Changes to RisingWave CI configuration files and scripts (example scopes: `.github`, `ci` (Buildkite))
-- `chore`: Other changes that don't modify src or test files
-- `revert`: Reverts a previous commit
-
-For example, a PR title could be:
-
-- `refactor: modify executor protobuf package path`
-- `feat(execution): enable comparison between nullable data arrays`, where `(execution)` means that this PR mainly focuses on the execution component.
-
-You may also check out previous PRs in the [PR list](https://github.com/risingwavelabs/risingwave/pulls).
-
-### Pull Request description
-
-- If your PR is small (such as a typo fix), you can go brief.
-- If it is large and you have changed a lot, it's better to write more details.
-
-### Sign the CLA
-
-Contributors will need to sign RisingWave Labs' CLA.
-
-### Cherry pick the commit to release candidate branch
-We have a GitHub Action to help cherry-pick commits from `main` branch to a `release candidate` branch, such as `v*.*.*-rc` where `*` is a number.
-
-Checkout details at: https://github.com/risingwavelabs/risingwave/blob/main/.github/workflows/cherry-pick-to-release-branch.yml
-
-To trigger the action, we give a correct label to the PR on `main` branch :
-https://github.com/risingwavelabs/risingwave/blob/main/.github/workflows/cherry-pick-to-release-branch.yml#L10
-
-It will act when the PR on `main` branch merged:
-- If `git cherry-pick` does not find any conflicts, it will open a PR to the `release candidate` branch, and assign the original author as the reviewer.
-
-- If there is a conflict, it will open an issue and make the original author the assignee.
+[Community Slack]: https://risingwave.com/slack
+[file an issue]: https://github.com/risingwavelabs/risingwave/issues/new/choose
+[the process]: https://risingwavelabs.github.io/risingwave/contribution

Cargo.toml

@@ -25,6 +25,7 @@ members = [
     "src/frontend/planner_test",
     "src/java_binding",
     "src/jni_core",
+    "src/license",
     "src/meta",
     "src/meta/dashboard",
     "src/meta/model_v2",
@@ -217,6 +218,7 @@ risingwave_frontend = { path = "./src/frontend" }
 risingwave_hummock_sdk = { path = "./src/storage/hummock_sdk" }
 risingwave_hummock_test = { path = "./src/storage/hummock_test" }
 risingwave_hummock_trace = { path = "./src/storage/hummock_trace" }
+risingwave_license = { path = "./src/license" }
 risingwave_mem_table_spill_test = { path = "./src/stream/spill_test" }
 risingwave_meta = { path = "./src/meta" }
 risingwave_meta_dashboard = { path = "./src/meta/dashboard" }

Makefile.toml

@@ -1294,7 +1294,7 @@ echo "All processes has exited."
 
 [tasks.slt]
 category = "RiseDev - Test - SQLLogicTest"
-install_crate = { min_version = "0.20.6", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
+install_crate = { min_version = "0.21.0", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
     "--help",
 ], install_command = "binstall" }
 dependencies = ["check-and-load-risedev-env-file"]

ci/Dockerfile

@@ -70,7 +70,7 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
     [email protected] \
-    sqllogictest-bin@0.20.6 \
+    sqllogictest-bin@0.21.0 \
     [email protected] \
     && cargo cache -a \
     && rm -rf "/root/.cargo/registry/index" \

ci/build-ci-image.sh

@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240624
+export BUILD_ENV_VERSION=v20240628
 
 export BUILD_TAG="public.ecr.aws/w1p7b4n3/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240624
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240628
     depends_on:
       - mysql
       - db
@@ -84,7 +84,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240624
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240628
     depends_on:
       - mysql
       - db
@@ -107,12 +107,12 @@ services:
 
 
   rw-build-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240624
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240628
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240624
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240628
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -123,7 +123,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240624
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240628
     depends_on:
       db:
         condition: service_healthy

ci/scripts/deterministic-e2e-test.sh

@@ -49,7 +49,7 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, batch"
 seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation -j 16 ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/parallel-batch-{}.log && rm $LOGDIR/parallel-batch-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (pre-generated-queries)"
-timeout 10m seq 64 | parallel MADSIM_TEST_SEED={} './risingwave_simulation  --run-sqlsmith-queries ./src/tests/sqlsmith/tests/sqlsmith-query-snapshots/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
+timeout 10m seq 64 | parallel MADSIM_TEST_SEED={} RUST_MIN_STACK=4194304 './risingwave_simulation  --run-sqlsmith-queries ./src/tests/sqlsmith/tests/sqlsmith-query-snapshots/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, e2e extended mode test"
 seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation -e 2> $LOGDIR/extended-{}.log && rm $LOGDIR/extended-{}.log'

ci/scripts/run-e2e-test.sh

@@ -80,9 +80,6 @@ RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=i
 cluster_start
 # Please make sure the regression is expected before increasing the timeout.
 sqllogictest -p 4566 -d dev './e2e_test/streaming/**/*.slt' --junit "streaming-${profile}"
-if [[ "$mode" != "single-node" ]]; then
-  sqllogictest -p 4566 -d dev './e2e_test/streaming_now/**/*.slt' --junit "streaming-${profile}"
-fi
 sqllogictest -p 4566 -d dev './e2e_test/backfill/sink/different_pk_and_dist_key.slt'
 
 echo "--- Kill cluster"