Merge branch 'main' into xxchan/latest-nightly-rust

Makefile.toml

@@ -90,9 +90,9 @@ if ${is_not_ci}
   no_rust_log = not ${rust_log}
 
   if ${no_rust_log}
-    set_env RUST_LOG "pgwire_query_log=info,hyper::client::connect::http=info"
+    set_env RUST_LOG "pgwire_query_log=info"
   else
-    set_env RUST_LOG "pgwire_query_log=info,hyper::client::connect::http=info,${rust_log}"
+    set_env RUST_LOG "pgwire_query_log=info,${rust_log}"
   end
 end
 
@@ -128,6 +128,17 @@ rm -rf "${PREFIX_CONFIG}"
 rm -rf "${PREFIX_PROFILING}"
 '''
 
+[tasks.reset-rw]
+category = "RiseDev - Start/Stop"
+description = "Clean all data in the default database dev of the running RisingWave"
+dependencies = ["check-risedev-env-file"]
+env_files = ["${PREFIX_CONFIG}/risedev-env"]
+script = '''
+#!/usr/bin/env bash
+psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev -c "CREATE DATABASE risedev_tmp;"
+psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d risedev_tmp -c "DROP DATABASE dev; CREATE DATABASE dev;"
+psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev -c "DROP DATABASE risedev_tmp;"
+'''
 
 [tasks.l]
 alias = "logs"

ci/scripts/integration-tests.sh

@@ -24,17 +24,17 @@ shift $((OPTIND -1))
 
 echo "export INTEGRATION_TEST_CASE=${case}" > env_vars.sh
 
-echo "--- clean up docker"
+echo "~~~ clean up docker"
 if [ $(docker ps -aq |wc -l) -gt 0 ]; then
   docker rm -f $(docker ps -aq)
 fi
 docker network prune -f
 docker volume prune -f
 
-echo "--- ghcr login"
+echo "~~~ ghcr login"
 echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
 
-echo "--- case: ${case}, format: ${format}"
+echo "+++ set RW_IMAGE"
 
 if [[ -n "${RW_IMAGE_TAG+x}" ]]; then
   export RW_IMAGE="ghcr.io/risingwavelabs/risingwave:${RW_IMAGE_TAG}"
@@ -47,18 +47,24 @@ if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
   echo Docker image: "$RW_IMAGE"
 fi
 
+if [ -z "${RW_IMAGE+x}" ]; then
+  echo "RW_IMAGE is not set. The image defined in docker-compose.yml will be used."
+fi
+
+echo "--- case: ${case}, format: ${format}"
+
 if [ "${case}" == "client-library" ]; then
   cd integration_tests/client-library
   python3 client_test.py
   exit 0
 fi
 
-echo "--- install postgresql"
+echo "~~~ install postgresql"
 sudo yum install -y postgresql15
 
-echo "--- install poetry"
+echo "~~~ install poetry"
 curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.0 python3 -
-
+export PATH=$PATH:$HOME/.local/bin
 
 
 echo "--- download rwctest-key"

ci/scripts/run-e2e-test.sh

@@ -141,9 +141,9 @@ sqllogictest -p 4566 -d dev './e2e_test/generated/**/*.slt' --junit "generated-$
 echo "--- Kill cluster"
 cluster_stop
 
-echo "--- e2e, $mode, error ui"
+echo "--- e2e, ci-3cn-1fe-with-recovery, error ui"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
-cluster_start
+risedev ci-start ci-3cn-1fe-with-recovery
 sqllogictest -p 4566 -d dev './e2e_test/error_ui/simple/**/*.slt'
 sqllogictest -p 4566 -d dev -e postgres-extended './e2e_test/error_ui/extended/**/*.slt'
 

docs/aggregation.md

@@ -0,0 +1,49 @@
+# Aggregations
+
+We will cover internal implementation of common aggregations in this document.
+
+
+## Frontend
+
+TODO
+
+## Expression Framework
+
+TODO
+
+## HashAggExecutor
+
+![aggregation components](./images/aggregation/agg-components.png)
+
+Within the `HashAggExecutor`, there are 4 main components:
+1. AggCalls.
+2. AggState.
+3. AggGroups.
+4. Persisted State.
+
+AggCalls are the aggregation calls for the query. For instance `SUM(v1)`, `COUNT(v2)` has the AggCalls `SUM` and `COUNT`.
+
+AggState is the state we use to compute to the result (output) of the aggregation call.
+Within each aggregation group, it will have an AggState for each AggCall.
+
+AggGroups are created per aggregation group.
+For instance with `GROUP BY x1, x2`, there will be a group for each unique combination of `x1` and `x2`.
+
+Whenever stream chunks come in, the executor will update the aggregation state for each group, per agg call.
+
+On barrier, we will persist the in-memory states.
+For `value` type aggregations, we will persist the state to the intermediate state table.
+This state table will store all value aggregations per group on a single row.
+
+For `MaterializedInput` type aggregations, these require tracking input state. For example, non-append-only min/max.
+For each of these aggregations, they have 1 state table (`AggStateStorage::MaterializedInput`) each. Within the state table, it will store the input state for each group.
+
+### Initialization of `AggGroups`
+
+![init-agg-group](./images/aggregation/init-agg-group.png)
+
+AggGroups are initialized when corresponding aggregation groups are not found in `AggGroupCache`.
+This could be either because the `AggGroupCache` got evicted,
+or its a new group key.
+
+It could take a while to initialize agg groups, hence we cache them in `AggGroupCache`.