From d7a65c150ee99475617d03c2728a1102f8d77c4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Sep 2023 09:28:39 +0800 Subject: [PATCH 01/14] chore(deps): Bump syn from 2.0.32 to 2.0.33 (#12290) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 98 +++++++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1069bd493b209..56b77ea58fbc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,7 +220,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -475,7 +475,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -497,7 +497,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -514,7 +514,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -532,7 +532,7 @@ dependencies = [ "derive_utils", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -1470,7 +1470,7 @@ checksum = "cf85d5384815558275789d91d1895d1d9919a6e2534d6144650f036ac65691a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -1581,7 +1581,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -1875,7 +1875,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2068,7 +2068,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f34ba9a9bcb8645379e9de8cb3ecfcf4d1c85ba66d90deb3259206fa5aa193b" dependencies = [ "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2108,7 +2108,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2125,7 +2125,7 @@ checksum = "2fa16a70dd58129e4dfffdff535fb1bce66673f7bbeec4a5a1765a504e1ccd84" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2197,7 +2197,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2230,7 +2230,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2350,7 +2350,7 @@ checksum = "9abcad25e9720609ccb3dcdb795d845e37d8ce34183330a9f48b03a1a71c8e21" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2526,7 +2526,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2546,7 +2546,7 @@ checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2559,7 +2559,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2896,7 +2896,7 @@ dependencies = [ "regex", "regex-automata 0.3.8", "regex-syntax 0.7.5", - "syn 2.0.32", + "syn 2.0.33", "tokio", "tracing", "tracing-core", @@ -2933,7 +2933,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2945,7 +2945,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2957,7 +2957,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -3090,7 +3090,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -4513,7 +4513,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", "termcolor", "thiserror", ] @@ -5009,7 +5009,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -5304,7 +5304,7 @@ dependencies = [ "regex", "regex-syntax 0.7.5", "structmeta", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -5509,7 +5509,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -5636,7 +5636,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -5832,9 +5832,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -5946,7 +5946,7 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", "workspace-hack", ] @@ -7002,7 +7002,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -8092,7 +8092,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -8153,7 +8153,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -8241,7 +8241,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -8253,7 +8253,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -8291,7 +8291,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -8669,7 +8669,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -8680,7 +8680,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -8721,7 +8721,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -8776,9 +8776,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.32" +version = "2.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" +checksum = "9caece70c63bfba29ec2fed841a09851b14a235c60010fa4de58089b6c025668" dependencies = [ "proc-macro2", "quote", @@ -8880,7 +8880,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -9054,7 +9054,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -9336,7 +9336,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -9702,7 +9702,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", "wasm-bindgen-shared", ] @@ -9736,7 +9736,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10093,7 +10093,7 @@ dependencies = [ "smallvec", "subtle", "syn 1.0.109", - "syn 2.0.32", + "syn 2.0.33", "time", "time-macros", "tinyvec", @@ -10188,7 +10188,7 @@ checksum = "56097d5b91d711293a42be9289403896b68654625021732067eac7a4ca388a1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -10208,7 +10208,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] From 96bca904e265c6c753c779c43ee3b77b1aaf7f95 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Sep 2023 03:53:08 +0000 Subject: [PATCH 02/14] chore(deps): Bump libc from 0.2.147 to 0.2.148 (#12288) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: TennyZhuang --- Cargo.lock | 4 ++-- src/common/Cargo.toml | 2 +- src/storage/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 56b77ea58fbc4..f062a0bb9c98a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3973,9 +3973,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.147" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libflate" diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml index 18cd6c1509cc7..5484b2b88c412 100644 --- a/src/common/Cargo.toml +++ b/src/common/Cargo.toml @@ -114,7 +114,7 @@ libc = "0.2" [target.'cfg(target_os = "macos")'.dependencies] darwin-libproc = { git = "https://github.com/risingwavelabs/darwin-libproc.git", rev = "a502be24bd0971463f5bcbfe035a248d8ba503b7" } -libc = "0.2.147" +libc = "0.2.148" mach2 = "0.4" [dev-dependencies] diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index 1cdaccf3cdba5..6a07f6a990897 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -79,7 +79,7 @@ nix = { version = "0.27", features = ["fs", "mman"] } [target.'cfg(target_os = "macos")'.dependencies] darwin-libproc = { git = "https://github.com/risingwavelabs/darwin-libproc.git", rev = "a502be24bd0971463f5bcbfe035a248d8ba503b7" } -libc = "0.2.147" +libc = "0.2.148" mach2 = "0.4" [target.'cfg(not(madsim))'.dependencies] From 8e58cb96cb97bdb69849daef1198aeff6cc52f78 Mon Sep 17 00:00:00 2001 From: xiangjinwu <17769960+xiangjinwu@users.noreply.github.com> Date: Thu, 14 Sep 2023 11:57:24 +0800 Subject: [PATCH 03/14] fix(integration_tests): datagen timestamp and timestamptz accordingly (#12239) --- integration_tests/datagen/ad_click/ad_click.go | 4 ++-- integration_tests/datagen/ad_ctr/ad_ctr.go | 4 ++-- integration_tests/datagen/cdn_metrics/nics.go | 2 +- integration_tests/datagen/cdn_metrics/tcp.go | 2 +- integration_tests/datagen/clickstream/clickstream.go | 2 +- integration_tests/datagen/delivery/delivery.go | 2 +- integration_tests/datagen/ecommerce/ecommerce.go | 2 +- integration_tests/datagen/gen/generator.go | 4 +++- integration_tests/datagen/twitter/twitter.go | 4 ++-- integration_tests/twitter-pulsar/pb/create_mv.sql | 2 +- integration_tests/twitter/pb/create_mv.sql | 2 +- 11 files changed, 16 insertions(+), 14 deletions(-) diff --git a/integration_tests/datagen/ad_click/ad_click.go b/integration_tests/datagen/ad_click/ad_click.go index 9ce71ae3f36bc..27928d3694e26 100644 --- a/integration_tests/datagen/ad_click/ad_click.go +++ b/integration_tests/datagen/ad_click/ad_click.go @@ -54,8 +54,8 @@ func (g *adClickGen) Load(ctx context.Context, outCh chan<- sink.SinkRecord) { record := &clickEvent{ UserId: rand.Int63n(100000), AdId: rand.Int63n(10), - ClickTimestamp: now.Add(time.Duration(rand.Intn(1000)) * time.Millisecond).Format(gen.RwTimestampLayout), - ImpressionTimestamp: now.Format(gen.RwTimestampLayout), + ClickTimestamp: now.Add(time.Duration(rand.Intn(1000)) * time.Millisecond).Format(gen.RwTimestamptzLayout), + ImpressionTimestamp: now.Format(gen.RwTimestamptzLayout), } select { case <-ctx.Done(): diff --git a/integration_tests/datagen/ad_ctr/ad_ctr.go b/integration_tests/datagen/ad_ctr/ad_ctr.go index 1134ce4c1e895..cd3000e33407e 100644 --- a/integration_tests/datagen/ad_ctr/ad_ctr.go +++ b/integration_tests/datagen/ad_ctr/ad_ctr.go @@ -96,14 +96,14 @@ func (g *adCtrGen) generate() []sink.SinkRecord { &adImpressionEvent{ BidId: bidId, AdId: adId, - ImpressionTimestamp: time.Now().Format(gen.RwTimestampLayout), + ImpressionTimestamp: time.Now().Format(gen.RwTimestamptzLayout), }, } if g.hasClick(adId) { randomDelay := time.Duration(g.faker.IntRange(1, 10) * int(time.Second)) events = append(events, &adClickEvent{ BidId: bidId, - ClickTimestamp: time.Now().Add(randomDelay).Format(gen.RwTimestampLayout), + ClickTimestamp: time.Now().Add(randomDelay).Format(gen.RwTimestamptzLayout), }) } return events diff --git a/integration_tests/datagen/cdn_metrics/nics.go b/integration_tests/datagen/cdn_metrics/nics.go index 6aae95479ec9f..a95be61012115 100644 --- a/integration_tests/datagen/cdn_metrics/nics.go +++ b/integration_tests/datagen/cdn_metrics/nics.go @@ -109,7 +109,7 @@ func (impl *deviceNicsMonitor) newMetrics( MetricName: metricName, Aggregation: aggregation, NicName: "eth" + strconv.Itoa(NicId), - ReportTime: reportTime.Format(gen.RwTimestampLayout), + ReportTime: reportTime.Format(gen.RwTimestamptzLayout), Bandwidth: maxBandwidth, Value: float64(value), } diff --git a/integration_tests/datagen/cdn_metrics/tcp.go b/integration_tests/datagen/cdn_metrics/tcp.go index da7ce31d76dd3..f315a7572d4b6 100644 --- a/integration_tests/datagen/cdn_metrics/tcp.go +++ b/integration_tests/datagen/cdn_metrics/tcp.go @@ -90,7 +90,7 @@ func (m *deviceTcpMonitor) newMetrics(metricName string, reportTime time.Time, v return &tcpMetric{ DeviceId: m.deviceId, MetricName: metricName, - ReportTime: reportTime.Format(gen.RwTimestampLayout), + ReportTime: reportTime.Format(gen.RwTimestamptzLayout), Value: value, } } diff --git a/integration_tests/datagen/clickstream/clickstream.go b/integration_tests/datagen/clickstream/clickstream.go index c0e9350b3f2b1..201610a299283 100644 --- a/integration_tests/datagen/clickstream/clickstream.go +++ b/integration_tests/datagen/clickstream/clickstream.go @@ -138,7 +138,7 @@ func (g *clickStreamGen) generate() sink.SinkRecord { UserId: fmt.Sprint(userId), TargetId: string(target) + fmt.Sprint(targetId), TargetType: string(target), - EventTimestamp: time.Now().Format(gen.RwTimestampLayout), + EventTimestamp: time.Now().Format(gen.RwTimestamptzLayout), BehaviorType: behavior, ParentTargetType: parentTargetType, ParentTargetId: parentTargetId, diff --git a/integration_tests/datagen/delivery/delivery.go b/integration_tests/datagen/delivery/delivery.go index 0ca20dd689fea..d8e1133f71497 100644 --- a/integration_tests/datagen/delivery/delivery.go +++ b/integration_tests/datagen/delivery/delivery.go @@ -69,7 +69,7 @@ func (g *orderEventGen) Load(ctx context.Context, outCh chan<- sink.SinkRecord) OrderId: g.seqOrderId, RestaurantId: rand.Int63n(num_of_restaurants), OrderState: order_states[rand.Intn(len(order_states))], - OrderTimestamp: now.Add(time.Duration(rand.Intn(total_minutes)) * time.Minute).Format(gen.RwTimestampLayout), + OrderTimestamp: now.Add(time.Duration(rand.Intn(total_minutes)) * time.Minute).Format(gen.RwTimestampNaiveLayout), } g.seqOrderId++ select { diff --git a/integration_tests/datagen/ecommerce/ecommerce.go b/integration_tests/datagen/ecommerce/ecommerce.go index 18520c9b7eb60..34ee31cde6931 100644 --- a/integration_tests/datagen/ecommerce/ecommerce.go +++ b/integration_tests/datagen/ecommerce/ecommerce.go @@ -103,7 +103,7 @@ func (g *ecommerceGen) KafkaTopics() []string { } func (g *ecommerceGen) generate() []sink.SinkRecord { - ts := time.Now().Format(gen.RwTimestampLayout) + ts := time.Now().Format(gen.RwTimestampNaiveLayout) if g.faker.Bool() && g.seqShipId >= g.seqOrderId { // New order. diff --git a/integration_tests/datagen/gen/generator.go b/integration_tests/datagen/gen/generator.go index d519beec08c35..f84ffe3fcdea4 100644 --- a/integration_tests/datagen/gen/generator.go +++ b/integration_tests/datagen/gen/generator.go @@ -9,6 +9,7 @@ import ( "datagen/sink/postgres" "datagen/sink/pulsar" "datagen/sink/s3" + "time" "gonum.org/v1/gonum/stat/distuv" ) @@ -47,7 +48,8 @@ type LoadGenerator interface { Load(ctx context.Context, outCh chan<- sink.SinkRecord) } -const RwTimestampLayout = "2006-01-02 15:04:05.07+01:00" +const RwTimestampNaiveLayout = time.DateTime +const RwTimestamptzLayout = time.RFC3339 type RandDist interface { // Rand returns a random number ranging from [0, max]. diff --git a/integration_tests/datagen/twitter/twitter.go b/integration_tests/datagen/twitter/twitter.go index 06a235aaf7d02..1daf193c36e6f 100644 --- a/integration_tests/datagen/twitter/twitter.go +++ b/integration_tests/datagen/twitter/twitter.go @@ -120,7 +120,7 @@ func NewTwitterGen() gen.LoadGenerator { endTime, _ := time.Parse("2006-01-01", fmt.Sprintf("%d-01-01", endYear)) startTime, _ := time.Parse("2006-01-01", fmt.Sprintf("%d-01-01", startYear)) users[id] = &twitterUser{ - CreatedAt: faker.DateRange(startTime, endTime).Format(gen.RwTimestampLayout), + CreatedAt: faker.DateRange(startTime, endTime).Format(gen.RwTimestamptzLayout), Id: id, Name: fmt.Sprintf("%s %s", faker.Name(), faker.Adverb()), UserName: faker.Username(), @@ -152,7 +152,7 @@ func (t *twitterGen) generate() twitterEvent { return twitterEvent{ Data: tweetData{ Id: id, - CreatedAt: time.Now().Format(gen.RwTimestampLayout), + CreatedAt: time.Now().Format(gen.RwTimestamptzLayout), Text: sentence, Lang: gofakeit.Language(), }, diff --git a/integration_tests/twitter-pulsar/pb/create_mv.sql b/integration_tests/twitter-pulsar/pb/create_mv.sql index c08722bacdbb3..06d2eb14e4074 100644 --- a/integration_tests/twitter-pulsar/pb/create_mv.sql +++ b/integration_tests/twitter-pulsar/pb/create_mv.sql @@ -4,7 +4,7 @@ CREATE MATERIALIZED VIEW hot_hashtags AS WITH tags AS ( SELECT unnest(regexp_matches((data).text, '#\w+', 'g')) AS hashtag, - (data).created_at :: timestamp AS created_at + (data).created_at :: timestamptz AS created_at FROM twitter ) diff --git a/integration_tests/twitter/pb/create_mv.sql b/integration_tests/twitter/pb/create_mv.sql index c08722bacdbb3..06d2eb14e4074 100644 --- a/integration_tests/twitter/pb/create_mv.sql +++ b/integration_tests/twitter/pb/create_mv.sql @@ -4,7 +4,7 @@ CREATE MATERIALIZED VIEW hot_hashtags AS WITH tags AS ( SELECT unnest(regexp_matches((data).text, '#\w+', 'g')) AS hashtag, - (data).created_at :: timestamp AS created_at + (data).created_at :: timestamptz AS created_at FROM twitter ) From 7a02d12ef2eb771d605013b1af522998db4209b1 Mon Sep 17 00:00:00 2001 From: Runji Wang Date: Thu, 14 Sep 2023 13:05:03 +0800 Subject: [PATCH 04/14] feat(agg): support streaming `bool_and` and `bool_or` agg (#11956) Signed-off-by: Runji Wang --- e2e_test/streaming/aggregate/boolean.slt | 67 ++++++ proto/stream_plan.proto | 4 +- src/batch/src/executor/aggregation/filter.rs | 8 +- src/batch/src/executor/aggregation/mod.rs | 2 +- src/common/src/util/stream_graph_visitor.rs | 4 +- src/expr/benches/expr.rs | 4 +- src/expr/macro/src/gen.rs | 120 ++++++++--- src/expr/macro/src/lib.rs | 65 +++++- src/expr/macro/src/parse.rs | 44 +++- src/expr/src/agg/approx_count_distinct/mod.rs | 201 +++++++++++------- src/expr/src/agg/array_agg.rs | 6 +- src/expr/src/agg/bool_and.rs | 125 +++++++++++ src/expr/src/agg/bool_or.rs | 125 +++++++++++ src/expr/src/agg/def.rs | 10 +- src/expr/src/agg/general.rs | 101 +-------- src/expr/src/agg/mod.rs | 34 ++- src/expr/src/agg/string_agg.rs | 4 +- src/expr/src/error.rs | 3 + src/expr/src/expr/build.rs | 1 + src/expr/src/sig/agg.rs | 20 +- src/expr/src/sig/func.rs | 1 + src/expr/src/sig/mod.rs | 13 +- src/expr/src/sig/table_function.rs | 1 + src/expr/src/table_function/mod.rs | 1 + .../src/window_function/state/aggregate.rs | 4 +- src/expr/src/window_function/state/mod.rs | 1 + .../tests/testdata/output/ch_benchmark.yaml | 53 ++--- .../testdata/output/distribution_derive.yaml | 48 +++-- .../testdata/output/emit_on_window_close.yaml | 4 +- .../tests/testdata/output/except.yaml | 4 +- .../tests/testdata/output/intersect.yaml | 4 +- .../tests/testdata/output/nexmark.yaml | 50 ++--- .../tests/testdata/output/nexmark_source.yaml | 50 ++--- .../output/nexmark_temporal_filter.yaml | 30 +-- .../testdata/output/nexmark_watermark.yaml | 36 ++-- .../tests/testdata/output/share.yaml | 4 +- .../testdata/output/stream_dist_agg.yaml | 72 +++---- .../tests/testdata/output/tpch.yaml | 58 ++--- .../tests/testdata/output/tpch_variant.yaml | 19 +- .../tests/testdata/output/union.yaml | 4 +- .../src/optimizer/plan_node/generic/agg.rs | 38 +++- .../src/optimizer/plan_node/stream.rs | 16 +- .../optimizer/plan_node/stream_hash_agg.rs | 10 +- .../optimizer/plan_node/stream_simple_agg.rs | 6 +- .../plan_node/stream_stateless_simple_agg.rs | 2 +- .../src/utils/stream_graph_formatter.rs | 8 +- src/meta/src/stream/test_fragmenter.rs | 4 +- src/stream/src/cache/managed_lru.rs | 55 ----- src/stream/src/common/table/state_table.rs | 23 +- src/stream/src/executor/agg_common.rs | 2 +- .../src/executor/aggregation/agg_group.rs | 79 ++++++- .../src/executor/aggregation/agg_state.rs | 8 +- src/stream/src/executor/aggregation/minput.rs | 20 +- src/stream/src/executor/hash_agg.rs | 137 ++++++------ src/stream/src/executor/integration_tests.rs | 3 +- src/stream/src/executor/simple_agg.rs | 48 ++--- src/stream/src/executor/sort_buffer.rs | 12 ++ .../src/executor/stateless_simple_agg.rs | 4 +- src/stream/src/executor/test_utils.rs | 14 +- src/stream/src/from_proto/hash_agg.rs | 7 +- src/stream/src/from_proto/simple_agg.rs | 12 +- 61 files changed, 1253 insertions(+), 660 deletions(-) create mode 100644 e2e_test/streaming/aggregate/boolean.slt create mode 100644 src/expr/src/agg/bool_and.rs create mode 100644 src/expr/src/agg/bool_or.rs diff --git a/e2e_test/streaming/aggregate/boolean.slt b/e2e_test/streaming/aggregate/boolean.slt new file mode 100644 index 0000000000000..86cf018fda4e1 --- /dev/null +++ b/e2e_test/streaming/aggregate/boolean.slt @@ -0,0 +1,67 @@ +statement ok +SET RW_IMPLICIT_FLUSH TO true; + +statement ok +create table t (v boolean); + +statement ok +create materialized view mv as select + bool_and(v), + bool_or(v) +from t; + +query BB +select * from mv; +---- +NULL NULL + + +statement ok +insert into t values (true); + +# table values: true + +query BB +select * from mv; +---- +t t + + +statement ok +insert into t values (false); + +# table values: true, false + +query BB +select * from mv; +---- +f t + + +statement ok +delete from t where v = true; + +# table values: false + +query BB +select * from mv; +---- +f f + + +statement ok +delete from t; + +# table values: empty + +query BB +select * from mv; +---- +NULL NULL + + +statement ok +drop materialized view mv; + +statement ok +drop table t; diff --git a/proto/stream_plan.proto b/proto/stream_plan.proto index 33fe96a71c803..de8cfe14746cf 100644 --- a/proto/stream_plan.proto +++ b/proto/stream_plan.proto @@ -251,7 +251,7 @@ message SimpleAggNode { // Only used for stateless simple agg. repeated uint32 distribution_key = 2; repeated AggCallState agg_call_states = 3; - catalog.Table result_table = 4; + catalog.Table intermediate_state_table = 4; // Whether to optimize for append only stream. // It is true when the input is append-only bool is_append_only = 5; @@ -263,7 +263,7 @@ message HashAggNode { repeated uint32 group_key = 1; repeated expr.AggCall agg_calls = 2; repeated AggCallState agg_call_states = 3; - catalog.Table result_table = 4; + catalog.Table intermediate_state_table = 4; // Whether to optimize for append only stream. // It is true when the input is append-only bool is_append_only = 5; diff --git a/src/batch/src/executor/aggregation/filter.rs b/src/batch/src/executor/aggregation/filter.rs index 490bea5c342b3..9b85c2fbdddee 100644 --- a/src/batch/src/executor/aggregation/filter.rs +++ b/src/batch/src/executor/aggregation/filter.rs @@ -74,7 +74,7 @@ impl AggregateFunction for Filter { #[cfg(test)] mod tests { use risingwave_common::test_prelude::StreamChunkTestExt; - use risingwave_expr::agg::{build, AggCall}; + use risingwave_expr::agg::{build_append_only, AggCall}; use risingwave_expr::expr::{build_from_pretty, Expression, LiteralExpression}; use super::*; @@ -84,7 +84,7 @@ mod tests { let condition = LiteralExpression::new(DataType::Boolean, Some(true.into())).boxed(); let agg = Filter::new( condition.into(), - build(&AggCall::from_pretty("(count:int8 $0:int8)")).unwrap(), + build_append_only(&AggCall::from_pretty("(count:int8 $0:int8)")).unwrap(), ); let mut state = agg.create_state(); @@ -113,7 +113,7 @@ mod tests { let expr = build_from_pretty("(greater_than:boolean $0:int8 5:int8)"); let agg = Filter::new( expr.into(), - build(&AggCall::from_pretty("(count:int8 $0:int8)")).unwrap(), + build_append_only(&AggCall::from_pretty("(count:int8 $0:int8)")).unwrap(), ); let mut state = agg.create_state(); @@ -145,7 +145,7 @@ mod tests { let expr = build_from_pretty("(equal:boolean $0:int8 null:int8)"); let agg = Filter::new( expr.into(), - build(&AggCall::from_pretty("(count:int8 $0:int8)")).unwrap(), + build_append_only(&AggCall::from_pretty("(count:int8 $0:int8)")).unwrap(), ); let mut state = agg.create_state(); diff --git a/src/batch/src/executor/aggregation/mod.rs b/src/batch/src/executor/aggregation/mod.rs index 64191efa14dd9..a794823e75636 100644 --- a/src/batch/src/executor/aggregation/mod.rs +++ b/src/batch/src/executor/aggregation/mod.rs @@ -30,7 +30,7 @@ use self::projection::Projection; /// Build an `BoxedAggregateFunction` from `AggCall`. pub fn build(agg: &AggCall) -> Result { - let mut aggregator = agg::build(agg)?; + let mut aggregator = agg::build_append_only(agg)?; if agg.distinct { aggregator = Box::new(Distinct::new(aggregator)); diff --git a/src/common/src/util/stream_graph_visitor.rs b/src/common/src/util/stream_graph_visitor.rs index 83112475201a6..cc8e950e42b12 100644 --- a/src/common/src/util/stream_graph_visitor.rs +++ b/src/common/src/util/stream_graph_visitor.rs @@ -95,7 +95,7 @@ fn visit_stream_node_tables_inner( // Aggregation NodeBody::HashAgg(node) => { assert_eq!(node.agg_call_states.len(), node.agg_calls.len()); - always!(node.result_table, "HashAggResult"); + always!(node.intermediate_state_table, "HashAggState"); for (call_idx, state) in node.agg_call_states.iter_mut().enumerate() { match state.inner.as_mut().unwrap() { agg_call_state::Inner::ValueState(_) => {} @@ -110,7 +110,7 @@ fn visit_stream_node_tables_inner( } NodeBody::SimpleAgg(node) => { assert_eq!(node.agg_call_states.len(), node.agg_calls.len()); - always!(node.result_table, "SimpleAggResult"); + always!(node.intermediate_state_table, "SimpleAggState"); for (call_idx, state) in node.agg_call_states.iter_mut().enumerate() { match state.inner.as_mut().unwrap() { agg_call_state::Inner::ValueState(_) => {} diff --git a/src/expr/benches/expr.rs b/src/expr/benches/expr.rs index 6b6e7c9e41657..7dca801e057d6 100644 --- a/src/expr/benches/expr.rs +++ b/src/expr/benches/expr.rs @@ -24,7 +24,7 @@ use criterion::{criterion_group, criterion_main, Criterion}; use risingwave_common::array::*; use risingwave_common::types::test_utils::IntervalTestExt; use risingwave_common::types::*; -use risingwave_expr::agg::{build as build_agg, AggArgs, AggCall, AggKind}; +use risingwave_expr::agg::{build_append_only, AggArgs, AggCall, AggKind}; use risingwave_expr::expr::*; use risingwave_expr::sig::agg::agg_func_sigs; use risingwave_expr::sig::func::func_sigs; @@ -361,7 +361,7 @@ fn bench_expr(c: &mut Criterion) { println!("todo: {sig:?}"); continue; } - let agg = match build_agg(&AggCall { + let agg = match build_append_only(&AggCall { kind: sig.func, args: match sig.inputs_type { [] => AggArgs::None, diff --git a/src/expr/macro/src/gen.rs b/src/expr/macro/src/gen.rs index 9dcde3c6ffdc2..6b7d33ac74680 100644 --- a/src/expr/macro/src/gen.rs +++ b/src/expr/macro/src/gen.rs @@ -421,9 +421,11 @@ impl FunctionAttr { /// Generate a descriptor of the aggregate function. /// /// The types of arguments and return value should not contain wildcard. + /// `user_fn` could be either `fn` or `impl`. + /// If `build_fn` is true, `user_fn` must be a `fn` that builds the aggregate function. pub fn generate_aggregate_descriptor( &self, - user_fn: &UserFunctionAttr, + user_fn: &AggregateFnOrImpl, build_fn: bool, ) -> Result { let name = self.name.clone(); @@ -433,12 +435,23 @@ impl FunctionAttr { args.push(data_type_name(ty)); } let ret = data_type_name(&self.ret); + let state_type = match &self.state { + Some(ty) if ty != "ref" => data_type_name(ty), + _ => data_type_name(&self.ret), + }; + let append_only = match build_fn { + false => !user_fn.has_retract(), + true => self.append_only, + }; let pb_type = format_ident!("{}", utils::to_camel_case(&name)); - let ctor_name = format_ident!("{}", self.ident_name()); + let ctor_name = match append_only { + false => format_ident!("{}", self.ident_name()), + true => format_ident!("{}_append_only", self.ident_name()), + }; let descriptor_type = quote! { crate::sig::agg::AggFuncSig }; let build_fn = if build_fn { - let name = format_ident!("{}", user_fn.name); + let name = format_ident!("{}", user_fn.as_fn().name); quote! { #name } } else { self.generate_agg_build_fn(user_fn)? @@ -450,18 +463,20 @@ impl FunctionAttr { unsafe { crate::sig::agg::_register(#descriptor_type { func: crate::agg::AggKind::#pb_type, inputs_type: &[#(#args),*], + state_type: #state_type, ret_type: #ret, build: #build_fn, + append_only: #append_only, }) }; } }) } /// Generate build function for aggregate function. - fn generate_agg_build_fn(&self, user_fn: &UserFunctionAttr) -> Result { + fn generate_agg_build_fn(&self, user_fn: &AggregateFnOrImpl) -> Result { let state_type: TokenStream2 = match &self.state { Some(state) if state == "ref" => types::ref_type(&self.ret).parse().unwrap(), - Some(state) if state != "ref" => state.parse().unwrap(), + Some(state) if state != "ref" => types::owned_type(state).parse().unwrap(), _ => types::owned_type(&self.ret).parse().unwrap(), }; let let_arrays = self @@ -503,28 +518,49 @@ impl FunctionAttr { } } }); - let fn_name = format_ident!("{}", user_fn.name); let args = (0..self.args.len()).map(|i| format_ident!("v{i}")); let args = quote! { #(#args,)* }; - let retract = match user_fn.retract { - true => quote! { matches!(op, Op::Delete | Op::UpdateDelete) }, - false => quote! {}, - }; - let check_retract = match user_fn.retract { - true => quote! {}, - false => { - let msg = format!("aggregate function {} only supports append", self.name); - quote! { assert_eq!(op, Op::Insert, #msg); } + let panic_on_retract = { + let msg = format!( + "attempt to retract on aggregate function {}, but it is append-only", + self.name + ); + quote! { assert_eq!(op, Op::Insert, #msg); } + }; + let mut next_state = match user_fn { + AggregateFnOrImpl::Fn(f) => { + let fn_name = format_ident!("{}", f.name); + match f.retract { + true => { + quote! { #fn_name(state, #args matches!(op, Op::Delete | Op::UpdateDelete)) } + } + false => quote! {{ + #panic_on_retract + #fn_name(state, #args) + }}, + } + } + AggregateFnOrImpl::Impl(i) => { + let retract = match i.retract { + Some(_) => quote! { self.function.retract(state, #args) }, + None => panic_on_retract, + }; + quote! { + if matches!(op, Op::Delete | Op::UpdateDelete) { + #retract + } else { + self.function.accumulate(state, #args) + } + } } }; - let mut next_state = quote! { #fn_name(state, #args #retract) }; - next_state = match user_fn.return_type_kind { + next_state = match user_fn.accumulate().return_type_kind { ReturnTypeKind::T => quote! { Some(#next_state) }, ReturnTypeKind::Option => next_state, ReturnTypeKind::Result => quote! { Some(#next_state?) }, ReturnTypeKind::ResultOption => quote! { #next_state? }, }; - if !user_fn.arg_option { + if !user_fn.accumulate().arg_option { match self.args.len() { 0 => { next_state = quote! { @@ -535,9 +571,16 @@ impl FunctionAttr { }; } 1 => { - let first_state = match &self.init_state { - Some(_) => quote! { unreachable!() }, - _ => quote! { Some(v0.into()) }, + let first_state = if self.init_state.is_some() { + quote! { unreachable!() } + } else if let Some(s) = &self.state && s == "ref" { + // for min/max/first/last, the state is the first value + quote! { Some(v0) } + } else { + quote! {{ + let state = #state_type::default(); + #next_state + }} }; next_state = quote! { match (state, v0) { @@ -550,6 +593,32 @@ impl FunctionAttr { _ => todo!("multiple arguments are not supported for non-option function"), } } + let get_result = match user_fn { + AggregateFnOrImpl::Impl(impl_) if impl_.finalize.is_some() => { + quote! { + let state = match state { + Some(s) => s.as_scalar_ref_impl().try_into().unwrap(), + None => return Ok(None), + }; + Ok(Some(self.function.finalize(state).into())) + } + } + _ => quote! { Ok(state.clone()) }, + }; + let function_field = match user_fn { + AggregateFnOrImpl::Fn(_) => quote! {}, + AggregateFnOrImpl::Impl(i) => { + let struct_name = format_ident!("{}", i.struct_name); + quote! { function: #struct_name, } + } + }; + let function_new = match user_fn { + AggregateFnOrImpl::Fn(_) => quote! {}, + AggregateFnOrImpl::Impl(i) => { + let struct_name = format_ident!("{}", i.struct_name); + quote! { function: #struct_name::default(), } + } + }; Ok(quote! { |agg| { @@ -567,6 +636,7 @@ impl FunctionAttr { #[derive(Clone)] struct Agg { return_type: DataType, + #function_field } #[async_trait::async_trait] @@ -585,7 +655,6 @@ impl FunctionAttr { Vis::Bitmap(bitmap) => { for row_id in bitmap.iter_ones() { let op = unsafe { *input.ops().get_unchecked(row_id) }; - #check_retract #(#let_values)* state = #next_state; } @@ -593,7 +662,6 @@ impl FunctionAttr { Vis::Compact(_) => { for row_id in 0..input.capacity() { let op = unsafe { *input.ops().get_unchecked(row_id) }; - #check_retract #(#let_values)* state = #next_state; } @@ -617,7 +685,6 @@ impl FunctionAttr { break; } let op = unsafe { *input.ops().get_unchecked(row_id) }; - #check_retract #(#let_values)* state = #next_state; } @@ -625,7 +692,6 @@ impl FunctionAttr { Vis::Compact(_) => { for row_id in range { let op = unsafe { *input.ops().get_unchecked(row_id) }; - #check_retract #(#let_values)* state = #next_state; } @@ -636,12 +702,14 @@ impl FunctionAttr { } async fn get_result(&self, state: &AggregateState) -> Result { - Ok(state.as_datum().clone()) + let state = state.as_datum(); + #get_result } } Ok(Box::new(Agg { return_type: agg.return_type.clone(), + #function_new })) } }) diff --git a/src/expr/macro/src/lib.rs b/src/expr/macro/src/lib.rs index 0f490b6f18802..82e7c8014220a 100644 --- a/src/expr/macro/src/lib.rs +++ b/src/expr/macro/src/lib.rs @@ -403,7 +403,7 @@ pub fn build_function(attr: TokenStream, item: TokenStream) -> TokenStream { pub fn aggregate(attr: TokenStream, item: TokenStream) -> TokenStream { fn inner(attr: TokenStream, item: TokenStream) -> Result { let fn_attr: FunctionAttr = syn::parse(attr)?; - let user_fn: UserFunctionAttr = syn::parse(item.clone())?; + let user_fn: AggregateFnOrImpl = syn::parse(item.clone())?; let mut tokens: TokenStream2 = item.into(); for attr in fn_attr.expand() { @@ -421,7 +421,7 @@ pub fn aggregate(attr: TokenStream, item: TokenStream) -> TokenStream { pub fn build_aggregate(attr: TokenStream, item: TokenStream) -> TokenStream { fn inner(attr: TokenStream, item: TokenStream) -> Result { let fn_attr: FunctionAttr = syn::parse(attr)?; - let user_fn: UserFunctionAttr = syn::parse(item.clone())?; + let user_fn: AggregateFnOrImpl = syn::parse(item.clone())?; let mut tokens: TokenStream2 = item.into(); for attr in fn_attr.expand() { @@ -437,15 +437,30 @@ pub fn build_aggregate(attr: TokenStream, item: TokenStream) -> TokenStream { #[derive(Debug, Clone, Default)] struct FunctionAttr { + /// Function name name: String, + /// Input argument types args: Vec, + /// Return type ret: String, + /// Whether it is a table function is_table_function: bool, + /// Whether it is an append-only aggregate function + append_only: bool, + /// Optional function for batch evaluation. batch_fn: Option, + /// State type for aggregate function. + /// If not specified, it will be the same as return type. state: Option, + /// Initial state value for aggregate function. + /// If not specified, it will be NULL. init_state: Option, + /// Prebuild function for arguments. + /// This could be any Rust expression. prebuild: Option, + /// Type inference function. type_infer: Option, + /// Whether the function is deprecated. deprecated: bool, } @@ -474,6 +489,52 @@ struct UserFunctionAttr { return_type_span: proc_macro2::Span, } +#[derive(Debug, Clone)] +struct AggregateImpl { + struct_name: String, + accumulate: UserFunctionAttr, + retract: Option, + #[allow(dead_code)] // TODO(wrj): add merge to trait + merge: Option, + finalize: Option, + #[allow(dead_code)] // TODO(wrj): support encode + encode_state: Option, + #[allow(dead_code)] // TODO(wrj): support decode + decode_state: Option, +} + +#[derive(Debug, Clone)] +#[allow(clippy::large_enum_variant)] +enum AggregateFnOrImpl { + /// A simple accumulate/retract function. + Fn(UserFunctionAttr), + /// A full impl block. + Impl(AggregateImpl), +} + +impl AggregateFnOrImpl { + fn as_fn(&self) -> &UserFunctionAttr { + match self { + AggregateFnOrImpl::Fn(attr) => attr, + _ => panic!("expect fn"), + } + } + + fn accumulate(&self) -> &UserFunctionAttr { + match self { + AggregateFnOrImpl::Fn(attr) => attr, + AggregateFnOrImpl::Impl(impl_) => &impl_.accumulate, + } + } + + fn has_retract(&self) -> bool { + match self { + AggregateFnOrImpl::Fn(fn_) => fn_.retract, + AggregateFnOrImpl::Impl(impl_) => impl_.retract.is_some(), + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] enum ReturnTypeKind { T, diff --git a/src/expr/macro/src/parse.rs b/src/expr/macro/src/parse.rs index e45ec20d174fe..dfcd08aafd3f9 100644 --- a/src/expr/macro/src/parse.rs +++ b/src/expr/macro/src/parse.rs @@ -76,6 +76,8 @@ impl Parse for FunctionAttr { parsed.type_infer = Some(get_value()?); } else if meta.path().is_ident("deprecated") { parsed.deprecated = true; + } else if meta.path().is_ident("append_only") { + parsed.append_only = true; } else { return Err(Error::new( meta.span(), @@ -90,7 +92,12 @@ impl Parse for FunctionAttr { impl Parse for UserFunctionAttr { fn parse(input: ParseStream<'_>) -> Result { let itemfn: syn::ItemFn = input.parse()?; - let sig = &itemfn.sig; + Ok(UserFunctionAttr::from(&itemfn.sig)) + } +} + +impl From<&syn::Signature> for UserFunctionAttr { + fn from(sig: &syn::Signature) -> Self { let (return_type_kind, iterator_item_kind, core_return_type) = match &sig.output { syn::ReturnType::Default => (ReturnTypeKind::T, None, "()".into()), syn::ReturnType::Type(_, ty) => { @@ -104,7 +111,7 @@ impl Parse for UserFunctionAttr { } } }; - Ok(UserFunctionAttr { + UserFunctionAttr { name: sig.ident.to_string(), write: sig.inputs.iter().any(arg_is_write), context: sig.inputs.iter().any(arg_is_context), @@ -115,10 +122,43 @@ impl Parse for UserFunctionAttr { core_return_type, generic: sig.generics.params.len(), return_type_span: sig.output.span(), + } + } +} + +impl Parse for AggregateImpl { + fn parse(input: ParseStream<'_>) -> Result { + let itemimpl: syn::ItemImpl = input.parse()?; + let parse_function = |name: &str| { + itemimpl.items.iter().find_map(|item| match item { + syn::ImplItem::Fn(syn::ImplItemFn { sig, .. }) if sig.ident == name => { + Some(UserFunctionAttr::from(sig)) + } + _ => None, + }) + }; + Ok(AggregateImpl { + struct_name: itemimpl.self_ty.to_token_stream().to_string(), + accumulate: parse_function("accumulate").expect("expect accumulate function"), + retract: parse_function("retract"), + merge: parse_function("merge"), + finalize: parse_function("finalize"), + encode_state: parse_function("encode_state"), + decode_state: parse_function("decode_state"), }) } } +impl Parse for AggregateFnOrImpl { + fn parse(input: ParseStream<'_>) -> Result { + if input.peek(Token![impl]) { + Ok(AggregateFnOrImpl::Impl(input.parse()?)) + } else { + Ok(AggregateFnOrImpl::Fn(input.parse()?)) + } + } +} + /// Check if the argument is `&mut impl Write`. fn arg_is_write(arg: &syn::FnArg) -> bool { let syn::FnArg::Typed(arg) = arg else { diff --git a/src/expr/src/agg/approx_count_distinct/mod.rs b/src/expr/src/agg/approx_count_distinct/mod.rs index 98eb203015a82..8a78edeb5a415 100644 --- a/src/expr/src/agg/approx_count_distinct/mod.rs +++ b/src/expr/src/agg/approx_count_distinct/mod.rs @@ -15,7 +15,6 @@ use std::collections::hash_map::DefaultHasher; use std::fmt::Debug; use std::hash::{Hash, Hasher}; -use std::marker::PhantomData; use std::ops::Range; use risingwave_common::array::{Op, StreamChunk}; @@ -28,7 +27,7 @@ use risingwave_expr_macro::build_aggregate; use self::append_only::AppendOnlyBucket; use self::updatable::UpdatableBucket; use super::{AggCall, AggStateDyn, AggregateFunction, AggregateState}; -use crate::Result; +use crate::{ExprError, Result}; mod append_only; mod updatable; @@ -43,19 +42,21 @@ const LOG_COUNT_BITS: u8 = 6; const BIAS_CORRECTION: f64 = 0.7213 / (1. + (1.079 / NUM_OF_REGISTERS as f64)); /// Count the approximate number of unique non-null values. -#[build_aggregate("approx_count_distinct(*) -> int64")] -fn build(_agg: &AggCall) -> Result> { - Ok(Box::new(ApproxCountDistinct:: { - _mark: PhantomData, - })) +#[build_aggregate("approx_count_distinct(*) -> int64", state = "int64")] +fn build_updatable(_agg: &AggCall) -> Result> { + Ok(Box::new(UpdatableApproxCountDistinct)) } -struct ApproxCountDistinct { - _mark: PhantomData, +/// Count the approximate number of unique non-null values. +#[build_aggregate("approx_count_distinct(*) -> int64", state = "int64[]", append_only)] +fn build_append_only(_agg: &AggCall) -> Result> { + Ok(Box::new(AppendOnlyApproxCountDistinct)) } +struct UpdatableApproxCountDistinct; + #[async_trait::async_trait] -impl AggregateFunction for ApproxCountDistinct { +impl AggregateFunction for UpdatableApproxCountDistinct { fn return_type(&self) -> DataType { DataType::Int64 } @@ -95,6 +96,115 @@ impl AggregateFunction for ApproxCountDistinct { let state = state.downcast_ref::(); Ok(Some(state.calculate_result().into())) } + + fn encode_state(&self, state: &AggregateState) -> Result { + let state = state.downcast_ref::(); + // FIXME: store state of updatable registers properly + Ok(Some(ScalarImpl::Int64(state.calculate_result()))) + } + + fn decode_state(&self, datum: Datum) -> Result { + // FIXME: restore state of updatable registers properly + let Some(ScalarImpl::Int64(initial_count)) = datum else { + return Err(ExprError::InvalidState("expect int64".into())); + }; + Ok(AggregateState::Any(Box::new(UpdatableRegisters { + initial_count, + ..UpdatableRegisters::default() + }))) + } +} + +struct AppendOnlyApproxCountDistinct; + +#[async_trait::async_trait] +impl AggregateFunction for AppendOnlyApproxCountDistinct { + fn return_type(&self) -> DataType { + DataType::Int64 + } + + fn create_state(&self) -> AggregateState { + AggregateState::Any(Box::::default()) + } + + async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> { + let state = state.downcast_mut::(); + for (op, row) in input.rows() { + let retract = matches!(op, Op::Delete | Op::UpdateDelete); + if let Some(scalar) = row.datum_at(0) { + state.update(scalar, retract)?; + } + } + Ok(()) + } + + async fn update_range( + &self, + state: &mut AggregateState, + input: &StreamChunk, + range: Range, + ) -> Result<()> { + let state = state.downcast_mut::(); + for (op, row) in input.rows_in(range) { + let retract = matches!(op, Op::Delete | Op::UpdateDelete); + if let Some(scalar) = row.datum_at(0) { + state.update(scalar, retract)?; + } + } + Ok(()) + } + + async fn get_result(&self, state: &AggregateState) -> Result { + let state = state.downcast_ref::(); + Ok(Some(state.calculate_result().into())) + } + + fn encode_state(&self, state: &AggregateState) -> Result { + let reg = state.downcast_ref::(); + + let buckets = ®.registers[..]; + let result_len = (buckets.len() * LOG_COUNT_BITS as usize - 1) / (i64::BITS as usize) + 1; + let mut result = vec![0u64; result_len]; + for (i, bucket_val) in buckets.iter().enumerate() { + let (start_idx, begin_bit, post_end_bit) = pos_in_serialized(i); + result[start_idx] |= (buckets[i].0 as u64) << begin_bit; + if post_end_bit > i64::BITS { + result[start_idx + 1] |= (bucket_val.0 as u64) >> (i64::BITS - begin_bit as u32); + } + } + Ok(Some(ScalarImpl::List(ListValue::new( + result + .into_iter() + .map(|x| Some(ScalarImpl::Int64(x as i64))) + .collect(), + )))) + } + + fn decode_state(&self, datum: Datum) -> Result { + let scalar = datum.unwrap(); + let list = scalar.as_list().values(); + let bucket_num = list.len() * i64::BITS as usize / LOG_COUNT_BITS as usize; + let registers = (0..bucket_num) + .map(|i| { + let (start_idx, begin_bit, post_end_bit) = pos_in_serialized(i); + let val = *list[start_idx].as_ref().unwrap().as_int64(); + let v = if post_end_bit <= i64::BITS { + (val as u64) << (i64::BITS - post_end_bit) + >> (i64::BITS - LOG_COUNT_BITS as u32) + } else { + ((val as u64) >> begin_bit) + + (((*list[start_idx + 1].as_ref().unwrap().as_int64() as u64) + & ((1 << (post_end_bit - i64::BITS)) - 1)) + << (i64::BITS - begin_bit as u32)) + }; + AppendOnlyBucket(v as u8) + }) + .collect(); + Ok(AggregateState::Any(Box::new(AppendOnlyRegisters { + registers, + initial_count: 0, + }))) + } } /// Approximates the count of non-null rows using a modified version of the `HyperLogLog` algorithm. @@ -215,75 +325,6 @@ impl EstimateSize for Registers { } } -/// Serialize the state into a scalar. -impl From for ScalarImpl { - fn from(reg: AppendOnlyRegisters) -> Self { - let buckets = ®.registers[..]; - let result_len = (buckets.len() * LOG_COUNT_BITS as usize - 1) / (i64::BITS as usize) + 1; - let mut result = vec![0u64; result_len]; - for (i, bucket_val) in buckets.iter().enumerate() { - let (start_idx, begin_bit, post_end_bit) = pos_in_serialized(i); - result[start_idx] |= (buckets[i].0 as u64) << begin_bit; - if post_end_bit > i64::BITS { - result[start_idx + 1] |= (bucket_val.0 as u64) >> (i64::BITS - begin_bit as u32); - } - } - ScalarImpl::List(ListValue::new( - result - .into_iter() - .map(|x| Some(ScalarImpl::Int64(x as i64))) - .collect(), - )) - } -} - -/// Deserialize the state from a scalar. -impl From for AppendOnlyRegisters { - fn from(state: ScalarImpl) -> Self { - let list = state.as_list().values(); - let bucket_num = list.len() * i64::BITS as usize / LOG_COUNT_BITS as usize; - let registers = (0..bucket_num) - .map(|i| { - let (start_idx, begin_bit, post_end_bit) = pos_in_serialized(i); - let val = *list[start_idx].as_ref().unwrap().as_int64(); - let v = if post_end_bit <= i64::BITS { - (val as u64) << (i64::BITS - post_end_bit) - >> (i64::BITS - LOG_COUNT_BITS as u32) - } else { - ((val as u64) >> begin_bit) - + (((*list[start_idx + 1].as_ref().unwrap().as_int64() as u64) - & ((1 << (post_end_bit - i64::BITS)) - 1)) - << (i64::BITS - begin_bit as u32)) - }; - AppendOnlyBucket(v as u8) - }) - .collect(); - Self { - registers, - initial_count: 0, - } - } -} - -/// Serialize the state into a scalar. -impl From for ScalarImpl { - fn from(reg: UpdatableRegisters) -> Self { - // FIXME: store state of updatable registers properly - ScalarImpl::Int64(reg.calculate_result()) - } -} - -/// Deserialize the state from a scalar. -impl From for UpdatableRegisters { - fn from(state: ScalarImpl) -> Self { - // FIXME: restore state of updatable registers properly - Self { - initial_count: state.into_int64(), - ..Self::default() - } - } -} - fn pos_in_serialized(bucket_idx: usize) -> (usize, usize, u32) { // rust compiler will optimize for us let start_idx = bucket_idx * LOG_COUNT_BITS as usize / i64::BITS as usize; @@ -301,7 +342,7 @@ mod tests { #[test] fn test() { - let approx_count_distinct = crate::agg::build(&AggCall::from_pretty( + let approx_count_distinct = crate::agg::build_append_only(&AggCall::from_pretty( "(approx_count_distinct:int8 $0:int4)", )) .unwrap(); diff --git a/src/expr/src/agg/array_agg.rs b/src/expr/src/agg/array_agg.rs index 5e22b68dba70d..02f29a308fa64 100644 --- a/src/expr/src/agg/array_agg.rs +++ b/src/expr/src/agg/array_agg.rs @@ -39,7 +39,8 @@ mod tests { + 456 + 789", ); - let array_agg = crate::agg::build(&AggCall::from_pretty("(array_agg:int4[] $0:int4)"))?; + let array_agg = + crate::agg::build_append_only(&AggCall::from_pretty("(array_agg:int4[] $0:int4)"))?; let mut state = array_agg.create_state(); array_agg.update(&mut state, &chunk).await?; let actual = array_agg.get_result(&state).await?; @@ -52,7 +53,8 @@ mod tests { #[tokio::test] async fn test_array_agg_empty() -> Result<()> { - let array_agg = crate::agg::build(&AggCall::from_pretty("(array_agg:int4[] $0:int4)"))?; + let array_agg = + crate::agg::build_append_only(&AggCall::from_pretty("(array_agg:int4[] $0:int4)"))?; let mut state = array_agg.create_state(); assert_eq!(array_agg.get_result(&state).await?, None); diff --git a/src/expr/src/agg/bool_and.rs b/src/expr/src/agg/bool_and.rs new file mode 100644 index 0000000000000..ac716403ddbd1 --- /dev/null +++ b/src/expr/src/agg/bool_and.rs @@ -0,0 +1,125 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_expr_macro::aggregate; + +/// Returns true if all non-null input values are true, otherwise false. +/// +/// # Example +/// +/// ```slt +/// statement ok +/// create table t (b1 boolean, b2 boolean, b3 boolean, b4 boolean); +/// +/// query T +/// select bool_and(b1) from t; +/// ---- +/// NULL +/// +/// statement ok +/// insert into t values +/// (true, null, false, null), +/// (false, true, null, null), +/// (null, true, false, null); +/// +/// query TTTTTT +/// select +/// bool_and(b1), +/// bool_and(b2), +/// bool_and(b3), +/// bool_and(b4), +/// bool_and(NOT b2), +/// bool_and(NOT b3) +/// FROM t; +/// ---- +/// f t f NULL f t +/// +/// statement ok +/// drop table t; +/// ``` +#[aggregate("bool_and(boolean) -> boolean", state = "ref")] +fn bool_and_append_only(state: bool, input: bool) -> bool { + state && input +} + +/// Returns true if all non-null input values are true, otherwise false. +/// +/// # Example +/// +/// ```slt +/// statement ok +/// create table t (b boolean); +/// +/// statement ok +/// create materialized view mv as select bool_and(b) from t; +/// +/// query T +/// select * from mv; +/// ---- +/// NULL +/// +/// statement ok +/// insert into t values (true), (false), (null); +/// +/// query T +/// select * from mv; +/// ---- +/// f +/// +/// statement ok +/// delete from t where b is false; +/// +/// query T +/// select * from mv; +/// ---- +/// t +/// +/// statement ok +/// drop materialized view mv; +/// +/// statement ok +/// drop table t; +/// ``` +#[derive(Debug, Default, Clone)] +struct BoolAndUpdatable; + +#[aggregate("bool_and(boolean) -> boolean", state = "int64")] +impl BoolAndUpdatable { + // state is the number of false values + + fn accumulate(&self, state: i64, input: bool) -> i64 { + if input { + state + } else { + state + 1 + } + } + + fn retract(&self, state: i64, input: bool) -> i64 { + if input { + state + } else { + state - 1 + } + } + + #[allow(dead_code)] // TODO: support merge + fn merge(&self, state1: i64, state2: i64) -> i64 { + state1 + state2 + } + + fn finalize(&self, state: i64) -> bool { + state == 0 + } +} diff --git a/src/expr/src/agg/bool_or.rs b/src/expr/src/agg/bool_or.rs new file mode 100644 index 0000000000000..ad0c74bedacf3 --- /dev/null +++ b/src/expr/src/agg/bool_or.rs @@ -0,0 +1,125 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_expr_macro::aggregate; + +/// Returns true if any non-null input value is true, otherwise false. +/// +/// # Example +/// +/// ```slt +/// statement ok +/// create table t (b1 boolean, b2 boolean, b3 boolean, b4 boolean); +/// +/// query T +/// select bool_or(b1) from t; +/// ---- +/// NULL +/// +/// statement ok +/// insert into t values +/// (true, null, false, null), +/// (false, true, null, null), +/// (null, true, false, null); +/// +/// query TTTTTT +/// select +/// bool_or(b1), +/// bool_or(b2), +/// bool_or(b3), +/// bool_or(b4), +/// bool_or(NOT b2), +/// bool_or(NOT b3) +/// FROM t; +/// ---- +/// t t f NULL f t +/// +/// statement ok +/// drop table t; +/// ``` +#[aggregate("bool_or(boolean) -> boolean")] +fn bool_or_append_only(state: bool, input: bool) -> bool { + state || input +} + +/// Returns true if any non-null input value is true, otherwise false. +/// +/// # Example +/// +/// ```slt +/// statement ok +/// create table t (b boolean); +/// +/// statement ok +/// create materialized view mv as select bool_or(b) from t; +/// +/// query T +/// select * from mv; +/// ---- +/// NULL +/// +/// statement ok +/// insert into t values (true), (false), (null); +/// +/// query T +/// select * from mv; +/// ---- +/// t +/// +/// statement ok +/// delete from t where b is true; +/// +/// query T +/// select * from mv; +/// ---- +/// f +/// +/// statement ok +/// drop materialized view mv; +/// +/// statement ok +/// drop table t; +/// ``` +#[derive(Debug, Default, Clone)] +struct BoolOrUpdatable; + +#[aggregate("bool_or(boolean) -> boolean", state = "int64")] +impl BoolOrUpdatable { + // state is the number of true values + + fn accumulate(&self, state: i64, input: bool) -> i64 { + if input { + state + 1 + } else { + state + } + } + + fn retract(&self, state: i64, input: bool) -> i64 { + if input { + state - 1 + } else { + state + } + } + + #[allow(dead_code)] // TODO: support merge + fn merge(&self, state1: i64, state2: i64) -> i64 { + state1 + state2 + } + + fn finalize(&self, state: i64) -> bool { + state != 0 + } +} diff --git a/src/expr/src/agg/def.rs b/src/expr/src/agg/def.rs index 570e910b68714..d9829a3dbb37c 100644 --- a/src/expr/src/agg/def.rs +++ b/src/expr/src/agg/def.rs @@ -314,8 +314,6 @@ pub mod agg_kinds { () => { AggKind::BitAnd | AggKind::BitOr - | AggKind::BoolAnd - | AggKind::BoolOr | AggKind::JsonbAgg | AggKind::JsonbObjectAgg | AggKind::PercentileCont @@ -410,6 +408,10 @@ pub mod agg_kinds { | AggKind::PercentileCont | AggKind::PercentileDisc | AggKind::Mode + // FIXME(wrj): move `BoolAnd` and `BoolOr` out + // after we support general merge in stateless_simple_agg + | AggKind::BoolAnd + | AggKind::BoolOr }; } pub use simply_cannot_two_phase; @@ -423,6 +425,8 @@ pub mod agg_kinds { | AggKind::Sum0 | AggKind::Count | AggKind::BitXor + | AggKind::BoolAnd + | AggKind::BoolOr | AggKind::ApproxCountDistinct }; } @@ -455,8 +459,6 @@ impl AggKind { AggKind::BitAnd | AggKind::BitOr | AggKind::BitXor - | AggKind::BoolAnd - | AggKind::BoolOr | AggKind::Min | AggKind::Max | AggKind::Sum => Some(self), diff --git a/src/expr/src/agg/general.rs b/src/expr/src/agg/general.rs index 1ee2017686ee9..8971c48c278c7 100644 --- a/src/expr/src/agg/general.rs +++ b/src/expr/src/agg/general.rs @@ -30,24 +30,19 @@ use crate::{ExprError, Result}; #[aggregate("sum(interval) -> interval")] #[aggregate("sum(int256) -> int256")] #[aggregate("sum0(int64) -> int64", init_state = "0i64")] -fn sum(state: Option, input: Option, retract: bool) -> Result> +fn sum(state: S, input: T, retract: bool) -> Result where S: Default + From + CheckedAdd + CheckedSub, { - let Some(input) = input else { - return Ok(state); - }; - let state = state.unwrap_or_default(); - let result = if retract { + if retract { state .checked_sub(&S::from(input)) - .ok_or_else(|| ExprError::NumericOutOfRange)? + .ok_or_else(|| ExprError::NumericOutOfRange) } else { state .checked_add(&S::from(input)) - .ok_or_else(|| ExprError::NumericOutOfRange)? - }; - Ok(Some(result)) + .ok_or_else(|| ExprError::NumericOutOfRange) + } } #[aggregate("min(*) -> auto", state = "ref")] @@ -60,7 +55,9 @@ fn max(state: T, input: T) -> T { state.max(input) } -#[aggregate("bit_and(*int) -> auto")] +// XXX: state = "ref" is required so that +// for the first non-null value, the state is set to that value. +#[aggregate("bit_and(*int) -> auto", state = "ref")] fn bit_and(state: T, input: T) -> T where T: BitAnd, @@ -139,84 +136,6 @@ fn count_star(state: i64, retract: bool) -> i64 { } } -/// Returns true if all non-null input values are true, otherwise false. -/// -/// # Example -/// -/// ```slt -/// statement ok -/// create table t (b1 boolean, b2 boolean, b3 boolean, b4 boolean); -/// -/// query T -/// select bool_and(b1) from t; -/// ---- -/// NULL -/// -/// statement ok -/// insert into t values -/// (true, null, false, null), -/// (false, true, null, null), -/// (null, true, false, null); -/// -/// query TTTTTT -/// select -/// bool_and(b1), -/// bool_and(b2), -/// bool_and(b3), -/// bool_and(b4), -/// bool_and(NOT b2), -/// bool_and(NOT b3) -/// FROM t; -/// ---- -/// f t f NULL f t -/// -/// statement ok -/// drop table t; -/// ``` -#[aggregate("bool_and(boolean) -> boolean")] -fn bool_and(state: bool, input: bool) -> bool { - state && input -} - -/// Returns true if any non-null input value is true, otherwise false. -/// -/// # Example -/// -/// ```slt -/// statement ok -/// create table t (b1 boolean, b2 boolean, b3 boolean, b4 boolean); -/// -/// query T -/// select bool_or(b1) from t; -/// ---- -/// NULL -/// -/// statement ok -/// insert into t values -/// (true, null, false, null), -/// (false, true, null, null), -/// (null, true, false, null); -/// -/// query TTTTTT -/// select -/// bool_or(b1), -/// bool_or(b2), -/// bool_or(b3), -/// bool_or(b4), -/// bool_or(NOT b2), -/// bool_or(NOT b3) -/// FROM t; -/// ---- -/// t t f NULL f t -/// -/// statement ok -/// drop table t; -/// ``` -#[aggregate("bool_or(boolean) -> boolean")] -fn bool_or(state: bool, input: bool) -> bool { - state || input -} - #[cfg(test)] mod tests { extern crate test; @@ -232,7 +151,7 @@ mod tests { use crate::agg::AggCall; fn test_agg(pretty: &str, input: StreamChunk, expected: Datum) { - let agg = crate::agg::build(&AggCall::from_pretty(pretty)).unwrap(); + let agg = crate::agg::build_append_only(&AggCall::from_pretty(pretty)).unwrap(); let mut state = agg.create_state(); agg.update(&mut state, &input) .now_or_never() @@ -515,7 +434,7 @@ mod tests { }; let chunk = StreamChunk::from_parts(ops, DataChunk::new(vec![Arc::new(data)], vis)); let pretty = format!("({agg_desc}:int8 $0:int8)"); - let agg = crate::agg::build(&AggCall::from_pretty(pretty)).unwrap(); + let agg = crate::agg::build_append_only(&AggCall::from_pretty(pretty)).unwrap(); let mut state = agg.create_state(); b.iter(|| { agg.update(&mut state, &chunk) diff --git a/src/expr/src/agg/mod.rs b/src/expr/src/agg/mod.rs index d6f05498b8527..15236ee642440 100644 --- a/src/expr/src/agg/mod.rs +++ b/src/expr/src/agg/mod.rs @@ -29,6 +29,8 @@ mod def; // concrete AggregateFunctions mod approx_count_distinct; mod array_agg; +mod bool_and; +mod bool_or; mod general; mod jsonb_agg; mod mode; @@ -62,6 +64,19 @@ pub trait AggregateFunction: Send + Sync + 'static { /// Get aggregate result from the state. async fn get_result(&self, state: &AggregateState) -> Result; + + /// Encode the state into a datum that can be stored in state table. + fn encode_state(&self, state: &AggregateState) -> Result { + match state { + AggregateState::Datum(d) => Ok(d.clone()), + _ => panic!("cannot encode state"), + } + } + + /// Decode the state from a datum in state table. + fn decode_state(&self, datum: Datum) -> Result { + Ok(AggregateState::Datum(datum)) + } } /// Intermediate state of an aggregate function. @@ -118,19 +133,27 @@ impl AggregateState { pub type BoxedAggregateFunction = Box; -/// Build an `AggregateFunction` from `AggCall`. +/// Build an append-only `Aggregator` from `AggCall`. +pub fn build_append_only(agg: &AggCall) -> Result { + build(agg, true) +} + +/// Build a retractable `Aggregator` from `AggCall`. +pub fn build_retractable(agg: &AggCall) -> Result { + build(agg, false) +} + +/// Build an `Aggregator` from `AggCall`. /// /// NOTE: This function ignores argument indices, `column_orders`, `filter` and `distinct` in /// `AggCall`. Such operations should be done in batch or streaming executors. -pub fn build(agg: &AggCall) -> Result { - // NOTE: The function signature is checked by `AggCall::infer_return_type` in the frontend. - +pub fn build(agg: &AggCall, append_only: bool) -> Result { let args = (agg.args.arg_types().iter()) .map(|t| t.into()) .collect::>(); let ret_type = (&agg.return_type).into(); let desc = crate::sig::agg::AGG_FUNC_SIG_MAP - .get(agg.kind, &args, ret_type) + .get(agg.kind, &args, ret_type, append_only) .ok_or_else(|| { ExprError::UnsupportedFunction(format!( "{:?}", @@ -140,6 +163,7 @@ pub fn build(agg: &AggCall) -> Result { ret_type, set_returning: false, deprecated: false, + append_only, } )) })?; diff --git a/src/expr/src/agg/string_agg.rs b/src/expr/src/agg/string_agg.rs index 2c37970c7413c..7b7a06ffe5567 100644 --- a/src/expr/src/agg/string_agg.rs +++ b/src/expr/src/agg/string_agg.rs @@ -47,7 +47,7 @@ mod tests { + ccc , + ddd ,", ); - let string_agg = crate::agg::build(&AggCall::from_pretty( + let string_agg = crate::agg::build_append_only(&AggCall::from_pretty( "(string_agg:varchar $0:varchar $1:varchar)", ))?; let mut state = string_agg.create_state(); @@ -68,7 +68,7 @@ mod tests { + ccc _ + ddd .", ); - let string_agg = crate::agg::build(&AggCall::from_pretty( + let string_agg = crate::agg::build_append_only(&AggCall::from_pretty( "(string_agg:varchar $0:varchar $1:varchar)", ))?; let mut state = string_agg.create_state(); diff --git a/src/expr/src/error.rs b/src/expr/src/error.rs index e5136ac406a45..d1ae1eb35ad0c 100644 --- a/src/expr/src/error.rs +++ b/src/expr/src/error.rs @@ -76,6 +76,9 @@ pub enum ExprError { #[error("field name must not be null")] FieldNameNull, + + #[error("invalid state: {0}")] + InvalidState(String), } static_assertions::const_assert_eq!(std::mem::size_of::(), 40); diff --git a/src/expr/src/expr/build.rs b/src/expr/src/expr/build.rs index a945481592f0c..1f34adead8855 100644 --- a/src/expr/src/expr/build.rs +++ b/src/expr/src/expr/build.rs @@ -102,6 +102,7 @@ pub fn build_func( ret_type: (&ret_type).into(), set_returning: false, deprecated: false, + append_only: false, } )) })?; diff --git a/src/expr/src/sig/agg.rs b/src/expr/src/sig/agg.rs index a5321ce726aae..455dc368eaa47 100644 --- a/src/expr/src/sig/agg.rs +++ b/src/expr/src/sig/agg.rs @@ -36,8 +36,10 @@ pub static AGG_FUNC_SIG_MAP: LazyLock = LazyLock::new(|| unsafe { pub struct AggFuncSig { pub func: AggKind, pub inputs_type: &'static [DataTypeName], + pub state_type: DataTypeName, pub ret_type: DataTypeName, pub build: fn(agg: &AggCall) -> Result, + pub append_only: bool, } impl fmt::Debug for AggFuncSig { @@ -48,6 +50,7 @@ impl fmt::Debug for AggFuncSig { ret_type: self.ret_type, set_returning: false, deprecated: false, + append_only: self.append_only, } .fmt(f) } @@ -58,21 +61,32 @@ impl fmt::Debug for AggFuncSig { pub struct AggFuncSigMap(HashMap<(AggKind, usize), Vec>); impl AggFuncSigMap { + /// Inserts a function signature into the map. fn insert(&mut self, sig: AggFuncSig) { let arity = sig.inputs_type.len(); self.0.entry((sig.func, arity)).or_default().push(sig); } - /// Returns a function signature with the given type, argument types and return type. + /// Returns a function signature with the given type, argument types, return type. + /// + /// The `append_only` flag only works when both append-only and retractable version exist. + /// Otherwise, return the signature of the only version. pub fn get( &self, ty: AggKind, args: &[DataTypeName], ret: DataTypeName, + append_only: bool, ) -> Option<&AggFuncSig> { let v = self.0.get(&(ty, args.len()))?; - v.iter() - .find(|d| d.inputs_type == args && d.ret_type == ret) + let mut iter = v + .iter() + .filter(|d| d.inputs_type == args && d.ret_type == ret); + if iter.clone().count() == 2 { + iter.find(|d| d.append_only == append_only) + } else { + iter.next() + } } /// Returns the return type for the given function and arguments. diff --git a/src/expr/src/sig/func.rs b/src/expr/src/sig/func.rs index 5dca4da2f4486..afacf43aee6b0 100644 --- a/src/expr/src/sig/func.rs +++ b/src/expr/src/sig/func.rs @@ -89,6 +89,7 @@ impl fmt::Debug for FuncSign { ret_type: self.ret_type, set_returning: false, deprecated: self.deprecated, + append_only: false, } .fmt(f) } diff --git a/src/expr/src/sig/mod.rs b/src/expr/src/sig/mod.rs index cea417a3ca4ee..052b31ad060bc 100644 --- a/src/expr/src/sig/mod.rs +++ b/src/expr/src/sig/mod.rs @@ -29,20 +29,27 @@ pub(crate) struct FuncSigDebug<'a, T> { pub ret_type: DataTypeName, pub set_returning: bool, pub deprecated: bool, + pub append_only: bool, } impl<'a, T: std::fmt::Display> std::fmt::Debug for FuncSigDebug<'a, T> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let s = format!( - "{}({:?}) -> {}{:?}{}", + "{}({:?}) -> {}{:?}", self.func, self.inputs_type.iter().format(", "), if self.set_returning { "setof " } else { "" }, self.ret_type, - if self.deprecated { " [deprecated]" } else { "" }, ) .to_ascii_lowercase(); - f.write_str(&s) + f.write_str(&s)?; + if self.append_only { + write!(f, " [append-only]")?; + } + if self.deprecated { + write!(f, " [deprecated]")?; + } + Ok(()) } } diff --git a/src/expr/src/sig/table_function.rs b/src/expr/src/sig/table_function.rs index a8ebce5e378bd..f4b2ac6de1e4b 100644 --- a/src/expr/src/sig/table_function.rs +++ b/src/expr/src/sig/table_function.rs @@ -91,6 +91,7 @@ impl fmt::Debug for FuncSign { ret_type: self.ret_type, set_returning: true, deprecated: false, + append_only: false, } .fmt(f) } diff --git a/src/expr/src/table_function/mod.rs b/src/expr/src/table_function/mod.rs index 2ca3f28513b46..23453d9f7b956 100644 --- a/src/expr/src/table_function/mod.rs +++ b/src/expr/src/table_function/mod.rs @@ -150,6 +150,7 @@ pub fn build( ret_type: (&return_type).into(), set_returning: true, deprecated: false, + append_only: false, } )) })?; diff --git a/src/expr/src/window_function/state/aggregate.rs b/src/expr/src/window_function/state/aggregate.rs index 15b01fabc2711..b67d42042bc91 100644 --- a/src/expr/src/window_function/state/aggregate.rs +++ b/src/expr/src/window_function/state/aggregate.rs @@ -24,7 +24,7 @@ use smallvec::SmallVec; use super::buffer::WindowBuffer; use super::{StateEvictHint, StateKey, StatePos, WindowState}; -use crate::agg::{build as builg_agg, AggArgs, AggCall, BoxedAggregateFunction}; +use crate::agg::{build_append_only, AggArgs, AggCall, BoxedAggregateFunction}; use crate::function::window::{WindowFuncCall, WindowFuncKind}; use crate::Result; @@ -86,7 +86,7 @@ impl WindowState for AggregateState { fn curr_output(&self) -> Result { let wrapper = AggregatorWrapper { - agg: builg_agg(&self.agg_call)?, + agg: build_append_only(&self.agg_call)?, arg_data_types: &self.arg_data_types, }; wrapper.aggregate(self.buffer.curr_window_values().map(SmallVec::as_slice)) diff --git a/src/expr/src/window_function/state/mod.rs b/src/expr/src/window_function/state/mod.rs index becf633107df6..fe6e47b016d40 100644 --- a/src/expr/src/window_function/state/mod.rs +++ b/src/expr/src/window_function/state/mod.rs @@ -128,6 +128,7 @@ pub fn create_window_state(call: &WindowFuncCall) -> Result 200:Decimal) } └── StreamProject { exprs: [customer.c_id, customer.c_last, orders.o_id, orders.o_d_id, orders.o_w_id, orders.o_entry_d, orders.o_ol_cnt, sum(order_line.ol_amount)] } - └── StreamHashAgg { group_key: [customer.c_id, customer.c_last, orders.o_id, orders.o_d_id, orders.o_w_id, orders.o_entry_d, orders.o_ol_cnt], aggs: [sum(order_line.ol_amount), count] } { result table: 0, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [customer.c_id, customer.c_last, orders.o_id, orders.o_d_id, orders.o_w_id, orders.o_entry_d, orders.o_ol_cnt], aggs: [sum(order_line.ol_amount), count] } { intermediate state table: 0, state tables: [], distinct tables: [] } └── StreamHashJoin { type: Inner, predicate: orders.o_w_id = order_line.ol_w_id AND orders.o_d_id = order_line.ol_d_id AND orders.o_id = order_line.ol_o_id, output: [customer.c_id, customer.c_last, orders.o_id, orders.o_d_id, orders.o_w_id, orders.o_entry_d, orders.o_ol_cnt, order_line.ol_amount, customer.c_w_id, customer.c_d_id, order_line.ol_w_id, order_line.ol_d_id, order_line.ol_o_id, order_line.ol_number] } ├── left table: 1 ├── right table: 3 @@ -3060,7 +3063,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(sum(order_line.ol_amount))] } └── StreamSimpleAgg { aggs: [sum(sum(order_line.ol_amount)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -3218,7 +3221,7 @@ StreamProject { exprs: [((stock.s_i_id * stock.s_w_id) % 10000:Int32)::Int64 as $expr1, stock.s_i_id, stock.s_w_id, stock.s_quantity] } └── StreamFilter { predicate: ((2:Int32 * stock.s_quantity) > sum(order_line.ol_quantity)) } └── StreamProject { exprs: [stock.s_i_id, stock.s_w_id, stock.s_quantity, sum(order_line.ol_quantity)] } - └── StreamHashAgg { group_key: [stock.s_i_id, stock.s_w_id, stock.s_quantity], aggs: [sum(order_line.ol_quantity), count] } { result table: 10, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [stock.s_i_id, stock.s_w_id, stock.s_quantity], aggs: [sum(order_line.ol_quantity), count] } { intermediate state table: 10, state tables: [], distinct tables: [] } └── StreamHashJoin { type: LeftSemi, predicate: stock.s_i_id = item.i_id, output: all } { left table: 11, right table: 13, left degree table: 12, right degree table: 14 } ├── StreamHashJoin { type: Inner, predicate: stock.s_i_id = order_line.ol_i_id, output: [stock.s_i_id, stock.s_w_id, stock.s_quantity, order_line.ol_quantity, order_line.ol_w_id, order_line.ol_d_id, order_line.ol_o_id, order_line.ol_number] } │ ├── left table: 15 @@ -3389,7 +3392,7 @@ StreamMaterialize { columns: [s_name, numwait], stream_key: [s_name], pk_columns: [numwait, s_name], pk_conflict: NoCheck } ├── materialized table: 4294967294 └── StreamHashAgg { group_key: [supplier.s_name], aggs: [count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 diff --git a/src/frontend/planner_test/tests/testdata/output/distribution_derive.yaml b/src/frontend/planner_test/tests/testdata/output/distribution_derive.yaml index 00b9c2339413d..0846fb23192fa 100644 --- a/src/frontend/planner_test/tests/testdata/output/distribution_derive.yaml +++ b/src/frontend/planner_test/tests/testdata/output/distribution_derive.yaml @@ -240,7 +240,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(a.v), a.k1] } └── StreamHashAgg { group_key: [a.k1], aggs: [max(a.v), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -303,7 +303,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(ak1.v), ak1.k1] } └── StreamHashAgg { group_key: [ak1.k1], aggs: [max(ak1.v), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── Chain { table: ak1, columns: [ak1.k1, ak1.v, ak1.a._row_id], pk: [ak1.a._row_id], dist: UpstreamHashShard(ak1.k1) } @@ -362,7 +362,10 @@ StreamMaterialize { columns: [max_v, ak1k2.k1(hidden)], stream_key: [ak1k2.k1], pk_columns: [ak1k2.k1], pk_conflict: NoCheck } ├── materialized table: 4294967294 └── StreamProject { exprs: [max(ak1k2.v), ak1k2.k1] } - └── StreamHashAgg { group_key: [ak1k2.k1], aggs: [max(ak1k2.v), count] } { result table: 1, state tables: [ 0 ], distinct tables: [] } + └── StreamHashAgg { group_key: [ak1k2.k1], aggs: [max(ak1k2.v), count] } + ├── intermediate state table: 1 + ├── state tables: [ 0 ] + ├── distinct tables: [] └── Chain { table: ak1k2, columns: [ak1k2.k1, ak1k2.v, ak1k2.k2, ak1k2.a._row_id], pk: [ak1k2.a._row_id], dist: UpstreamHashShard(ak1k2.k1) } ├── state table: 2 ├── Upstream @@ -422,7 +425,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(ak1k2.v), ak1k2.k2] } └── StreamHashAgg { group_key: [ak1k2.k2], aggs: [max(ak1k2.v), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -484,7 +487,7 @@ StreamMaterialize { columns: [sum_v, ak1k2.k1(hidden), ak1k2.k2(hidden)], stream_key: [ak1k2.k1, ak1k2.k2], pk_columns: [ak1k2.k1, ak1k2.k2], pk_conflict: NoCheck } ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(ak1k2.v), ak1k2.k1, ak1k2.k2] } - └── StreamHashAgg { group_key: [ak1k2.k1, ak1k2.k2], aggs: [sum(ak1k2.v), count] } { result table: 0, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [ak1k2.k1, ak1k2.k2], aggs: [sum(ak1k2.v), count] } { intermediate state table: 0, state tables: [], distinct tables: [] } └── Chain { table: ak1k2, columns: [ak1k2.k1, ak1k2.k2, ak1k2.v, ak1k2.a._row_id], pk: [ak1k2.a._row_id], dist: UpstreamHashShard(ak1k2.k1) } ├── state table: 1 ├── Upstream @@ -534,7 +537,7 @@ StreamMaterialize { columns: [sum_v, ak1.k1(hidden), ak1.k2(hidden)], stream_key: [ak1.k1, ak1.k2], pk_columns: [ak1.k1, ak1.k2], pk_conflict: NoCheck } ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(ak1.v), ak1.k1, ak1.k2] } - └── StreamHashAgg { group_key: [ak1.k1, ak1.k2], aggs: [sum(ak1.v), count] } { result table: 0, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [ak1.k1, ak1.k2], aggs: [sum(ak1.v), count] } { intermediate state table: 0, state tables: [], distinct tables: [] } └── Chain { table: ak1, columns: [ak1.k1, ak1.k2, ak1.v, ak1.a._row_id], pk: [ak1.a._row_id], dist: UpstreamHashShard(ak1.k1) } ├── state table: 1 ├── Upstream @@ -595,11 +598,11 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(count), a.k1] } └── StreamHashAgg { group_key: [a.k1], aggs: [max(count), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamHashAgg { group_key: [a.k1], aggs: [count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -694,13 +697,16 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(count), a.k1] } └── StreamHashAgg { group_key: [a.k1], aggs: [max(count), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 Fragment 1 - StreamHashAgg { group_key: [a.k1, a.k2], aggs: [count] } { result table: 2, state tables: [], distinct tables: [] } + StreamHashAgg { group_key: [a.k1, a.k2], aggs: [count] } + ├── intermediate state table: 2 + ├── state tables: [] + ├── distinct tables: [] └── StreamExchange Hash([0, 1]) from 2 Fragment 2 @@ -793,13 +799,16 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(count), a.k2] } └── StreamHashAgg { group_key: [a.k2], aggs: [max(count), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([1]) from 1 Fragment 1 - StreamHashAgg { group_key: [a.k1, a.k2], aggs: [count] } { result table: 2, state tables: [], distinct tables: [] } + StreamHashAgg { group_key: [a.k1, a.k2], aggs: [count] } + ├── intermediate state table: 2 + ├── state tables: [] + ├── distinct tables: [] └── StreamExchange Hash([0, 1]) from 2 Fragment 2 @@ -876,8 +885,11 @@ StreamMaterialize { columns: [max_num, a.k1(hidden), a.k2(hidden)], stream_key: [a.k1, a.k2], pk_columns: [a.k1, a.k2], pk_conflict: NoCheck } ├── materialized table: 4294967294 └── StreamProject { exprs: [max(count), a.k1, a.k2] } - └── StreamHashAgg { group_key: [a.k1, a.k2], aggs: [max(count), count] } { result table: 1, state tables: [ 0 ], distinct tables: [] } - └── StreamHashAgg { group_key: [a.k1, a.k2], aggs: [count] } { result table: 2, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [a.k1, a.k2], aggs: [max(count), count] } + ├── intermediate state table: 1 + ├── state tables: [ 0 ] + ├── distinct tables: [] + └── StreamHashAgg { group_key: [a.k1, a.k2], aggs: [count] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1]) from 1 Fragment 1 @@ -963,7 +975,7 @@ ├── materialized table: 4294967294 └── StreamHashJoin { type: Inner, predicate: ak1.k1 = a.k1, output: [ak1.v, count, ak1.a._row_id, ak1.k1, a.k1] } { left table: 0, right table: 2, left degree table: 1, right degree table: 3 } ├── StreamExchange Hash([0]) from 1 - └── StreamHashAgg { group_key: [a.k1], aggs: [count] } { result table: 5, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [a.k1], aggs: [count] } { intermediate state table: 5, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 2 Fragment 1 @@ -1036,7 +1048,7 @@ ├── right table: 2 ├── left degree table: 1 ├── right degree table: 3 - ├── StreamHashAgg { group_key: [a.k1], aggs: [count] } { result table: 4, state tables: [], distinct tables: [] } + ├── StreamHashAgg { group_key: [a.k1], aggs: [count] } { intermediate state table: 4, state tables: [], distinct tables: [] } │ └── StreamExchange Hash([0]) from 1 └── StreamExchange Hash([0]) from 2 @@ -1139,9 +1151,9 @@ ├── right table: 2 ├── left degree table: 1 ├── right degree table: 3 - ├── StreamHashAgg { group_key: [a.k1], aggs: [count] } { result table: 4, state tables: [], distinct tables: [] } + ├── StreamHashAgg { group_key: [a.k1], aggs: [count] } { intermediate state table: 4, state tables: [], distinct tables: [] } │ └── StreamExchange Hash([0]) from 1 - └── StreamHashAgg { group_key: [b.k1], aggs: [count] } { result table: 6, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [b.k1], aggs: [count] } { intermediate state table: 6, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 2 Fragment 1 diff --git a/src/frontend/planner_test/tests/testdata/output/emit_on_window_close.yaml b/src/frontend/planner_test/tests/testdata/output/emit_on_window_close.yaml index 7bbe8da04ff82..9a0a6bcae433e 100644 --- a/src/frontend/planner_test/tests/testdata/output/emit_on_window_close.yaml +++ b/src/frontend/planner_test/tests/testdata/output/emit_on_window_close.yaml @@ -36,7 +36,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [v1, min(v2), count(distinct v3)], output_watermarks: [v1] } └── StreamHashAgg [append_only, eowc] { group_key: [v1], aggs: [min(v2), count(distinct v3), count], output_watermarks: [v1] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [ (distinct key: v3, table id: 1) ] └── StreamExchange Hash([0]) from 1 @@ -109,7 +109,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [$expr1, max(t.b)], output_watermarks: [$expr1] } └── StreamHashAgg [append_only, eowc] { group_key: [$expr1], aggs: [max(t.b), count], output_watermarks: [$expr1] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 diff --git a/src/frontend/planner_test/tests/testdata/output/except.yaml b/src/frontend/planner_test/tests/testdata/output/except.yaml index 655d826dc5ab3..e57bf061d5335 100644 --- a/src/frontend/planner_test/tests/testdata/output/except.yaml +++ b/src/frontend/planner_test/tests/testdata/output/except.yaml @@ -36,7 +36,7 @@ Fragment 0 StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [t1.a, t1.b, t1.c] } - └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { result table: 0, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { intermediate state table: 0, state tables: [], distinct tables: [] } └── StreamHashJoin { type: LeftAnti, predicate: t1.a IS NOT DISTINCT FROM t2.a AND t1.b IS NOT DISTINCT FROM t2.b AND t1.c IS NOT DISTINCT FROM t2.c, output: all } ├── left table: 1 ├── right table: 3 @@ -131,7 +131,7 @@ Fragment 0 StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [t1.a, t1.b, t1.c] } - └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { result table: 0, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { intermediate state table: 0, state tables: [], distinct tables: [] } └── StreamHashJoin { type: LeftAnti, predicate: t1.a IS NOT DISTINCT FROM t2.a AND t1.b IS NOT DISTINCT FROM t2.b AND t1.c IS NOT DISTINCT FROM t2.c, output: all } ├── left table: 1 ├── right table: 3 diff --git a/src/frontend/planner_test/tests/testdata/output/intersect.yaml b/src/frontend/planner_test/tests/testdata/output/intersect.yaml index 1037a3a401599..06804444a4f9b 100644 --- a/src/frontend/planner_test/tests/testdata/output/intersect.yaml +++ b/src/frontend/planner_test/tests/testdata/output/intersect.yaml @@ -36,7 +36,7 @@ Fragment 0 StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [t1.a, t1.b, t1.c] } - └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { result table: 0, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { intermediate state table: 0, state tables: [], distinct tables: [] } └── StreamHashJoin { type: LeftSemi, predicate: t1.a IS NOT DISTINCT FROM t2.a AND t1.b IS NOT DISTINCT FROM t2.b AND t1.c IS NOT DISTINCT FROM t2.c, output: all } ├── left table: 1 ├── right table: 3 @@ -131,7 +131,7 @@ Fragment 0 StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [t1.a, t1.b, t1.c] } - └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { result table: 0, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { intermediate state table: 0, state tables: [], distinct tables: [] } └── StreamHashJoin { type: LeftSemi, predicate: t1.a IS NOT DISTINCT FROM t2.a AND t1.b IS NOT DISTINCT FROM t2.b AND t1.c IS NOT DISTINCT FROM t2.c, output: all } ├── left table: 1 ├── right table: 3 diff --git a/src/frontend/planner_test/tests/testdata/output/nexmark.yaml b/src/frontend/planner_test/tests/testdata/output/nexmark.yaml index b4dd68f7b04fc..c3bed3a331fc6 100644 --- a/src/frontend/planner_test/tests/testdata/output/nexmark.yaml +++ b/src/frontend/planner_test/tests/testdata/output/nexmark.yaml @@ -272,7 +272,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [auction.category, (sum(max(bid.price)) / count(max(bid.price))::Decimal) as $expr1] } └── StreamHashAgg { group_key: [auction.category], aggs: [sum(max(bid.price)), count(max(bid.price)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([1]) from 1 @@ -280,7 +280,7 @@ Fragment 1 StreamProject { exprs: [auction.id, auction.category, max(bid.price)] } └── StreamHashAgg { group_key: [auction.id, auction.category], aggs: [max(bid.price), count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [ 1 ] ├── distinct tables: [] └── StreamProject { exprs: [auction.id, auction.category, bid.price, bid._row_id] } @@ -463,7 +463,7 @@ └── StreamHashJoin { type: Inner, predicate: window_start = window_start, output: all } { left table: 0, right table: 2, left degree table: 1, right degree table: 3 } ├── StreamExchange Hash([1]) from 1 └── StreamProject { exprs: [window_start, max(count)] } - └── StreamHashAgg { group_key: [window_start], aggs: [max(count), count] } { result table: 7, state tables: [ 6 ], distinct tables: [] } + └── StreamHashAgg { group_key: [window_start], aggs: [max(count), count] } { intermediate state table: 7, state tables: [ 6 ], distinct tables: [] } └── StreamExchange Hash([1]) from 4 Fragment 1 @@ -471,7 +471,7 @@ └── StreamExchange NoShuffle from 2 Fragment 2 - StreamHashAgg [append_only] { group_key: [bid.auction, window_start], aggs: [count] } { result table: 4, state tables: [], distinct tables: [] } + StreamHashAgg [append_only] { group_key: [bid.auction, window_start], aggs: [count] } { intermediate state table: 4, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1]) from 3 Fragment 3 @@ -737,7 +737,7 @@ Fragment 2 StreamProject { exprs: [$expr1, max(bid.price), ($expr1 - '00:00:10':Interval) as $expr2] } - └── StreamHashAgg [append_only] { group_key: [$expr1], aggs: [max(bid.price), count] } { result table: 5, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr1], aggs: [max(bid.price), count] } { intermediate state table: 5, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 3 Fragment 3 @@ -838,12 +838,12 @@ └── StreamHashJoin { type: Inner, predicate: person.id = auction.seller AND $expr1 = $expr3 AND $expr2 = $expr4, output: all } { left table: 0, right table: 2, left degree table: 1, right degree table: 3 } ├── StreamExchange Hash([0, 2, 3]) from 1 └── StreamProject { exprs: [auction.seller, $expr3, $expr4] } - └── StreamHashAgg { group_key: [auction.seller, $expr3, $expr4], aggs: [count] } { result table: 6, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [auction.seller, $expr3, $expr4], aggs: [count] } { intermediate state table: 6, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1, 2]) from 2 Fragment 1 StreamProject { exprs: [person.id, person.name, $expr1, $expr2] } - └── StreamHashAgg { group_key: [person.id, person.name, $expr1, $expr2], aggs: [count] } { result table: 4, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [person.id, person.name, $expr1, $expr2], aggs: [count] } { intermediate state table: 4, state tables: [], distinct tables: [] } └── StreamProject { exprs: [person.id, person.name, $expr1, ($expr1 + '00:00:10':Interval) as $expr2] } └── StreamProject { exprs: [person.id, person.name, person.date_time, TumbleStart(person.date_time, '00:00:10':Interval) as $expr1] } └── Chain { table: person, columns: [person.id, person.name, person.date_time], pk: [person.id], dist: UpstreamHashShard(person.id) } { state table: 5 } @@ -1054,7 +1054,7 @@ StreamMaterialize { columns: [bidder, bid_count, window_start, window_end], stream_key: [bidder, window_start, window_end], pk_columns: [bidder, window_start, window_end], pk_conflict: NoCheck, watermark_columns: [window_start, window_end] } ├── materialized table: 4294967294 └── StreamProject { exprs: [bid.bidder, count, $expr1, $expr2], output_watermarks: [$expr1, $expr2] } - └── StreamHashAgg [append_only] { group_key: [bid.bidder, $expr1, $expr2], aggs: [count], output_watermarks: [$expr1, $expr2] } { result table: 0, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [bid.bidder, $expr1, $expr2], aggs: [count], output_watermarks: [$expr1, $expr2] } { intermediate state table: 0, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1, 2]) from 1 Fragment 1 @@ -1214,7 +1214,7 @@ Fragment 0 StreamMaterialize { columns: [day, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [day], pk_columns: [day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamHashAgg [append_only] { group_key: [$expr1], aggs: [count, count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), count(distinct bid.bidder), count(distinct bid.bidder) filter((bid.price < 10000:Int32)), count(distinct bid.bidder) filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count(distinct bid.bidder) filter((bid.price >= 1000000:Int32)), count(distinct bid.auction), count(distinct bid.auction) filter((bid.price < 10000:Int32)), count(distinct bid.auction) filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count(distinct bid.auction) filter((bid.price >= 1000000:Int32))] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [ (distinct key: bid.bidder, table id: 1), (distinct key: bid.auction, table id: 2) ] └── StreamExchange Hash([0]) from 1 @@ -1275,13 +1275,13 @@ StreamMaterialize { columns: [day, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [day], pk_columns: [day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [$expr1, sum0(count) filter((flag = 0:Int64)), sum0(count filter((bid.price < 10000:Int32))) filter((flag = 0:Int64)), sum0(count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32))) filter((flag = 0:Int64)), sum0(count filter((bid.price >= 1000000:Int32))) filter((flag = 0:Int64)), count(bid.bidder) filter((flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.auction) filter((flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64))] } └── StreamHashAgg { group_key: [$expr1], aggs: [sum0(count) filter((flag = 0:Int64)), sum0(count filter((bid.price < 10000:Int32))) filter((flag = 0:Int64)), sum0(count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32))) filter((flag = 0:Int64)), sum0(count filter((bid.price >= 1000000:Int32))) filter((flag = 0:Int64)), count(bid.bidder) filter((flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.auction) filter((flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 Fragment 1 - StreamHashAgg [append_only] { group_key: [$expr1, bid.bidder, bid.auction, flag], aggs: [count, count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32))] } { result table: 1, state tables: [], distinct tables: [] } + StreamHashAgg [append_only] { group_key: [$expr1, bid.bidder, bid.auction, flag], aggs: [count, count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32))] } { intermediate state table: 1, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 2, 3, 10]) from 2 Fragment 2 @@ -1347,7 +1347,7 @@ Fragment 0 StreamMaterialize { columns: [channel, day, minute, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [channel, day], pk_columns: [channel, day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamHashAgg [append_only] { group_key: [bid.channel, $expr1], aggs: [max($expr2), count, count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), count(distinct bid.bidder), count(distinct bid.bidder) filter((bid.price < 10000:Int32)), count(distinct bid.bidder) filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count(distinct bid.bidder) filter((bid.price >= 1000000:Int32)), count(distinct bid.auction), count(distinct bid.auction) filter((bid.price < 10000:Int32)), count(distinct bid.auction) filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count(distinct bid.auction) filter((bid.price >= 1000000:Int32))] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [ (distinct key: bid.bidder, table id: 1), (distinct key: bid.auction, table id: 2) ] └── StreamExchange Hash([0, 1]) from 1 @@ -1410,13 +1410,13 @@ StreamMaterialize { columns: [channel, day, minute, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [channel, day], pk_columns: [channel, day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [bid.channel, $expr1, max(max($expr2)) filter((flag = 0:Int64)), sum0(count) filter((flag = 0:Int64)), sum0(count filter((bid.price < 10000:Int32))) filter((flag = 0:Int64)), sum0(count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32))) filter((flag = 0:Int64)), sum0(count filter((bid.price >= 1000000:Int32))) filter((flag = 0:Int64)), count(bid.bidder) filter((flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.auction) filter((flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64))] } └── StreamHashAgg { group_key: [bid.channel, $expr1], aggs: [max(max($expr2)) filter((flag = 0:Int64)), sum0(count) filter((flag = 0:Int64)), sum0(count filter((bid.price < 10000:Int32))) filter((flag = 0:Int64)), sum0(count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32))) filter((flag = 0:Int64)), sum0(count filter((bid.price >= 1000000:Int32))) filter((flag = 0:Int64)), count(bid.bidder) filter((flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.bidder) filter((count filter((bid.price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bid.auction) filter((flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(bid.auction) filter((count filter((bid.price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([0, 1]) from 1 Fragment 1 - StreamHashAgg [append_only] { group_key: [bid.channel, $expr1, bid.bidder, bid.auction, flag], aggs: [max($expr2), count, count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32))] } { result table: 2, state tables: [], distinct tables: [] } + StreamHashAgg [append_only] { group_key: [bid.channel, $expr1, bid.bidder, bid.auction, flag], aggs: [max($expr2), count, count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32))] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1, 4, 5, 14]) from 2 Fragment 2 @@ -1479,7 +1479,7 @@ StreamMaterialize { columns: [auction, day, total_bids, rank1_bids, rank2_bids, rank3_bids, min_price, max_price, avg_price, sum_price], stream_key: [auction, day], pk_columns: [auction, day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [bid.auction, $expr1, count, count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), min(bid.price), max(bid.price), (sum(bid.price) / count(bid.price)::Decimal) as $expr2, sum(bid.price)] } └── StreamHashAgg [append_only] { group_key: [bid.auction, $expr1], aggs: [count, count filter((bid.price < 10000:Int32)), count filter((bid.price >= 10000:Int32) AND (bid.price < 1000000:Int32)), count filter((bid.price >= 1000000:Int32)), min(bid.price), max(bid.price), sum(bid.price), count(bid.price)] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0, 1]) from 1 @@ -1909,7 +1909,7 @@ ├── right degree table: 3 ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [bid.auction, max(bid.price)] } - └── StreamHashAgg [append_only] { group_key: [bid.auction], aggs: [max(bid.price), count] } { result table: 5, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [bid.auction], aggs: [max(bid.price), count] } { intermediate state table: 5, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 2 Fragment 1 @@ -2001,7 +2001,7 @@ ├── materialized table: 4294967294 └── StreamDynamicFilter { predicate: (count(bid.auction) >= $expr1), output: [auction.id, auction.item_name, count(bid.auction)] } { left table: 0, right table: 1 } ├── StreamProject { exprs: [auction.id, auction.item_name, count(bid.auction)] } - │ └── StreamHashAgg { group_key: [auction.id, auction.item_name], aggs: [count(bid.auction), count] } { result table: 2, state tables: [], distinct tables: [] } + │ └── StreamHashAgg { group_key: [auction.id, auction.item_name], aggs: [count(bid.auction), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } │ └── StreamHashJoin { type: Inner, predicate: auction.id = bid.auction, output: all } { left table: 3, right table: 5, left degree table: 4, right degree table: 6 } │ ├── StreamExchange Hash([0]) from 1 │ └── StreamExchange Hash([0]) from 2 @@ -2019,12 +2019,12 @@ Fragment 3 StreamProject { exprs: [(sum0(sum0(count)) / sum0(count(bid.auction))) as $expr1] } - └── StreamSimpleAgg { aggs: [sum0(sum0(count)), sum0(count(bid.auction)), count] } { result table: 9, state tables: [], distinct tables: [] } + └── StreamSimpleAgg { aggs: [sum0(sum0(count)), sum0(count(bid.auction)), count] } { intermediate state table: 9, state tables: [], distinct tables: [] } └── StreamExchange Single from 4 Fragment 4 StreamStatelessSimpleAgg { aggs: [sum0(count), count(bid.auction)] } - └── StreamHashAgg [append_only] { group_key: [bid.auction], aggs: [count] } { result table: 10, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [bid.auction], aggs: [count] } { intermediate state table: 10, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 5 Fragment 5 @@ -2134,7 +2134,7 @@ └── StreamProject { exprs: [bid.auction] } └── StreamFilter { predicate: (count >= 20:Int32) } └── StreamHashAgg [append_only] { group_key: [bid.auction], aggs: [count] } - ├── result table: 5 + ├── intermediate state table: 5 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 2 @@ -2252,7 +2252,7 @@ └── StreamProject { exprs: [bid.auction] } └── StreamFilter { predicate: (count < 20:Int32) } └── StreamHashAgg [append_only] { group_key: [bid.auction], aggs: [count] } - ├── result table: 5 + ├── intermediate state table: 5 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 2 @@ -2371,7 +2371,7 @@ Fragment 1 StreamGroupTopN { order: [count(bid.auction) DESC], limit: 1000, offset: 0, group_key: [$expr1] } { state table: 1 } └── StreamProject { exprs: [auction.id, auction.item_name, count(bid.auction), Vnode(auction.id) as $expr1] } - └── StreamHashAgg { group_key: [auction.id, auction.item_name], aggs: [count(bid.auction), count] } { result table: 2, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [auction.id, auction.item_name], aggs: [count(bid.auction), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamHashJoin { type: Inner, predicate: auction.id = bid.auction, output: all } { left table: 3, right table: 5, left degree table: 4, right degree table: 6 } ├── StreamExchange Hash([0]) from 2 └── StreamExchange Hash([0]) from 3 @@ -2471,19 +2471,19 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [min(min(max(bid.price)))] } └── StreamSimpleAgg { aggs: [min(min(max(bid.price))), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 Fragment 1 StreamHashAgg { group_key: [$expr1], aggs: [min(max(bid.price)), count] } - ├── result table: 3 + ├── intermediate state table: 3 ├── state tables: [ 2 ] ├── distinct tables: [] └── StreamProject { exprs: [auction.id, max(bid.price), Vnode(auction.id) as $expr1] } └── StreamHashAgg { group_key: [auction.id], aggs: [max(bid.price), count] } - ├── result table: 5 + ├── intermediate state table: 5 ├── state tables: [ 4 ] ├── distinct tables: [] └── StreamProject { exprs: [auction.id, bid.price, bid._row_id] } diff --git a/src/frontend/planner_test/tests/testdata/output/nexmark_source.yaml b/src/frontend/planner_test/tests/testdata/output/nexmark_source.yaml index 8a96d59de67bd..eecfd41ef5c00 100644 --- a/src/frontend/planner_test/tests/testdata/output/nexmark_source.yaml +++ b/src/frontend/planner_test/tests/testdata/output/nexmark_source.yaml @@ -261,7 +261,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [category, (sum(max(price)) / count(max(price))::Decimal) as $expr1] } └── StreamHashAgg { group_key: [category], aggs: [sum(max(price)), count(max(price)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([1]) from 1 @@ -269,7 +269,7 @@ Fragment 1 StreamProject { exprs: [id, category, max(price)] } └── StreamHashAgg [append_only] { group_key: [id, category], aggs: [max(price), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [] ├── distinct tables: [] └── StreamProject { exprs: [id, category, price, _row_id, _row_id] } @@ -441,7 +441,7 @@ └── StreamHashJoin { type: Inner, predicate: window_start = window_start, output: all } { left table: 0, right table: 2, left degree table: 1, right degree table: 3 } ├── StreamExchange Hash([1]) from 1 └── StreamProject { exprs: [window_start, max(count)] } - └── StreamHashAgg { group_key: [window_start], aggs: [max(count), count] } { result table: 7, state tables: [ 6 ], distinct tables: [] } + └── StreamHashAgg { group_key: [window_start], aggs: [max(count), count] } { intermediate state table: 7, state tables: [ 6 ], distinct tables: [] } └── StreamExchange Hash([1]) from 4 Fragment 1 @@ -449,7 +449,7 @@ └── StreamExchange NoShuffle from 2 Fragment 2 - StreamHashAgg [append_only] { group_key: [auction, window_start], aggs: [count] } { result table: 4, state tables: [], distinct tables: [] } + StreamHashAgg [append_only] { group_key: [auction, window_start], aggs: [count] } { intermediate state table: 4, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1]) from 3 Fragment 3 @@ -654,7 +654,7 @@ Fragment 3 StreamProject { exprs: [$expr1, max(price), ($expr1 - '00:00:10':Interval) as $expr2] } - └── StreamHashAgg [append_only] { group_key: [$expr1], aggs: [max(price), count] } { result table: 5, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr1], aggs: [max(price), count] } { intermediate state table: 5, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 4 Fragment 4 @@ -1056,7 +1056,7 @@ Fragment 0 StreamMaterialize { columns: [day, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [day], pk_columns: [day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamHashAgg [append_only] { group_key: [$expr1], aggs: [count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count(distinct bidder), count(distinct bidder) filter((price < 10000:Int32)), count(distinct bidder) filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count(distinct bidder) filter((price >= 1000000:Int32)), count(distinct auction), count(distinct auction) filter((price < 10000:Int32)), count(distinct auction) filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count(distinct auction) filter((price >= 1000000:Int32))] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [ (distinct key: bidder, table id: 1), (distinct key: auction, table id: 2) ] └── StreamExchange Hash([0]) from 1 @@ -1117,13 +1117,13 @@ StreamMaterialize { columns: [day, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [day], pk_columns: [day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [$expr1, sum0(count) filter((flag = 0:Int64)), sum0(count filter((price < 10000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 10000:Int32) AND (price < 1000000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 1000000:Int32))) filter((flag = 0:Int64)), count(bidder) filter((flag = 1:Int64)), count(bidder) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(auction) filter((flag = 2:Int64)), count(auction) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64))] } └── StreamHashAgg { group_key: [$expr1], aggs: [sum0(count) filter((flag = 0:Int64)), sum0(count filter((price < 10000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 10000:Int32) AND (price < 1000000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 1000000:Int32))) filter((flag = 0:Int64)), count(bidder) filter((flag = 1:Int64)), count(bidder) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(auction) filter((flag = 2:Int64)), count(auction) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 Fragment 1 - StreamHashAgg [append_only] { group_key: [$expr1, bidder, auction, flag], aggs: [count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32))] } { result table: 1, state tables: [], distinct tables: [] } + StreamHashAgg [append_only] { group_key: [$expr1, bidder, auction, flag], aggs: [count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32))] } { intermediate state table: 1, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 2, 3, 10]) from 2 Fragment 2 @@ -1185,15 +1185,15 @@ StreamMaterialize { columns: [day, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [day], pk_columns: [day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [$expr1, sum0(sum0(count) filter((flag = 0:Int64))), sum0(sum0(count filter((price < 10000:Int32))) filter((flag = 0:Int64))), sum0(sum0(count filter((price >= 10000:Int32) AND (price < 1000000:Int32))) filter((flag = 0:Int64))), sum0(sum0(count filter((price >= 1000000:Int32))) filter((flag = 0:Int64))), sum0(count(bidder) filter((flag = 1:Int64))), sum0(count(bidder) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64))), sum0(count(bidder) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64))), sum0(count(bidder) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64))), sum0(count(auction) filter((flag = 2:Int64))), sum0(count(auction) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64))), sum0(count(auction) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64))), sum0(count(auction) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)))] } └── StreamHashAgg { group_key: [$expr1], aggs: [sum0(sum0(count) filter((flag = 0:Int64))), sum0(sum0(count filter((price < 10000:Int32))) filter((flag = 0:Int64))), sum0(sum0(count filter((price >= 10000:Int32) AND (price < 1000000:Int32))) filter((flag = 0:Int64))), sum0(sum0(count filter((price >= 1000000:Int32))) filter((flag = 0:Int64))), sum0(count(bidder) filter((flag = 1:Int64))), sum0(count(bidder) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64))), sum0(count(bidder) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64))), sum0(count(bidder) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64))), sum0(count(auction) filter((flag = 2:Int64))), sum0(count(auction) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64))), sum0(count(auction) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64))), sum0(count(auction) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64))), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 Fragment 1 - StreamHashAgg { group_key: [$expr1, $expr2], aggs: [sum0(count) filter((flag = 0:Int64)), sum0(count filter((price < 10000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 10000:Int32) AND (price < 1000000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 1000000:Int32))) filter((flag = 0:Int64)), count(bidder) filter((flag = 1:Int64)), count(bidder) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(auction) filter((flag = 2:Int64)), count(auction) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count] } { result table: 1, state tables: [], distinct tables: [] } + StreamHashAgg { group_key: [$expr1, $expr2], aggs: [sum0(count) filter((flag = 0:Int64)), sum0(count filter((price < 10000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 10000:Int32) AND (price < 1000000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 1000000:Int32))) filter((flag = 0:Int64)), count(bidder) filter((flag = 1:Int64)), count(bidder) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(auction) filter((flag = 2:Int64)), count(auction) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count] } { intermediate state table: 1, state tables: [], distinct tables: [] } └── StreamProject { exprs: [$expr1, bidder, auction, flag, count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), Vnode($expr1, bidder, auction, flag) as $expr2] } - └── StreamHashAgg [append_only] { group_key: [$expr1, bidder, auction, flag], aggs: [count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32))] } { result table: 2, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr1, bidder, auction, flag], aggs: [count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32))] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 2, 3, 10]) from 2 Fragment 2 @@ -1268,7 +1268,7 @@ Fragment 0 StreamMaterialize { columns: [channel, day, minute, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [channel, day], pk_columns: [channel, day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamHashAgg [append_only] { group_key: [channel, $expr1], aggs: [max($expr2), count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count(distinct bidder), count(distinct bidder) filter((price < 10000:Int32)), count(distinct bidder) filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count(distinct bidder) filter((price >= 1000000:Int32)), count(distinct auction), count(distinct auction) filter((price < 10000:Int32)), count(distinct auction) filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count(distinct auction) filter((price >= 1000000:Int32))] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [ (distinct key: bidder, table id: 1), (distinct key: auction, table id: 2) ] └── StreamExchange Hash([0, 1]) from 1 @@ -1331,13 +1331,13 @@ StreamMaterialize { columns: [channel, day, minute, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [channel, day], pk_columns: [channel, day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [channel, $expr1, max(max($expr2)) filter((flag = 0:Int64)), sum0(count) filter((flag = 0:Int64)), sum0(count filter((price < 10000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 10000:Int32) AND (price < 1000000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 1000000:Int32))) filter((flag = 0:Int64)), count(bidder) filter((flag = 1:Int64)), count(bidder) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(auction) filter((flag = 2:Int64)), count(auction) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64))] } └── StreamHashAgg { group_key: [channel, $expr1], aggs: [max(max($expr2)) filter((flag = 0:Int64)), sum0(count) filter((flag = 0:Int64)), sum0(count filter((price < 10000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 10000:Int32) AND (price < 1000000:Int32))) filter((flag = 0:Int64)), sum0(count filter((price >= 1000000:Int32))) filter((flag = 0:Int64)), count(bidder) filter((flag = 1:Int64)), count(bidder) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(bidder) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 1:Int64)), count(auction) filter((flag = 2:Int64)), count(auction) filter((count filter((price < 10000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 10000:Int32) AND (price < 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count(auction) filter((count filter((price >= 1000000:Int32)) > 0:Int64) AND (flag = 2:Int64)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([0, 1]) from 1 Fragment 1 - StreamHashAgg [append_only] { group_key: [channel, $expr1, bidder, auction, flag], aggs: [max($expr2), count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32))] } { result table: 2, state tables: [], distinct tables: [] } + StreamHashAgg [append_only] { group_key: [channel, $expr1, bidder, auction, flag], aggs: [max($expr2), count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32))] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1, 4, 5, 14]) from 2 Fragment 2 @@ -1400,7 +1400,7 @@ StreamMaterialize { columns: [auction, day, total_bids, rank1_bids, rank2_bids, rank3_bids, min_price, max_price, avg_price, sum_price], stream_key: [auction, day], pk_columns: [auction, day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [auction, $expr1, count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), min(price), max(price), (sum(price) / count(price)::Decimal) as $expr2, sum(price)] } └── StreamHashAgg [append_only] { group_key: [auction, $expr1], aggs: [count, count filter((price < 10000:Int32)), count filter((price >= 10000:Int32) AND (price < 1000000:Int32)), count filter((price >= 1000000:Int32)), min(price), max(price), sum(price), count(price)] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0, 1]) from 1 @@ -1781,7 +1781,7 @@ └── StreamHashJoin { type: LeftOuter, predicate: id = auction, output: [id, item_name, max(price), _row_id, auction] } { left table: 0, right table: 2, left degree table: 1, right degree table: 3 } ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [auction, max(price)] } - └── StreamHashAgg [append_only] { group_key: [auction], aggs: [max(price), count] } { result table: 5, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [auction], aggs: [max(price), count] } { intermediate state table: 5, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 2 Fragment 1 @@ -1883,7 +1883,7 @@ ├── materialized table: 4294967294 └── StreamDynamicFilter { predicate: (count(auction) >= $expr1), output: [id, item_name, count(auction)] } { left table: 0, right table: 1 } ├── StreamProject { exprs: [id, item_name, count(auction)] } - │ └── StreamHashAgg [append_only] { group_key: [id, item_name], aggs: [count(auction), count] } { result table: 2, state tables: [], distinct tables: [] } + │ └── StreamHashAgg [append_only] { group_key: [id, item_name], aggs: [count(auction), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } │ └── StreamHashJoin [append_only] { type: Inner, predicate: id = auction, output: [id, item_name, auction, _row_id, _row_id] } │ ├── left table: 3 │ ├── right table: 5 @@ -1908,12 +1908,12 @@ Fragment 4 StreamProject { exprs: [(sum0(sum0(count)) / sum0(count(auction))) as $expr1] } - └── StreamSimpleAgg { aggs: [sum0(sum0(count)), sum0(count(auction)), count] } { result table: 9, state tables: [], distinct tables: [] } + └── StreamSimpleAgg { aggs: [sum0(sum0(count)), sum0(count(auction)), count] } { intermediate state table: 9, state tables: [], distinct tables: [] } └── StreamExchange Single from 5 Fragment 5 StreamStatelessSimpleAgg { aggs: [sum0(count), count(auction)] } - └── StreamHashAgg [append_only] { group_key: [auction], aggs: [count] } { result table: 10, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [auction], aggs: [count] } { intermediate state table: 10, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 6 Fragment 6 @@ -2004,7 +2004,7 @@ ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [auction] } └── StreamFilter { predicate: (count >= 20:Int32) } - └── StreamHashAgg [append_only] { group_key: [auction], aggs: [count] } { result table: 5, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [auction], aggs: [count] } { intermediate state table: 5, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 2 Fragment 1 @@ -2092,7 +2092,7 @@ ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [auction] } └── StreamFilter { predicate: (count < 20:Int32) } - └── StreamHashAgg [append_only] { group_key: [auction], aggs: [count] } { result table: 5, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [auction], aggs: [count] } { intermediate state table: 5, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 2 Fragment 1 @@ -2182,7 +2182,7 @@ Fragment 1 StreamGroupTopN { order: [count(auction) DESC], limit: 1000, offset: 0, group_key: [$expr1] } { state table: 1 } └── StreamProject { exprs: [id, item_name, count(auction), Vnode(id) as $expr1] } - └── StreamHashAgg [append_only] { group_key: [id, item_name], aggs: [count(auction), count] } { result table: 2, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [id, item_name], aggs: [count(auction), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamHashJoin [append_only] { type: Inner, predicate: id = auction, output: [id, item_name, auction, _row_id, _row_id] } ├── left table: 3 ├── right table: 5 @@ -2291,19 +2291,19 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [min(min(max(price)))] } └── StreamSimpleAgg { aggs: [min(min(max(price))), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 Fragment 1 StreamHashAgg { group_key: [$expr1], aggs: [min(max(price)), count] } - ├── result table: 3 + ├── intermediate state table: 3 ├── state tables: [ 2 ] ├── distinct tables: [] └── StreamProject { exprs: [id, max(price), Vnode(id) as $expr1] } └── StreamHashAgg [append_only] { group_key: [id], aggs: [max(price), count] } - ├── result table: 4 + ├── intermediate state table: 4 ├── state tables: [] ├── distinct tables: [] └── StreamProject { exprs: [id, price, _row_id, _row_id] } diff --git a/src/frontend/planner_test/tests/testdata/output/nexmark_temporal_filter.yaml b/src/frontend/planner_test/tests/testdata/output/nexmark_temporal_filter.yaml index 422fd4b798e8c..4b1c26e0a2be1 100644 --- a/src/frontend/planner_test/tests/testdata/output/nexmark_temporal_filter.yaml +++ b/src/frontend/planner_test/tests/testdata/output/nexmark_temporal_filter.yaml @@ -200,7 +200,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [$expr5, (sum(max($expr8)) / count(max($expr8))::Decimal) as $expr10] } └── StreamHashAgg { group_key: [$expr5], aggs: [sum(max($expr8)), count(max($expr8)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([1]) from 1 @@ -208,7 +208,7 @@ Fragment 1 StreamProject { exprs: [$expr2, $expr5, max($expr8)] } └── StreamHashAgg { group_key: [$expr2, $expr5], aggs: [max($expr8), count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [ 1 ] ├── distinct tables: [] └── StreamProject { exprs: [$expr2, $expr5, $expr8, _row_id, _row_id] } @@ -366,7 +366,7 @@ └── StreamHashJoin { type: Inner, predicate: window_start = window_start, output: all } { left table: 0, right table: 2, left degree table: 1, right degree table: 3 } ├── StreamExchange Hash([1]) from 1 └── StreamProject { exprs: [window_start, max(count)] } - └── StreamHashAgg { group_key: [window_start], aggs: [max(count), count] } { result table: 10, state tables: [ 9 ], distinct tables: [] } + └── StreamHashAgg { group_key: [window_start], aggs: [max(count), count] } { intermediate state table: 10, state tables: [ 9 ], distinct tables: [] } └── StreamExchange Hash([1]) from 5 Fragment 1 @@ -374,7 +374,7 @@ └── StreamExchange NoShuffle from 2 Fragment 2 - StreamHashAgg { group_key: [$expr3, window_start], aggs: [count] } { result table: 4, state tables: [], distinct tables: [] } + StreamHashAgg { group_key: [$expr3, window_start], aggs: [count] } { intermediate state table: 4, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1]) from 3 Fragment 3 @@ -633,7 +633,7 @@ Fragment 4 StreamProject { exprs: [$expr7, max($expr5), ($expr7 - '00:00:10':Interval) as $expr8] } - └── StreamHashAgg { group_key: [$expr7], aggs: [max($expr5), count] } { result table: 9, state tables: [ 8 ], distinct tables: [] } + └── StreamHashAgg { group_key: [$expr7], aggs: [max($expr5), count] } { intermediate state table: 9, state tables: [ 8 ], distinct tables: [] } └── StreamExchange Hash([0]) from 5 Fragment 5 @@ -1390,7 +1390,7 @@ └── StreamHashJoin { type: LeftOuter, predicate: $expr2 = $expr5, output: [$expr2, $expr3, max($expr6), _row_id, $expr5] } { left table: 0, right table: 2, left degree table: 1, right degree table: 3 } ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [$expr5, max($expr6)] } - └── StreamHashAgg { group_key: [$expr5], aggs: [max($expr6), count] } { result table: 6, state tables: [ 5 ], distinct tables: [] } + └── StreamHashAgg { group_key: [$expr5], aggs: [max($expr6), count] } { intermediate state table: 6, state tables: [ 5 ], distinct tables: [] } └── StreamExchange Hash([0]) from 3 Fragment 1 @@ -1516,7 +1516,7 @@ ├── materialized table: 4294967294 └── StreamDynamicFilter { predicate: (count($expr5) >= $expr6), output: [$expr2, $expr3, count($expr5)] } { left table: 0, right table: 1 } ├── StreamProject { exprs: [$expr2, $expr3, count($expr5)] } - │ └── StreamHashAgg { group_key: [$expr2, $expr3], aggs: [count($expr5), count] } { result table: 2, state tables: [], distinct tables: [] } + │ └── StreamHashAgg { group_key: [$expr2, $expr3], aggs: [count($expr5), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } │ └── StreamHashJoin { type: Inner, predicate: $expr2 = $expr5, output: [$expr2, $expr3, $expr5, _row_id, _row_id] } │ ├── left table: 3 │ ├── right table: 5 @@ -1557,12 +1557,12 @@ Fragment 6 StreamProject { exprs: [(sum0(sum0(count)) / sum0(count($expr5))) as $expr6] } - └── StreamSimpleAgg { aggs: [sum0(sum0(count)), sum0(count($expr5)), count] } { result table: 11, state tables: [], distinct tables: [] } + └── StreamSimpleAgg { aggs: [sum0(sum0(count)), sum0(count($expr5)), count] } { intermediate state table: 11, state tables: [], distinct tables: [] } └── StreamExchange Single from 7 Fragment 7 StreamStatelessSimpleAgg { aggs: [sum0(count), count($expr5)] } - └── StreamHashAgg { group_key: [$expr5], aggs: [count] } { result table: 12, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [$expr5], aggs: [count] } { intermediate state table: 12, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 8 Fragment 8 @@ -1659,7 +1659,7 @@ ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [$expr5] } └── StreamFilter { predicate: (count >= 20:Int32) } - └── StreamHashAgg { group_key: [$expr5], aggs: [count] } { result table: 5, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [$expr5], aggs: [count] } { intermediate state table: 5, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 3 Fragment 1 @@ -1769,7 +1769,7 @@ ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [$expr5] } └── StreamFilter { predicate: (count < 20:Int32) } - └── StreamHashAgg { group_key: [$expr5], aggs: [count] } { result table: 5, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [$expr5], aggs: [count] } { intermediate state table: 5, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 3 Fragment 1 @@ -1885,7 +1885,7 @@ Fragment 1 StreamGroupTopN { order: [count($expr5) DESC], limit: 1000, offset: 0, group_key: [$expr6] } { state table: 1 } └── StreamProject { exprs: [$expr2, $expr3, count($expr5), Vnode($expr2) as $expr6] } - └── StreamHashAgg { group_key: [$expr2, $expr3], aggs: [count($expr5), count] } { result table: 2, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [$expr2, $expr3], aggs: [count($expr5), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamHashJoin { type: Inner, predicate: $expr2 = $expr5, output: [$expr2, $expr3, $expr5, _row_id, _row_id] } ├── left table: 3 ├── right table: 5 @@ -2011,19 +2011,19 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [min(min(max($expr7)))] } └── StreamSimpleAgg { aggs: [min(min(max($expr7))), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 Fragment 1 StreamHashAgg { group_key: [$expr9], aggs: [min(max($expr7)), count] } - ├── result table: 3 + ├── intermediate state table: 3 ├── state tables: [ 2 ] ├── distinct tables: [] └── StreamProject { exprs: [$expr2, max($expr7), Vnode($expr2) as $expr9] } └── StreamHashAgg { group_key: [$expr2], aggs: [max($expr7), count] } - ├── result table: 5 + ├── intermediate state table: 5 ├── state tables: [ 4 ] ├── distinct tables: [] └── StreamProject { exprs: [$expr2, $expr7, _row_id, _row_id] } diff --git a/src/frontend/planner_test/tests/testdata/output/nexmark_watermark.yaml b/src/frontend/planner_test/tests/testdata/output/nexmark_watermark.yaml index 34e8d6af3b5e3..9b9ff1d660dd5 100644 --- a/src/frontend/planner_test/tests/testdata/output/nexmark_watermark.yaml +++ b/src/frontend/planner_test/tests/testdata/output/nexmark_watermark.yaml @@ -264,14 +264,14 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [$expr4, (sum(max($expr6)) / count(max($expr6))::Decimal) as $expr7] } └── StreamHashAgg { group_key: [$expr4], aggs: [sum(max($expr6)), count(max($expr6)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([1]) from 1 Fragment 1 StreamProject { exprs: [$expr2, $expr4, max($expr6)] } - └── StreamHashAgg [append_only] { group_key: [$expr2, $expr4], aggs: [max($expr6), count] } { result table: 1, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr2, $expr4], aggs: [max($expr6), count] } { intermediate state table: 1, state tables: [], distinct tables: [] } └── StreamHashJoin [append_only] { type: Inner, predicate: $expr2 = $expr5 AND ($expr1 >= $expr1) AND ($expr1 <= $expr3), conditions_to_clean_right_state_table: ($expr1 >= $expr1), output: [$expr2, $expr4, $expr6, _row_id, _row_id] } ├── left table: 2 ├── right table: 4 @@ -433,7 +433,7 @@ └── StreamHashJoin [window] { type: Inner, predicate: window_start = window_start, output_watermarks: [window_start, window_start], output: all } { left table: 0, right table: 2, left degree table: 1, right degree table: 3 } ├── StreamExchange Hash([1]) from 1 └── StreamProject { exprs: [window_start, max(count)], output_watermarks: [window_start] } - └── StreamHashAgg { group_key: [window_start], aggs: [max(count), count], output_watermarks: [window_start] } { result table: 8, state tables: [ 7 ], distinct tables: [] } + └── StreamHashAgg { group_key: [window_start], aggs: [max(count), count], output_watermarks: [window_start] } { intermediate state table: 8, state tables: [ 7 ], distinct tables: [] } └── StreamExchange Hash([1]) from 4 Fragment 1 @@ -441,7 +441,7 @@ └── StreamExchange NoShuffle from 2 Fragment 2 - StreamHashAgg [append_only] { group_key: [$expr2, window_start], aggs: [count], output_watermarks: [window_start] } { result table: 4, state tables: [], distinct tables: [] } + StreamHashAgg [append_only] { group_key: [$expr2, window_start], aggs: [count], output_watermarks: [window_start] } { intermediate state table: 4, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1]) from 3 Fragment 3 @@ -742,7 +742,7 @@ Fragment 3 StreamProject { exprs: [$expr5, max($expr4), ($expr5 - '00:00:10':Interval) as $expr6], output_watermarks: [$expr5, $expr6] } - └── StreamHashAgg [append_only] { group_key: [$expr5], aggs: [max($expr4), count], output_watermarks: [$expr5] } { result table: 6, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr5], aggs: [max($expr4), count], output_watermarks: [$expr5] } { intermediate state table: 6, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 4 Fragment 4 @@ -1301,7 +1301,7 @@ Fragment 0 StreamMaterialize { columns: [day, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [day], pk_columns: [day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamHashAgg [append_only] { group_key: [$expr2], aggs: [count, count filter(($expr3 < 10000:Int32)), count filter(($expr3 >= 10000:Int32) AND ($expr3 < 1000000:Int32)), count filter(($expr3 >= 1000000:Int32)), count(distinct $expr4), count(distinct $expr4) filter(($expr3 < 10000:Int32)), count(distinct $expr4) filter(($expr3 >= 10000:Int32) AND ($expr3 < 1000000:Int32)), count(distinct $expr4) filter(($expr3 >= 1000000:Int32)), count(distinct $expr5), count(distinct $expr5) filter(($expr3 < 10000:Int32)), count(distinct $expr5) filter(($expr3 >= 10000:Int32) AND ($expr3 < 1000000:Int32)), count(distinct $expr5) filter(($expr3 >= 1000000:Int32))] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [ (distinct key: $expr4, table id: 1), (distinct key: $expr5, table id: 2) ] └── StreamExchange Hash([0]) from 1 @@ -1381,7 +1381,7 @@ Fragment 0 StreamMaterialize { columns: [channel, day, minute, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], stream_key: [channel, day], pk_columns: [channel, day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamHashAgg [append_only] { group_key: [$expr2, $expr3], aggs: [max($expr4), count, count filter(($expr5 < 10000:Int32)), count filter(($expr5 >= 10000:Int32) AND ($expr5 < 1000000:Int32)), count filter(($expr5 >= 1000000:Int32)), count(distinct $expr6), count(distinct $expr6) filter(($expr5 < 10000:Int32)), count(distinct $expr6) filter(($expr5 >= 10000:Int32) AND ($expr5 < 1000000:Int32)), count(distinct $expr6) filter(($expr5 >= 1000000:Int32)), count(distinct $expr7), count(distinct $expr7) filter(($expr5 < 10000:Int32)), count(distinct $expr7) filter(($expr5 >= 10000:Int32) AND ($expr5 < 1000000:Int32)), count(distinct $expr7) filter(($expr5 >= 1000000:Int32))] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [ (distinct key: $expr6, table id: 1), (distinct key: $expr7, table id: 2) ] └── StreamExchange Hash([0, 1]) from 1 @@ -1456,7 +1456,7 @@ StreamMaterialize { columns: [auction, day, total_bids, rank1_bids, rank2_bids, rank3_bids, min_price, max_price, avg_price, sum_price], stream_key: [auction, day], pk_columns: [auction, day], pk_conflict: NoCheck } { materialized table: 4294967294 } └── StreamProject { exprs: [$expr2, $expr3, count, count filter(($expr4 < 10000:Int32)), count filter(($expr4 >= 10000:Int32) AND ($expr4 < 1000000:Int32)), count filter(($expr4 >= 1000000:Int32)), min($expr4), max($expr4), (sum($expr4) / count($expr4)::Decimal) as $expr5, sum($expr4)] } └── StreamHashAgg [append_only] { group_key: [$expr2, $expr3], aggs: [count, count filter(($expr4 < 10000:Int32)), count filter(($expr4 >= 10000:Int32) AND ($expr4 < 1000000:Int32)), count filter(($expr4 >= 1000000:Int32)), min($expr4), max($expr4), sum($expr4), count($expr4)] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0, 1]) from 1 @@ -1939,7 +1939,7 @@ └── StreamHashJoin { type: LeftOuter, predicate: $expr2 = $expr4, output: [$expr2, $expr3, max($expr5), _row_id, $expr4] } { left table: 0, right table: 2, left degree table: 1, right degree table: 3 } ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [$expr4, max($expr5)] } - └── StreamHashAgg [append_only] { group_key: [$expr4], aggs: [max($expr5), count] } { result table: 6, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr4], aggs: [max($expr5), count] } { intermediate state table: 6, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 3 Fragment 1 @@ -2077,7 +2077,7 @@ ├── materialized table: 4294967294 └── StreamDynamicFilter { predicate: (count($expr4) >= $expr5), output: [$expr2, $expr3, count($expr4)] } { left table: 0, right table: 1 } ├── StreamProject { exprs: [$expr2, $expr3, count($expr4)] } - │ └── StreamHashAgg [append_only] { group_key: [$expr2, $expr3], aggs: [count($expr4), count] } { result table: 2, state tables: [], distinct tables: [] } + │ └── StreamHashAgg [append_only] { group_key: [$expr2, $expr3], aggs: [count($expr4), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } │ └── StreamHashJoin [append_only] { type: Inner, predicate: $expr2 = $expr4, output: [$expr2, $expr3, $expr4, _row_id, _row_id] } │ ├── left table: 3 │ ├── right table: 5 @@ -2111,12 +2111,12 @@ Fragment 5 StreamProject { exprs: [(sum0(sum0(count)) / sum0(count($expr4))) as $expr5] } - └── StreamSimpleAgg { aggs: [sum0(sum0(count)), sum0(count($expr4)), count] } { result table: 9, state tables: [], distinct tables: [] } + └── StreamSimpleAgg { aggs: [sum0(sum0(count)), sum0(count($expr4)), count] } { intermediate state table: 9, state tables: [], distinct tables: [] } └── StreamExchange Single from 6 Fragment 6 StreamStatelessSimpleAgg { aggs: [sum0(count), count($expr4)] } - └── StreamHashAgg [append_only] { group_key: [$expr4], aggs: [count] } { result table: 10, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr4], aggs: [count] } { intermediate state table: 10, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 7 Fragment 7 @@ -2216,7 +2216,7 @@ ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [$expr4] } └── StreamFilter { predicate: (count >= 20:Int32) } - └── StreamHashAgg [append_only] { group_key: [$expr4], aggs: [count] } { result table: 6, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr4], aggs: [count] } { intermediate state table: 6, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 3 Fragment 1 @@ -2322,7 +2322,7 @@ ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [$expr4] } └── StreamFilter { predicate: (count < 20:Int32) } - └── StreamHashAgg [append_only] { group_key: [$expr4], aggs: [count] } { result table: 6, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr4], aggs: [count] } { intermediate state table: 6, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 3 Fragment 1 @@ -2434,7 +2434,7 @@ Fragment 1 StreamGroupTopN { order: [count($expr4) DESC], limit: 1000, offset: 0, group_key: [$expr5] } { state table: 1 } └── StreamProject { exprs: [$expr2, $expr3, count($expr4), Vnode($expr2) as $expr5] } - └── StreamHashAgg [append_only] { group_key: [$expr2, $expr3], aggs: [count($expr4), count] } { result table: 2, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr2, $expr3], aggs: [count($expr4), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamHashJoin [append_only] { type: Inner, predicate: $expr2 = $expr4, output: [$expr2, $expr3, $expr4, _row_id, _row_id] } ├── left table: 3 ├── right table: 5 @@ -2558,15 +2558,15 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [min(min(max($expr5)))] } └── StreamSimpleAgg { aggs: [min(min(max($expr5))), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 Fragment 1 - StreamHashAgg { group_key: [$expr6], aggs: [min(max($expr5)), count] } { result table: 3, state tables: [ 2 ], distinct tables: [] } + StreamHashAgg { group_key: [$expr6], aggs: [min(max($expr5)), count] } { intermediate state table: 3, state tables: [ 2 ], distinct tables: [] } └── StreamProject { exprs: [$expr2, max($expr5), Vnode($expr2) as $expr6] } - └── StreamHashAgg [append_only] { group_key: [$expr2], aggs: [max($expr5), count] } { result table: 4, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [$expr2], aggs: [max($expr5), count] } { intermediate state table: 4, state tables: [], distinct tables: [] } └── StreamHashJoin [append_only] { type: Inner, predicate: $expr2 = $expr4 AND ($expr1 >= $expr1) AND ($expr1 <= $expr3), conditions_to_clean_right_state_table: ($expr1 >= $expr1), output: [$expr2, $expr5, _row_id, _row_id] } ├── left table: 5 ├── right table: 7 diff --git a/src/frontend/planner_test/tests/testdata/output/share.yaml b/src/frontend/planner_test/tests/testdata/output/share.yaml index 7e7b18e9de8fa..334de5ed0ea03 100644 --- a/src/frontend/planner_test/tests/testdata/output/share.yaml +++ b/src/frontend/planner_test/tests/testdata/output/share.yaml @@ -237,7 +237,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum0(count)] } └── StreamSimpleAgg { aggs: [sum0(count), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -255,7 +255,7 @@ Fragment 2 StreamProject { exprs: [t.a] } └── StreamHashAgg { group_key: [t.a], aggs: [count] } - ├── result table: 5 + ├── intermediate state table: 5 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 3 diff --git a/src/frontend/planner_test/tests/testdata/output/stream_dist_agg.yaml b/src/frontend/planner_test/tests/testdata/output/stream_dist_agg.yaml index 03b1629b931fa..8019780382bad 100644 --- a/src/frontend/planner_test/tests/testdata/output/stream_dist_agg.yaml +++ b/src/frontend/planner_test/tests/testdata/output/stream_dist_agg.yaml @@ -26,7 +26,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(s.v)] } └── StreamSimpleAgg { aggs: [max(s.v), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── Chain { table: s, columns: [s.v, s.o, s.t._row_id], pk: [s.t._row_id], dist: Single } @@ -83,7 +83,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(s.v)] } └── StreamSimpleAgg { aggs: [sum(s.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── Chain { table: s, columns: [s.v, s.o, s.t._row_id], pk: [s.t._row_id], dist: Single } @@ -133,7 +133,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [count(s.v)] } └── StreamSimpleAgg { aggs: [count(s.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── Chain { table: s, columns: [s.v, s.o, s.t._row_id], pk: [s.t._row_id], dist: Single } @@ -185,7 +185,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [string_agg(s.s, ',':Varchar order_by(s.v ASC))] } └── StreamSimpleAgg { aggs: [string_agg(s.s, ',':Varchar order_by(s.v ASC)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamProject { exprs: [s.s, ',':Varchar, s.v, s.t._row_id] } @@ -247,14 +247,14 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(max(t.v))] } └── StreamSimpleAgg { aggs: [max(max(t.v)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 Fragment 1 StreamHashAgg { group_key: [$expr1], aggs: [max(t.v), count] } - ├── result table: 3 + ├── intermediate state table: 3 ├── state tables: [ 2 ] ├── distinct tables: [] └── StreamProject { exprs: [t.v, t._row_id, Vnode(t._row_id) as $expr1] } @@ -326,7 +326,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(max(ao.v))] } └── StreamSimpleAgg [append_only] { aggs: [max(max(ao.v)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -383,7 +383,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(sum(t.v))] } └── StreamSimpleAgg { aggs: [sum(sum(t.v)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -435,7 +435,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(sum(ao.v))] } └── StreamSimpleAgg [append_only] { aggs: [sum(sum(ao.v)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -492,7 +492,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum0(count(t.v))] } └── StreamSimpleAgg { aggs: [sum0(count(t.v)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -544,7 +544,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum0(count(ao.v))] } └── StreamSimpleAgg [append_only] { aggs: [sum0(count(ao.v)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -601,7 +601,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [string_agg(t.s, ',':Varchar order_by(t.o ASC))] } └── StreamSimpleAgg { aggs: [string_agg(t.s, ',':Varchar order_by(t.o ASC)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -660,7 +660,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [string_agg(ao.s, ',':Varchar order_by(ao.o ASC))] } └── StreamSimpleAgg [append_only] { aggs: [string_agg(ao.s, ',':Varchar order_by(ao.o ASC)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -725,14 +725,14 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(max(t.v)), sum0(count(t.v))] } └── StreamSimpleAgg { aggs: [max(max(t.v)), sum0(count(t.v)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 Fragment 1 StreamHashAgg { group_key: [$expr1], aggs: [max(t.v), count(t.v), count] } - ├── result table: 3 + ├── intermediate state table: 3 ├── state tables: [ 2 ] ├── distinct tables: [] └── StreamProject { exprs: [t.v, t._row_id, Vnode(t._row_id) as $expr1] } @@ -804,7 +804,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(max(ao.v)), sum0(count(ao.v))] } └── StreamSimpleAgg [append_only] { aggs: [max(max(ao.v)), sum0(count(ao.v)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -861,7 +861,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [count(t.v), string_agg(t.s, ',':Varchar order_by(t.o ASC))] } └── StreamSimpleAgg { aggs: [count(t.v), string_agg(t.s, ',':Varchar order_by(t.o ASC)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -920,7 +920,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [count(ao.v), string_agg(ao.s, ',':Varchar order_by(ao.o ASC))] } └── StreamSimpleAgg [append_only] { aggs: [count(ao.v), string_agg(ao.s, ',':Varchar order_by(ao.o ASC)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -984,7 +984,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(t.v), string_agg(t.s, ',':Varchar order_by(t.o ASC))] } └── StreamSimpleAgg { aggs: [max(t.v), string_agg(t.s, ',':Varchar order_by(t.o ASC)), count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [ 0, 1 ] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -1050,7 +1050,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(ao.v), string_agg(ao.s, ',':Varchar order_by(ao.o ASC))] } └── StreamSimpleAgg [append_only] { aggs: [max(ao.v), string_agg(ao.s, ',':Varchar order_by(ao.o ASC)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -1114,7 +1114,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(t.v), t.k] } └── StreamHashAgg { group_key: [t.k], aggs: [max(t.v), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1175,7 +1175,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(tk.v), tk.k] } └── StreamHashAgg { group_key: [tk.k], aggs: [max(tk.v), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── Chain { table: tk, columns: [tk.k, tk.v, tk.t._row_id], pk: [tk.t._row_id], dist: UpstreamHashShard(tk.k) } @@ -1235,7 +1235,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(s.v), s.k] } └── StreamHashAgg { group_key: [s.k], aggs: [max(s.v), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1292,7 +1292,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [max(ao.v), ao.k] } └── StreamHashAgg [append_only] { group_key: [ao.k], aggs: [max(ao.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1348,7 +1348,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(t.v), t.k] } └── StreamHashAgg { group_key: [t.k], aggs: [sum(t.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1402,7 +1402,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(tk.v), tk.k] } └── StreamHashAgg { group_key: [tk.k], aggs: [sum(tk.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── Chain { table: tk, columns: [tk.k, tk.v, tk.t._row_id], pk: [tk.t._row_id], dist: UpstreamHashShard(tk.k) } @@ -1455,7 +1455,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(s.v), s.k] } └── StreamHashAgg { group_key: [s.k], aggs: [sum(s.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1505,7 +1505,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(ao.v), ao.k] } └── StreamHashAgg [append_only] { group_key: [ao.k], aggs: [sum(ao.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1561,7 +1561,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [count(t.v), t.k] } └── StreamHashAgg { group_key: [t.k], aggs: [count(t.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1615,7 +1615,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [count(tk.v), tk.k] } └── StreamHashAgg { group_key: [tk.k], aggs: [count(tk.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── Chain { table: tk, columns: [tk.k, tk.v, tk.t._row_id], pk: [tk.t._row_id], dist: UpstreamHashShard(tk.k) } @@ -1668,7 +1668,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [count(s.v), s.k] } └── StreamHashAgg { group_key: [s.k], aggs: [count(s.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1718,7 +1718,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [count(ao.v), ao.k] } └── StreamHashAgg [append_only] { group_key: [ao.k], aggs: [count(ao.v), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1776,7 +1776,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [string_agg(t.s, ',':Varchar order_by(t.o ASC)), t.k] } └── StreamHashAgg { group_key: [t.k], aggs: [string_agg(t.s, ',':Varchar order_by(t.o ASC)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1840,7 +1840,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [string_agg(tk.s, ',':Varchar order_by(tk.o ASC)), tk.k] } └── StreamHashAgg { group_key: [tk.k], aggs: [string_agg(tk.s, ',':Varchar order_by(tk.o ASC)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamProject { exprs: [tk.k, tk.s, ',':Varchar, tk.o, tk.t._row_id] } @@ -1903,7 +1903,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [string_agg(s.s, ',':Varchar order_by(s.o ASC)), s.k] } └── StreamHashAgg { group_key: [s.k], aggs: [string_agg(s.s, ',':Varchar order_by(s.o ASC)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1962,7 +1962,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [string_agg(ao.s, ',':Varchar order_by(ao.o ASC)), ao.k] } └── StreamHashAgg [append_only] { group_key: [ao.k], aggs: [string_agg(ao.s, ',':Varchar order_by(ao.o ASC)), count] } - ├── result table: 1 + ├── intermediate state table: 1 ├── state tables: [ 0 ] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 diff --git a/src/frontend/planner_test/tests/testdata/output/tpch.yaml b/src/frontend/planner_test/tests/testdata/output/tpch.yaml index 798765115e48d..b34d92178f43e 100644 --- a/src/frontend/planner_test/tests/testdata/output/tpch.yaml +++ b/src/frontend/planner_test/tests/testdata/output/tpch.yaml @@ -168,7 +168,7 @@ Fragment 1 StreamGroupTopN { order: [lineitem.l_returnflag ASC, lineitem.l_linestatus ASC], limit: 1, offset: 0, group_key: [$expr6] } { state table: 1 } └── StreamProject { exprs: [lineitem.l_returnflag, lineitem.l_linestatus, sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum($expr1), sum($expr2), (sum(lineitem.l_quantity) / count(lineitem.l_quantity)::Decimal) as $expr3, (sum(lineitem.l_extendedprice) / count(lineitem.l_extendedprice)::Decimal) as $expr4, (sum(lineitem.l_discount) / count(lineitem.l_discount)::Decimal) as $expr5, count, Vnode(lineitem.l_returnflag, lineitem.l_linestatus) as $expr6] } - └── StreamHashAgg { group_key: [lineitem.l_returnflag, lineitem.l_linestatus], aggs: [sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum($expr1), sum($expr2), count(lineitem.l_quantity), count(lineitem.l_extendedprice), sum(lineitem.l_discount), count(lineitem.l_discount), count] } { result table: 2, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [lineitem.l_returnflag, lineitem.l_linestatus], aggs: [sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum($expr1), sum($expr2), count(lineitem.l_quantity), count(lineitem.l_extendedprice), sum(lineitem.l_discount), count(lineitem.l_discount), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1]) from 2 Fragment 2 @@ -420,7 +420,7 @@ Fragment 7 StreamHashJoin { type: Inner, predicate: part.p_partkey IS NOT DISTINCT FROM part.p_partkey AND min(partsupp.ps_supplycost) = partsupp.ps_supplycost, output: [part.p_partkey, part.p_mfgr, partsupp.ps_suppkey, part.p_partkey, min(partsupp.ps_supplycost), partsupp.ps_partkey] } { left table: 17, right table: 19, left degree table: 18, right degree table: 20 } ├── StreamProject { exprs: [part.p_partkey, min(partsupp.ps_supplycost)] } - │ └── StreamHashAgg { group_key: [part.p_partkey], aggs: [min(partsupp.ps_supplycost), count] } { result table: 22, state tables: [ 21 ], distinct tables: [] } + │ └── StreamHashAgg { group_key: [part.p_partkey], aggs: [min(partsupp.ps_supplycost), count] } { intermediate state table: 22, state tables: [ 21 ], distinct tables: [] } │ └── StreamHashJoin { type: LeftOuter, predicate: part.p_partkey IS NOT DISTINCT FROM partsupp.ps_partkey, output: [part.p_partkey, partsupp.ps_supplycost, partsupp.ps_partkey, partsupp.ps_suppkey, supplier.s_suppkey, region.r_regionkey, nation.n_nationkey, supplier.s_nationkey] } { left table: 23, right table: 25, left degree table: 24, right degree table: 26 } │ ├── StreamExchange Hash([0]) from 8 │ └── StreamExchange Hash([0]) from 9 @@ -430,7 +430,7 @@ Fragment 8 StreamProject { exprs: [part.p_partkey] } - └── StreamHashAgg { group_key: [part.p_partkey], aggs: [count] } { result table: 27, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [part.p_partkey], aggs: [count] } { intermediate state table: 27, state tables: [], distinct tables: [] } └── StreamProject { exprs: [part.p_partkey] } └── StreamFilter { predicate: (part.p_size = 4:Int32) AND Like(part.p_type, '%TIN':Varchar) } └── Chain { table: part, columns: [part.p_partkey, part.p_type, part.p_size], pk: [part.p_partkey], dist: UpstreamHashShard(part.p_partkey) } { state table: 28 } @@ -696,7 +696,7 @@ Fragment 1 StreamGroupTopN { order: [sum($expr1) DESC, orders.o_orderdate ASC], limit: 10, offset: 0, group_key: [$expr2] } { state table: 1 } └── StreamProject { exprs: [lineitem.l_orderkey, sum($expr1), orders.o_orderdate, orders.o_shippriority, Vnode(lineitem.l_orderkey, orders.o_orderdate, orders.o_shippriority) as $expr2] } - └── StreamHashAgg { group_key: [lineitem.l_orderkey, orders.o_orderdate, orders.o_shippriority], aggs: [sum($expr1), count] } { result table: 2, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [lineitem.l_orderkey, orders.o_orderdate, orders.o_shippriority], aggs: [sum($expr1), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1, 2]) from 2 Fragment 2 @@ -859,7 +859,7 @@ StreamGroupTopN { order: [orders.o_orderpriority ASC], limit: 1, offset: 0, group_key: [$expr1] } { state table: 1 } └── StreamProject { exprs: [orders.o_orderpriority, count, Vnode(orders.o_orderpriority) as $expr1] } └── StreamHashAgg { group_key: [orders.o_orderpriority], aggs: [count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 2 @@ -1044,7 +1044,7 @@ StreamGroupTopN { order: [sum($expr1) DESC], limit: 1, offset: 0, group_key: [$expr2] } { state table: 1 } └── StreamProject { exprs: [nation.n_name, sum($expr1), Vnode(nation.n_name) as $expr2] } └── StreamHashAgg { group_key: [nation.n_name], aggs: [sum($expr1), count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 2 @@ -1228,7 +1228,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(sum($expr1))] } └── StreamSimpleAgg { aggs: [sum(sum($expr1)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -1391,7 +1391,7 @@ StreamGroupTopN { order: [nation.n_name ASC, nation.n_name ASC, $expr1 ASC], limit: 1, offset: 0, group_key: [$expr3] } { state table: 1 } └── StreamProject { exprs: [nation.n_name, nation.n_name, $expr1, sum($expr2), Vnode(nation.n_name, nation.n_name, $expr1) as $expr3] } └── StreamHashAgg { group_key: [nation.n_name, nation.n_name, $expr1], aggs: [sum($expr2), count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0, 1, 2]) from 2 @@ -1693,7 +1693,7 @@ StreamGroupTopN { order: [$expr1 ASC], limit: 1, offset: 0, group_key: [$expr5] } { state table: 1 } └── StreamProject { exprs: [$expr1, (sum($expr3) / sum($expr2)) as $expr4, Vnode($expr1) as $expr5] } └── StreamHashAgg { group_key: [$expr1], aggs: [sum($expr3), sum($expr2), count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 2 @@ -2011,7 +2011,7 @@ StreamGroupTopN { order: [nation.n_name ASC, $expr1 DESC], limit: 1, offset: 0, group_key: [$expr3] } { state table: 1 } └── StreamProject { exprs: [nation.n_name, $expr1, sum($expr2), Vnode(nation.n_name, $expr1) as $expr3] } └── StreamHashAgg { group_key: [nation.n_name, $expr1], aggs: [sum($expr2), count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0, 1]) from 2 @@ -2258,7 +2258,7 @@ Fragment 1 StreamGroupTopN { order: [sum($expr1) DESC], limit: 20, offset: 0, group_key: [$expr2] } { state table: 1 } └── StreamProject { exprs: [customer.c_custkey, customer.c_name, sum($expr1), customer.c_acctbal, nation.n_name, customer.c_address, customer.c_phone, customer.c_comment, Vnode(customer.c_custkey) as $expr2] } - └── StreamHashAgg { group_key: [customer.c_custkey, customer.c_name, customer.c_acctbal, customer.c_phone, nation.n_name, customer.c_address, customer.c_comment], aggs: [sum($expr1), count] } { result table: 2, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [customer.c_custkey, customer.c_name, customer.c_acctbal, customer.c_phone, nation.n_name, customer.c_address, customer.c_comment], aggs: [sum($expr1), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamProject { exprs: [customer.c_custkey, customer.c_name, customer.c_acctbal, customer.c_phone, nation.n_name, customer.c_address, customer.c_comment, (lineitem.l_extendedprice * (1.00:Decimal - lineitem.l_discount)) as $expr1, nation.n_nationkey, lineitem.l_orderkey, lineitem.l_linenumber, orders.o_orderkey] } └── StreamHashJoin { type: Inner, predicate: customer.c_custkey = orders.o_custkey, output: [customer.c_custkey, customer.c_name, customer.c_address, customer.c_phone, customer.c_acctbal, customer.c_comment, lineitem.l_extendedprice, lineitem.l_discount, nation.n_name, nation.n_nationkey, lineitem.l_orderkey, lineitem.l_linenumber, orders.o_orderkey] } ├── left table: 3 @@ -2501,7 +2501,7 @@ ├── right table: 3 ├── StreamProject { exprs: [partsupp.ps_partkey, sum($expr1)] } │ └── StreamHashAgg { group_key: [partsupp.ps_partkey], aggs: [sum($expr1), count] } - │ ├── result table: 4 + │ ├── intermediate state table: 4 │ ├── state tables: [] │ ├── distinct tables: [] │ └── StreamExchange Hash([0]) from 2 @@ -2549,7 +2549,7 @@ Fragment 8 StreamProject { exprs: [(sum(sum($expr2)) * 0.0001000000:Decimal) as $expr3] } - └── StreamSimpleAgg { aggs: [sum(sum($expr2)), count] } { result table: 16, state tables: [], distinct tables: [] } + └── StreamSimpleAgg { aggs: [sum(sum($expr2)), count] } { intermediate state table: 16, state tables: [], distinct tables: [] } └── StreamExchange Single from 9 Fragment 9 @@ -2690,7 +2690,7 @@ StreamGroupTopN { order: [lineitem.l_shipmode ASC], limit: 1, offset: 0, group_key: [$expr3] } { state table: 1 } └── StreamProject { exprs: [lineitem.l_shipmode, sum($expr1), sum($expr2), Vnode(lineitem.l_shipmode) as $expr3] } └── StreamHashAgg { group_key: [lineitem.l_shipmode], aggs: [sum($expr1), sum($expr2), count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 2 @@ -2821,7 +2821,7 @@ ├── state table: 1 └── StreamProject { exprs: [count(orders.o_orderkey), count, Vnode(count(orders.o_orderkey)) as $expr1] } └── StreamHashAgg { group_key: [count(orders.o_orderkey)], aggs: [count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([1]) from 2 @@ -2829,7 +2829,7 @@ Fragment 2 StreamProject { exprs: [customer.c_custkey, count(orders.o_orderkey)] } └── StreamHashAgg { group_key: [customer.c_custkey], aggs: [count(orders.o_orderkey), count] } - ├── result table: 3 + ├── intermediate state table: 3 ├── state tables: [] ├── distinct tables: [] └── StreamHashJoin { type: LeftOuter, predicate: customer.c_custkey = orders.o_custkey, output: [customer.c_custkey, orders.o_orderkey] } @@ -2977,7 +2977,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [((100.00:Decimal * sum(sum($expr1))) / sum(sum($expr2))) as $expr3] } └── StreamSimpleAgg { aggs: [sum(sum($expr1)), sum(sum($expr2)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -3159,7 +3159,7 @@ Fragment 4 StreamProject { exprs: [lineitem.l_suppkey, sum($expr1)] } - └── StreamHashAgg { group_key: [lineitem.l_suppkey], aggs: [sum($expr1), count] } { result table: 11, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [lineitem.l_suppkey], aggs: [sum($expr1), count] } { intermediate state table: 11, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 5 Fragment 5 @@ -3172,11 +3172,11 @@ Fragment 6 StreamProject { exprs: [max(max(sum($expr1)))] } - └── StreamSimpleAgg { aggs: [max(max(sum($expr1))), count] } { result table: 14, state tables: [ 13 ], distinct tables: [] } + └── StreamSimpleAgg { aggs: [max(max(sum($expr1))), count] } { intermediate state table: 14, state tables: [ 13 ], distinct tables: [] } └── StreamExchange Single from 7 Fragment 7 - StreamHashAgg { group_key: [$expr2], aggs: [max(sum($expr1)), count] } { result table: 16, state tables: [ 15 ], distinct tables: [] } + StreamHashAgg { group_key: [$expr2], aggs: [max(sum($expr1)), count] } { intermediate state table: 16, state tables: [ 15 ], distinct tables: [] } └── StreamProject { exprs: [lineitem.l_suppkey, sum($expr1), Vnode(lineitem.l_suppkey) as $expr2] } └── StreamExchange NoShuffle from 4 @@ -3326,7 +3326,7 @@ ├── state table: 1 └── StreamProject { exprs: [part.p_brand, part.p_type, part.p_size, count(distinct partsupp.ps_suppkey), Vnode(part.p_brand, part.p_type, part.p_size) as $expr1] } └── StreamHashAgg { group_key: [part.p_brand, part.p_type, part.p_size], aggs: [count(distinct partsupp.ps_suppkey), count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [] ├── distinct tables: [ (distinct key: partsupp.ps_suppkey, table id: 3) ] └── StreamExchange Hash([1, 2, 3]) from 2 @@ -3530,7 +3530,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [(sum(sum(lineitem.l_extendedprice)) / 7.0:Decimal) as $expr2] } └── StreamSimpleAgg { aggs: [sum(sum(lineitem.l_extendedprice)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -3542,7 +3542,7 @@ └── StreamHashJoin { type: Inner, predicate: part.p_partkey IS NOT DISTINCT FROM part.p_partkey, output: all } { left table: 1, right table: 3, left degree table: 2, right degree table: 4 } ├── StreamExchange Hash([2]) from 2 └── StreamProject { exprs: [part.p_partkey, (0.2:Decimal * (sum(lineitem.l_quantity) / count(lineitem.l_quantity)::Decimal)) as $expr1] } - └── StreamHashAgg { group_key: [part.p_partkey], aggs: [sum(lineitem.l_quantity), count(lineitem.l_quantity), count] } { result table: 11, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [part.p_partkey], aggs: [sum(lineitem.l_quantity), count(lineitem.l_quantity), count] } { intermediate state table: 11, state tables: [], distinct tables: [] } └── StreamHashJoin { type: LeftOuter, predicate: part.p_partkey IS NOT DISTINCT FROM lineitem.l_partkey, output: [part.p_partkey, lineitem.l_quantity, lineitem.l_orderkey, lineitem.l_linenumber] } ├── left table: 12 ├── right table: 14 @@ -3575,7 +3575,7 @@ Fragment 5 StreamProject { exprs: [part.p_partkey] } - └── StreamHashAgg { group_key: [part.p_partkey], aggs: [count] } { result table: 16, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [part.p_partkey], aggs: [count] } { intermediate state table: 16, state tables: [], distinct tables: [] } └── StreamProject { exprs: [part.p_partkey] } └── StreamFilter { predicate: (part.p_brand = 'Brand#13':Varchar) AND (part.p_container = 'JUMBO PKG':Varchar) } └── Chain { table: part, columns: [part.p_partkey, part.p_brand, part.p_container], pk: [part.p_partkey], dist: UpstreamHashShard(part.p_partkey) } { state table: 17 } @@ -3756,7 +3756,7 @@ Fragment 1 StreamGroupTopN { order: [orders.o_totalprice DESC, orders.o_orderdate ASC], limit: 100, offset: 0, group_key: [$expr1] } { state table: 1 } └── StreamProject { exprs: [customer.c_name, customer.c_custkey, orders.o_orderkey, orders.o_orderdate, orders.o_totalprice, sum(lineitem.l_quantity), Vnode(orders.o_orderkey) as $expr1] } - └── StreamHashAgg { group_key: [customer.c_custkey, customer.c_name, orders.o_orderkey, orders.o_totalprice, orders.o_orderdate], aggs: [sum(lineitem.l_quantity), count] } { result table: 2, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [customer.c_custkey, customer.c_name, orders.o_orderkey, orders.o_totalprice, orders.o_orderdate], aggs: [sum(lineitem.l_quantity), count] } { intermediate state table: 2, state tables: [], distinct tables: [] } └── StreamHashJoin { type: LeftSemi, predicate: orders.o_orderkey = lineitem.l_orderkey, output: all } { left table: 3, right table: 5, left degree table: 4, right degree table: 6 } ├── StreamHashJoin { type: Inner, predicate: orders.o_orderkey = lineitem.l_orderkey, output: [customer.c_custkey, customer.c_name, orders.o_orderkey, orders.o_totalprice, orders.o_orderdate, lineitem.l_quantity, lineitem.l_orderkey, lineitem.l_linenumber] } │ ├── left table: 7 @@ -3768,7 +3768,7 @@ └── StreamProject { exprs: [lineitem.l_orderkey] } └── StreamFilter { predicate: (sum(lineitem.l_quantity) > 1:Decimal) } └── StreamProject { exprs: [lineitem.l_orderkey, sum(lineitem.l_quantity)] } - └── StreamHashAgg { group_key: [lineitem.l_orderkey], aggs: [sum(lineitem.l_quantity), count] } { result table: 18, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [lineitem.l_orderkey], aggs: [sum(lineitem.l_quantity), count] } { intermediate state table: 18, state tables: [], distinct tables: [] } └── StreamExchange Hash([0]) from 6 Fragment 2 @@ -3948,7 +3948,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [sum(sum($expr1))] } └── StreamSimpleAgg { aggs: [sum(sum($expr1)), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Single from 1 @@ -4142,7 +4142,7 @@ ├── right degree table: 19 ├── StreamExchange Hash([0, 1]) from 7 └── StreamProject { exprs: [lineitem.l_partkey, lineitem.l_suppkey, (0.5:Decimal * sum(lineitem.l_quantity)) as $expr2] } - └── StreamHashAgg { group_key: [lineitem.l_partkey, lineitem.l_suppkey], aggs: [sum(lineitem.l_quantity), count] } { result table: 21, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [lineitem.l_partkey, lineitem.l_suppkey], aggs: [sum(lineitem.l_quantity), count] } { intermediate state table: 21, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1]) from 8 Fragment 7 @@ -4378,7 +4378,7 @@ ├── state table: 1 └── StreamProject { exprs: [supplier.s_name, count, Vnode(supplier.s_name) as $expr1] } └── StreamHashAgg { group_key: [supplier.s_name], aggs: [count] } - ├── result table: 2 + ├── intermediate state table: 2 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 2 diff --git a/src/frontend/planner_test/tests/testdata/output/tpch_variant.yaml b/src/frontend/planner_test/tests/testdata/output/tpch_variant.yaml index ebc0edb068a29..a484d4e2efddc 100644 --- a/src/frontend/planner_test/tests/testdata/output/tpch_variant.yaml +++ b/src/frontend/planner_test/tests/testdata/output/tpch_variant.yaml @@ -347,7 +347,7 @@ └── StreamHashJoin { type: Inner, predicate: p_partkey IS NOT DISTINCT FROM p_partkey AND ps_supplycost = min(ps_supplycost), output: [s_acctbal, s_name, n_name, p_partkey, p_mfgr, s_address, s_phone, s_comment, _row_id, _row_id, r_regionkey, _row_id, _row_id, _row_id, ps_suppkey, n_nationkey, ps_supplycost, p_partkey] } { left table: 0, right table: 2, left degree table: 1, right degree table: 3 } ├── StreamExchange Hash([0]) from 1 └── StreamProject { exprs: [p_partkey, min(ps_supplycost)] } - └── StreamHashAgg { group_key: [p_partkey], aggs: [min(ps_supplycost), count] } { result table: 26, state tables: [ 25 ], distinct tables: [] } + └── StreamHashAgg { group_key: [p_partkey], aggs: [min(ps_supplycost), count] } { intermediate state table: 26, state tables: [ 25 ], distinct tables: [] } └── StreamHashJoin { type: LeftOuter, predicate: p_partkey IS NOT DISTINCT FROM ps_partkey, output: [p_partkey, ps_supplycost, _row_id, _row_id, ps_suppkey, _row_id, _row_id, r_regionkey, s_nationkey] } { left table: 27, right table: 29, left degree table: 28, right degree table: 30 } ├── StreamAppendOnlyDedup { dedup_cols: [p_partkey] } { state table: 31 } │ └── StreamExchange Hash([0]) from 15 @@ -664,7 +664,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [n_name, sum($expr1)] } └── StreamHashAgg [append_only] { group_key: [n_name], aggs: [sum($expr1), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -937,7 +937,7 @@ StreamMaterialize { columns: [supp_nation, cust_nation, l_year, revenue], stream_key: [supp_nation, cust_nation, l_year], pk_columns: [supp_nation, cust_nation, l_year], pk_conflict: NoCheck } ├── materialized table: 4294967294 └── StreamProject { exprs: [n_name, n_name, $expr1, sum($expr2)] } - └── StreamHashAgg [append_only] { group_key: [n_name, n_name, $expr1], aggs: [sum($expr2), count] } { result table: 0, state tables: [], distinct tables: [] } + └── StreamHashAgg [append_only] { group_key: [n_name, n_name, $expr1], aggs: [sum($expr2), count] } { intermediate state table: 0, state tables: [], distinct tables: [] } └── StreamExchange Hash([0, 1, 2]) from 1 Fragment 1 @@ -1243,7 +1243,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [$expr1, RoundDigit((sum($expr3) / sum($expr2)), 6:Int32) as $expr4] } └── StreamHashAgg [append_only] { group_key: [$expr1], aggs: [sum($expr3), sum($expr2), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 @@ -1559,7 +1559,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [n_name, $expr1, RoundDigit(sum($expr2), 2:Int32) as $expr3] } └── StreamHashAgg [append_only] { group_key: [n_name, $expr1], aggs: [sum($expr2), count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0, 1]) from 1 @@ -1868,7 +1868,7 @@ └── StreamHashJoin { type: Inner, predicate: ps_partkey IS NOT DISTINCT FROM ps_partkey AND ps_suppkey IS NOT DISTINCT FROM ps_suppkey, output: all } { left table: 10, right table: 12, left degree table: 11, right degree table: 13 } ├── StreamExchange Hash([0, 1]) from 5 └── StreamProject { exprs: [ps_partkey, ps_suppkey, (0.5:Decimal * sum(l_quantity)) as $expr2] } - └── StreamHashAgg { group_key: [ps_partkey, ps_suppkey], aggs: [sum(l_quantity), count] } { result table: 20, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [ps_partkey, ps_suppkey], aggs: [sum(l_quantity), count] } { intermediate state table: 20, state tables: [], distinct tables: [] } └── StreamHashJoin { type: LeftOuter, predicate: ps_partkey IS NOT DISTINCT FROM l_partkey AND ps_suppkey IS NOT DISTINCT FROM l_suppkey, output: [ps_partkey, ps_suppkey, l_quantity, _row_id] } ├── left table: 21 ├── right table: 23 @@ -1897,7 +1897,7 @@ Fragment 9 StreamProject { exprs: [ps_partkey, ps_suppkey] } - └── StreamHashAgg { group_key: [ps_partkey, ps_suppkey], aggs: [count] } { result table: 25, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [ps_partkey, ps_suppkey], aggs: [count] } { intermediate state table: 25, state tables: [], distinct tables: [] } └── StreamExchange NoShuffle from 6 Fragment 10 @@ -2113,7 +2113,10 @@ Fragment 0 StreamMaterialize { columns: [s_name, numwait], stream_key: [s_name], pk_columns: [numwait, s_name], pk_conflict: NoCheck } ├── materialized table: 4294967294 - └── StreamHashAgg { group_key: [s_name], aggs: [count] } { result table: 0, state tables: [], distinct tables: [] } + └── StreamHashAgg { group_key: [s_name], aggs: [count] } + ├── intermediate state table: 0 + ├── state tables: [] + ├── distinct tables: [] └── StreamExchange Hash([0]) from 1 Fragment 1 diff --git a/src/frontend/planner_test/tests/testdata/output/union.yaml b/src/frontend/planner_test/tests/testdata/output/union.yaml index 6c86e00aa2f54..0700225cb98e5 100644 --- a/src/frontend/planner_test/tests/testdata/output/union.yaml +++ b/src/frontend/planner_test/tests/testdata/output/union.yaml @@ -80,7 +80,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [t1.a, t1.b, t1.c] } └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0, 1, 2]) from 1 @@ -170,7 +170,7 @@ ├── materialized table: 4294967294 └── StreamProject { exprs: [t1.a, t1.b, t1.c] } └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } - ├── result table: 0 + ├── intermediate state table: 0 ├── state tables: [] ├── distinct tables: [] └── StreamExchange Hash([0, 1, 2]) from 1 diff --git a/src/frontend/src/optimizer/plan_node/generic/agg.rs b/src/frontend/src/optimizer/plan_node/generic/agg.rs index 48f36e2c1dd36..86aee0c01a7c1 100644 --- a/src/frontend/src/optimizer/plan_node/generic/agg.rs +++ b/src/frontend/src/optimizer/plan_node/generic/agg.rs @@ -20,9 +20,11 @@ use itertools::{Either, Itertools}; use pretty_xmlish::{Pretty, StrAssocArr}; use risingwave_common::catalog::{Field, FieldDisplay, Schema}; use risingwave_common::types::DataType; +use risingwave_common::util::iter_util::ZipEqFast; use risingwave_common::util::sort_util::{ColumnOrder, ColumnOrderDisplay, OrderType}; use risingwave_common::util::value_encoding; use risingwave_expr::agg::{agg_kinds, AggKind}; +use risingwave_expr::sig::agg::AGG_FUNC_SIG_MAP; use risingwave_pb::data::PbDatum; use risingwave_pb::expr::{PbAggCall, PbConstant}; use risingwave_pb::stream_plan::{agg_call_state, AggCallState as AggCallStatePb}; @@ -273,7 +275,7 @@ impl Agg { HashMap, ) { ( - self.infer_result_table(me, vnode_col_idx, window_col_idx), + self.infer_intermediate_state_table(me, vnode_col_idx, window_col_idx), self.infer_stream_agg_state(me, vnode_col_idx, window_col_idx), self.infer_distinct_dedup_tables(me, vnode_col_idx, window_col_idx), ) @@ -470,13 +472,43 @@ impl Agg { .collect() } - pub fn infer_result_table( + /// table schema: + /// group key | state for AGG1 | state for AGG2 | ... + pub fn infer_intermediate_state_table( &self, me: &impl GenericPlanRef, vnode_col_idx: Option, window_col_idx: Option, ) -> TableCatalog { - let out_fields = me.schema().fields(); + let mut out_fields = me.schema().fields().to_vec(); + + // rewrite data types in fields + let in_append_only = self.input.append_only(); + for (agg_call, field) in self + .agg_calls + .iter() + .zip_eq_fast(&mut out_fields[self.group_key.len()..]) + { + let sig = AGG_FUNC_SIG_MAP + .get( + agg_call.agg_kind, + &agg_call + .inputs + .iter() + .map(|input| (&input.data_type).into()) + .collect_vec(), + (&agg_call.return_type).into(), + in_append_only, + ) + .expect("agg not found"); + if !in_append_only && sig.append_only { + // we use materialized input state for non-retractable aggregate function. + // for backward compatibility, the state type is same as the return type. + // its values in the intermediate state table are always null. + } else { + field.data_type = sig.state_type.into(); + } + } let in_dist_key = self.input.distribution().dist_column_indices().to_vec(); let n_group_key_cols = self.group_key.len(); diff --git a/src/frontend/src/optimizer/plan_node/stream.rs b/src/frontend/src/optimizer/plan_node/stream.rs index 1bb6b3226e2c2..3fa4f5aa026e9 100644 --- a/src/frontend/src/optimizer/plan_node/stream.rs +++ b/src/frontend/src/optimizer/plan_node/stream.rs @@ -567,7 +567,7 @@ pub fn to_stream_prost_body( }) } Node::SimpleAgg(me) => { - let result_table = me.core.infer_result_table(base, None, None); + let intermediate_state_table = me.core.infer_intermediate_state_table(base, None, None); let agg_states = me.core.infer_stream_agg_state(base, None, None); let distinct_dedup_tables = me.core.infer_distinct_dedup_tables(base, None, None); @@ -590,8 +590,8 @@ pub fn to_stream_prost_body( .into_iter() .map(|s| s.into_prost(state)) .collect(), - result_table: Some( - result_table + intermediate_state_table: Some( + intermediate_state_table .with_id(state.gen_table_id_wrapped()) .to_internal_table_prost(), ), @@ -624,9 +624,9 @@ pub fn to_stream_prost_body( PbNodeBody::GroupTopN(group_topn_node) } Node::HashAgg(me) => { - let result_table = + let intermediate_state_table = me.core - .infer_result_table(base, me.vnode_col_idx, me.window_col_idx); + .infer_intermediate_state_table(base, me.vnode_col_idx, me.window_col_idx); let agg_states = me.core .infer_stream_agg_state(base, me.vnode_col_idx, me.window_col_idx); @@ -648,8 +648,8 @@ pub fn to_stream_prost_body( .into_iter() .map(|s| s.into_prost(state)) .collect(), - result_table: Some( - result_table + intermediate_state_table: Some( + intermediate_state_table .with_id(state.gen_table_id_wrapped()) .to_internal_table_prost(), ), @@ -698,7 +698,7 @@ pub fn to_stream_prost_body( .map(|&idx| idx as u32) .collect(), agg_call_states: vec![], - result_table: None, + intermediate_state_table: None, is_append_only: me.input.0.append_only, distinct_dedup_tables: Default::default(), }) diff --git a/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs b/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs index d4e48a94e417b..5f0cfc16a4171 100644 --- a/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs +++ b/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs @@ -179,9 +179,9 @@ impl_plan_tree_node_for_unary! { StreamHashAgg } impl StreamNode for StreamHashAgg { fn to_stream_prost_body(&self, state: &mut BuildFragmentGraphState) -> PbNodeBody { use risingwave_pb::stream_plan::*; - let (result_table, agg_states, distinct_dedup_tables) = - self.logical - .infer_tables(&self.base, self.vnode_col_idx, self.window_col_idx); + let (intermediate_state_table, agg_states, distinct_dedup_tables) = self + .logical + .infer_tables(&self.base, self.vnode_col_idx, self.window_col_idx); PbNodeBody::HashAgg(HashAggNode { group_key: self.group_key().to_vec_as_u32(), @@ -196,8 +196,8 @@ impl StreamNode for StreamHashAgg { .into_iter() .map(|s| s.into_prost(state)) .collect(), - result_table: Some( - result_table + intermediate_state_table: Some( + intermediate_state_table .with_id(state.gen_table_id_wrapped()) .to_internal_table_prost(), ), diff --git a/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs b/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs index ce3de79153b31..f0c0bab5fae77 100644 --- a/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs +++ b/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs @@ -89,7 +89,7 @@ impl_plan_tree_node_for_unary! { StreamSimpleAgg } impl StreamNode for StreamSimpleAgg { fn to_stream_prost_body(&self, state: &mut BuildFragmentGraphState) -> PbNodeBody { use risingwave_pb::stream_plan::*; - let (result_table, agg_states, distinct_dedup_tables) = + let (intermediate_state_table, agg_states, distinct_dedup_tables) = self.logical.infer_tables(&self.base, None, None); PbNodeBody::SimpleAgg(SimpleAggNode { @@ -110,8 +110,8 @@ impl StreamNode for StreamSimpleAgg { .into_iter() .map(|s| s.into_prost(state)) .collect(), - result_table: Some( - result_table + intermediate_state_table: Some( + intermediate_state_table .with_id(state.gen_table_id_wrapped()) .to_internal_table_prost(), ), diff --git a/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs b/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs index b69ff2b518cbd..639b6c5782bb5 100644 --- a/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs +++ b/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs @@ -100,7 +100,7 @@ impl StreamNode for StreamStatelessSimpleAgg { .map(|idx| *idx as u32) .collect_vec(), agg_call_states: vec![], - result_table: None, + intermediate_state_table: None, is_append_only: self.input().append_only(), distinct_dedup_tables: Default::default(), }) diff --git a/src/frontend/src/utils/stream_graph_formatter.rs b/src/frontend/src/utils/stream_graph_formatter.rs index 28cab1380dfd0..93acc73bd7957 100644 --- a/src/frontend/src/utils/stream_graph_formatter.rs +++ b/src/frontend/src/utils/stream_graph_formatter.rs @@ -169,8 +169,8 @@ impl StreamGraphFormatter { )), stream_node::NodeBody::SimpleAgg(inner) => { fields.push(( - "result table", - self.pretty_add_table(inner.get_result_table().unwrap()), + "intermediate state table", + self.pretty_add_table(inner.get_intermediate_state_table().unwrap()), )); fields.push(("state tables", self.call_states(&inner.agg_call_states))); fields.push(( @@ -180,8 +180,8 @@ impl StreamGraphFormatter { } stream_node::NodeBody::HashAgg(inner) => { fields.push(( - "result table", - self.pretty_add_table(inner.get_result_table().unwrap()), + "intermediate state table", + self.pretty_add_table(inner.get_intermediate_state_table().unwrap()), )); fields.push(("state tables", self.call_states(&inner.agg_call_states))); fields.push(( diff --git a/src/meta/src/stream/test_fragmenter.rs b/src/meta/src/stream/test_fragmenter.rs index add6811272b04..68cb8125e67d0 100644 --- a/src/meta/src/stream/test_fragmenter.rs +++ b/src/meta/src/stream/test_fragmenter.rs @@ -271,7 +271,7 @@ fn make_stream_fragments() -> Vec { distribution_key: Default::default(), is_append_only: false, agg_call_states: vec![make_agg_call_result_state(), make_agg_call_result_state()], - result_table: Some(make_empty_table(1)), + intermediate_state_table: Some(make_empty_table(1)), ..Default::default() })), input: vec![filter_node], @@ -314,7 +314,7 @@ fn make_stream_fragments() -> Vec { distribution_key: Default::default(), is_append_only: false, agg_call_states: vec![make_agg_call_result_state(), make_agg_call_result_state()], - result_table: Some(make_empty_table(2)), + intermediate_state_table: Some(make_empty_table(2)), ..Default::default() })), fields: vec![], // TODO: fill this later diff --git a/src/stream/src/cache/managed_lru.rs b/src/stream/src/cache/managed_lru.rs index ab9703a557531..d686fbdd0f0ad 100644 --- a/src/stream/src/cache/managed_lru.rs +++ b/src/stream/src/cache/managed_lru.rs @@ -17,7 +17,6 @@ use std::borrow::Borrow; use std::cmp::min; use std::hash::{BuildHasher, Hash}; use std::ops::{Deref, DerefMut}; -use std::ptr::NonNull; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -171,18 +170,6 @@ impl Option> { - let v = self.inner.get_mut(k); - v.map(|inner| { - UnsafeMutGuard::new( - inner, - &mut self.kv_heap_size, - &mut self.last_reported_size_bytes, - &mut self.memory_usage_metrics, - ) - }) - } - pub fn get(&mut self, k: &Q) -> Option<&V> where KeyRef: Borrow, @@ -367,45 +354,3 @@ impl<'a, V: EstimateSize> DerefMut for MutGuard<'a, V> { self.inner } } - -pub struct UnsafeMutGuard { - inner: NonNull, - // The size of the original value - original_val_size: usize, - // The total size of a collection - total_size: NonNull, - last_reported_size_bytes: NonNull, - memory_usage_metrics: NonNull, -} - -impl UnsafeMutGuard { - pub fn new( - inner: &mut V, - total_size: &mut usize, - last_reported_size_bytes: &mut usize, - memory_usage_metrics: &mut IntGauge, - ) -> Self { - let original_val_size = inner.estimated_size(); - Self { - inner: inner.into(), - original_val_size, - total_size: total_size.into(), - last_reported_size_bytes: last_reported_size_bytes.into(), - memory_usage_metrics: memory_usage_metrics.into(), - } - } - - /// # Safety - /// - /// 1. Only 1 `MutGuard` should be held for each value. - /// 2. The returned `MutGuard` should not be moved to other threads. - pub unsafe fn as_mut_guard<'a>(&mut self) -> MutGuard<'a, V> { - MutGuard { - inner: self.inner.as_mut(), - original_val_size: self.original_val_size, - total_size: self.total_size.as_mut(), - last_reported_size_bytes: self.last_reported_size_bytes.as_mut(), - memory_usage_metrics: self.memory_usage_metrics.as_mut(), - } - } -} diff --git a/src/stream/src/common/table/state_table.rs b/src/stream/src/common/table/state_table.rs index dfdd11732c942..be3c2620ddfe9 100644 --- a/src/stream/src/common/table/state_table.rs +++ b/src/stream/src/common/table/state_table.rs @@ -728,9 +728,14 @@ where .unwrap_or_else(|e| self.handle_mem_table_error(e)); } - fn update_inner(&mut self, key_bytes: Bytes, old_value_bytes: Bytes, new_value_bytes: Bytes) { + fn update_inner( + &mut self, + key_bytes: Bytes, + old_value_bytes: Option, + new_value_bytes: Bytes, + ) { self.local_store - .insert(key_bytes, new_value_bytes, Some(old_value_bytes)) + .insert(key_bytes, new_value_bytes, old_value_bytes) .unwrap_or_else(|e| self.handle_mem_table_error(e)); } @@ -776,7 +781,19 @@ where let old_value_bytes = self.serialize_value(old_value); let new_value_bytes = self.serialize_value(new_value); - self.update_inner(new_key_bytes, old_value_bytes, new_value_bytes); + self.update_inner(new_key_bytes, Some(old_value_bytes), new_value_bytes); + } + + /// Update a row without giving old value. + /// + /// `is_consistent_op` should be set to false. + pub fn update_without_old_value(&mut self, new_value: impl Row) { + let new_pk = (&new_value).project(self.pk_indices()); + let new_key_bytes = + serialize_pk_with_vnode(new_pk, &self.pk_serde, self.compute_prefix_vnode(new_pk)); + let new_value_bytes = self.serialize_value(new_value); + + self.update_inner(new_key_bytes, None, new_value_bytes); } /// Write a record into state table. Must have the same schema with the table. diff --git a/src/stream/src/executor/agg_common.rs b/src/stream/src/executor/agg_common.rs index 56d9fec20d027..d6d63b4d65b8a 100644 --- a/src/stream/src/executor/agg_common.rs +++ b/src/stream/src/executor/agg_common.rs @@ -40,7 +40,7 @@ pub struct AggExecutorArgs { pub agg_calls: Vec, pub row_count_index: usize, pub storages: Vec>, - pub result_table: StateTable, + pub intermediate_state_table: StateTable, pub distinct_dedup_tables: HashMap>, pub watermark_epoch: AtomicU64Ref, pub metrics: Arc, diff --git a/src/stream/src/executor/aggregation/agg_group.rs b/src/stream/src/executor/aggregation/agg_group.rs index fb93f600e1fb6..8c7853e2fc7af 100644 --- a/src/stream/src/executor/aggregation/agg_group.rs +++ b/src/stream/src/executor/aggregation/agg_group.rs @@ -159,7 +159,7 @@ pub struct AggGroup { /// Current managed states for all [`AggCall`]s. states: Vec, - /// Previous outputs of managed states. Initializing with `None`. + /// Previous outputs of aggregate functions. Initializing with `None`. prev_outputs: Option, /// Index of row count agg call (`count(*)`) in the call list. @@ -195,17 +195,17 @@ impl AggGroup { agg_calls: &[AggCall], agg_funcs: &[BoxedAggregateFunction], storages: &[AggStateStorage], - result_table: &StateTable, + intermediate_state_table: &StateTable, pk_indices: &PkIndices, row_count_index: usize, extreme_cache_size: usize, input_schema: &Schema, ) -> StreamExecutorResult { - let prev_outputs: Option = result_table + let encoded_states = intermediate_state_table .get_row(group_key.as_ref().map(GroupKey::table_pk)) .await?; - if let Some(prev_outputs) = &prev_outputs { - assert_eq!(prev_outputs.len(), agg_calls.len()); + if let Some(encoded_states) = &encoded_states { + assert_eq!(encoded_states.len(), agg_calls.len()); } let mut states = Vec::with_capacity(agg_calls.len()); @@ -214,7 +214,50 @@ impl AggGroup { agg_call, agg_func, &storages[idx], - prev_outputs.as_ref().map(|outputs| &outputs[idx]), + encoded_states.as_ref().map(|outputs| &outputs[idx]), + pk_indices, + extreme_cache_size, + input_schema, + )?; + states.push(state); + } + + let mut this = Self { + group_key, + states, + prev_outputs: None, // will be initialized later + row_count_index, + _phantom: PhantomData, + }; + + if encoded_states.is_some() { + let (_, outputs) = this.get_outputs(storages, agg_funcs).await?; + this.prev_outputs = Some(outputs); + } + + Ok(this) + } + + /// Create a group from encoded states for EOWC. The previous output is set to `None`. + #[allow(clippy::too_many_arguments)] + pub fn create_eowc( + group_key: Option, + agg_calls: &[AggCall], + agg_funcs: &[BoxedAggregateFunction], + storages: &[AggStateStorage], + encoded_states: &OwnedRow, + pk_indices: &PkIndices, + row_count_index: usize, + extreme_cache_size: usize, + input_schema: &Schema, + ) -> StreamExecutorResult { + let mut states = Vec::with_capacity(agg_calls.len()); + for (idx, (agg_call, agg_func)) in agg_calls.iter().zip_eq_fast(agg_funcs).enumerate() { + let state = AggState::create( + agg_call, + agg_func, + &storages[idx], + Some(&encoded_states[idx]), pk_indices, extreme_cache_size, input_schema, @@ -225,7 +268,7 @@ impl AggGroup { Ok(Self { group_key, states, - prev_outputs, + prev_outputs: None, row_count_index, _phantom: PhantomData, }) @@ -314,6 +357,28 @@ impl AggGroup { } } + /// Encode intermediate states. + pub fn encode_states( + &self, + funcs: &[BoxedAggregateFunction], + ) -> StreamExecutorResult { + let mut encoded_states = Vec::with_capacity(self.states.len()); + for (state, func) in self.states.iter().zip_eq_fast(funcs) { + let encoded = match state { + AggState::Value(s) => func.encode_state(s)?, + // For minput state, we don't need to store it in state table. + AggState::MaterializedInput(_) => None, + }; + encoded_states.push(encoded); + } + let states = self + .group_key() + .map(GroupKey::table_row) + .chain(OwnedRow::new(encoded_states)) + .into_owned_row(); + Ok(states) + } + /// Get the outputs of all managed agg states, without group key prefix. /// Possibly need to read/sync from state table if the state not cached in memory. /// This method is idempotent, i.e. it can be called multiple times and the outputs are diff --git a/src/stream/src/executor/aggregation/agg_state.rs b/src/stream/src/executor/aggregation/agg_state.rs index 1bc4028eeac9d..bb59faf40f150 100644 --- a/src/stream/src/executor/aggregation/agg_state.rs +++ b/src/stream/src/executor/aggregation/agg_state.rs @@ -43,7 +43,7 @@ pub enum AggStateStorage { /// State for single aggregation call. It manages the state cache and interact with the /// underlying state store if necessary. pub enum AggState { - /// State as single scalar value. + /// State as a single scalar value. /// e.g. `count`, `sum`, append-only `min`/`max`. Value(AggregateState), @@ -67,15 +67,15 @@ impl AggState { agg_call: &AggCall, agg_func: &BoxedAggregateFunction, storage: &AggStateStorage, - prev_output: Option<&Datum>, + encoded_state: Option<&Datum>, pk_indices: &PkIndices, extreme_cache_size: usize, input_schema: &Schema, ) -> StreamExecutorResult { Ok(match storage { AggStateStorage::Value => { - let state = match prev_output { - Some(prev) => AggregateState::Datum(prev.clone()), + let state = match encoded_state { + Some(encoded) => agg_func.decode_state(encoded.clone())?, None => agg_func.create_state(), }; Self::Value(state) diff --git a/src/stream/src/executor/aggregation/minput.rs b/src/stream/src/executor/aggregation/minput.rs index 7d65e9f787938..0b50875adf847 100644 --- a/src/stream/src/executor/aggregation/minput.rs +++ b/src/stream/src/executor/aggregation/minput.rs @@ -247,7 +247,7 @@ mod tests { use risingwave_common::types::{DataType, ScalarImpl}; use risingwave_common::util::epoch::EpochPair; use risingwave_common::util::sort_util::OrderType; - use risingwave_expr::agg::{build, AggCall}; + use risingwave_expr::agg::{build_append_only, AggCall}; use risingwave_storage::memory::MemoryStateStore; use risingwave_storage::StateStore; @@ -306,7 +306,7 @@ mod tests { let input_schema = Schema::new(vec![field1, field2, field3, field4]); let agg_call = AggCall::from_pretty("(min:int4 $2:int4)"); // min(c) - let agg = build(&agg_call).unwrap(); + let agg = build_append_only(&agg_call).unwrap(); let group_key = None; let (mut table, mapping) = create_mem_state_table( @@ -399,7 +399,7 @@ mod tests { let input_schema = Schema::new(vec![field1, field2, field3, field4]); let agg_call = AggCall::from_pretty("(max:int4 $2:int4)"); // max(c) - let agg = build(&agg_call).unwrap(); + let agg = build_append_only(&agg_call).unwrap(); let group_key = None; let (mut table, mapping) = create_mem_state_table( @@ -494,8 +494,8 @@ mod tests { let agg_call_1 = AggCall::from_pretty("(min:varchar $0:varchar)"); // min(a) let agg_call_2 = AggCall::from_pretty("(max:int4 $1:int4)"); // max(b) - let agg1 = build(&agg_call_1).unwrap(); - let agg2 = build(&agg_call_2).unwrap(); + let agg1 = build_append_only(&agg_call_1).unwrap(); + let agg2 = build_append_only(&agg_call_2).unwrap(); let group_key = None; let (mut table_1, mapping_1) = create_mem_state_table( @@ -599,7 +599,7 @@ mod tests { let input_schema = Schema::new(vec![field1, field2, field3, field4]); let agg_call = AggCall::from_pretty("(max:int4 $1:int4)"); // max(b) - let agg = build(&agg_call).unwrap(); + let agg = build_append_only(&agg_call).unwrap(); let group_key = Some(GroupKey::new(OwnedRow::new(vec![Some(8.into())]), None)); let (mut table, mapping) = create_mem_state_table( @@ -691,7 +691,7 @@ mod tests { let input_schema = Schema::new(vec![field1, field2]); let agg_call = AggCall::from_pretty("(min:int4 $0:int4)"); // min(a) - let agg = build(&agg_call).unwrap(); + let agg = build_append_only(&agg_call).unwrap(); let group_key = None; let (mut table, mapping) = create_mem_state_table( @@ -793,7 +793,7 @@ mod tests { let input_schema = Schema::new(vec![field1, field2]); let agg_call = AggCall::from_pretty("(min:int4 $0:int4)"); // min(a) - let agg = build(&agg_call).unwrap(); + let agg = build_append_only(&agg_call).unwrap(); let group_key = None; let (mut table, mapping) = create_mem_state_table( @@ -899,7 +899,7 @@ mod tests { let agg_call = AggCall::from_pretty( "(string_agg:varchar $0:varchar $1:varchar orderby $2:asc $0:desc)", ); - let agg = build(&agg_call).unwrap(); + let agg = build_append_only(&agg_call).unwrap(); let group_key = None; let (mut table, mapping) = create_mem_state_table( @@ -978,7 +978,7 @@ mod tests { let input_schema = Schema::new(vec![field1, field2, field3, field4]); let agg_call = AggCall::from_pretty("(array_agg:int4[] $1:int4 orderby $2:asc $0:desc)"); - let agg = build(&agg_call).unwrap(); + let agg = build_append_only(&agg_call).unwrap(); let group_key = None; let (mut table, mapping) = create_mem_state_table( diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs index 610c2857728b3..38938d23dfc1e 100644 --- a/src/stream/src/executor/hash_agg.rs +++ b/src/stream/src/executor/hash_agg.rs @@ -18,16 +18,15 @@ use std::sync::Arc; use futures::{stream, StreamExt}; use futures_async_stream::try_stream; -use iter_chunks::IterChunks; use itertools::Itertools; -use risingwave_common::array::{Op, StreamChunk}; +use risingwave_common::array::StreamChunk; use risingwave_common::buffer::{Bitmap, BitmapBuilder}; use risingwave_common::catalog::Schema; use risingwave_common::hash::{HashKey, PrecomputedBuildHasher}; use risingwave_common::types::ScalarImpl; use risingwave_common::util::epoch::EpochPair; use risingwave_common::util::iter_util::ZipEqFast; -use risingwave_expr::agg::{build, AggCall, BoxedAggregateFunction}; +use risingwave_expr::agg::{build_retractable, AggCall, BoxedAggregateFunction}; use risingwave_storage::StateStore; use super::agg_common::{AggExecutorArgs, HashAggExecutorExtraArgs}; @@ -101,11 +100,11 @@ struct ExecutorInner { /// `None` means the agg call need not to maintain a state table by itself. storages: Vec>, - /// State table for the previous result of all agg calls. - /// The outputs of all managed agg states are collected and stored in this + /// Intermediate state table for value-state agg calls. + /// The state of all value-state aggregates are collected and stored in this /// table when `flush_data` is called. /// Also serves as EOWC sort buffer table. - result_table: StateTable, + intermediate_state_table: StateTable, /// State tables for deduplicating rows on distinct key for distinct agg calls. /// One table per distinct column (may be shared by multiple agg calls). @@ -130,7 +129,7 @@ impl ExecutorInner { fn all_state_tables_mut(&mut self) -> impl Iterator> { iter_table_storage(&mut self.storages) .chain(self.distinct_dedup_tables.values_mut()) - .chain(std::iter::once(&mut self.result_table)) + .chain(std::iter::once(&mut self.intermediate_state_table)) } } @@ -209,7 +208,8 @@ impl HashAggExecutor { let group_key_len = args.extra.group_key_indices.len(); // NOTE: we assume the prefix of table pk is exactly the group key - let group_key_table_pk_projection = &args.result_table.pk_indices()[..group_key_len]; + let group_key_table_pk_projection = + &args.intermediate_state_table.pk_indices()[..group_key_len]; assert!(group_key_table_pk_projection .iter() .sorted() @@ -230,11 +230,11 @@ impl HashAggExecutor { input_schema: input_info.schema, group_key_indices: args.extra.group_key_indices, group_key_table_pk_projection: group_key_table_pk_projection.to_vec().into(), - agg_funcs: args.agg_calls.iter().map(build).try_collect()?, + agg_funcs: args.agg_calls.iter().map(build_retractable).try_collect()?, agg_calls: args.agg_calls, row_count_index: args.row_count_index, storages: args.storages, - result_table: args.result_table, + intermediate_state_table: args.intermediate_state_table, distinct_dedup_tables: args.distinct_dedup_tables, watermark_epoch: args.watermark_epoch, extreme_cache_size: args.extreme_cache_size, @@ -287,7 +287,7 @@ impl HashAggExecutor { stats.lookup_miss_count += 1; Some(async { // Create `AggGroup` for the current group if not exists. This will - // fetch previous agg result from the result table. + // restore agg states from the intermediate state table. let agg_group = AggGroup::create( Some(GroupKey::new( key.deserialize(group_key_types)?, @@ -296,7 +296,7 @@ impl HashAggExecutor { &this.agg_calls, &this.agg_funcs, &this.storages, - &this.result_table, + &this.intermediate_state_table, &this.input_pk_indices, this.row_count_index, this.extreme_cache_size, @@ -405,7 +405,7 @@ impl HashAggExecutor { ) { // Update metrics. let actor_id_str = this.actor_ctx.id.to_string(); - let table_id_str = this.result_table.table_id().to_string(); + let table_id_str = this.intermediate_state_table.table_id().to_string(); this.metrics .agg_lookup_miss_count .with_label_values(&[&table_id_str, &actor_id_str]) @@ -434,70 +434,79 @@ impl HashAggExecutor { let window_watermark = vars.window_watermark.take(); let n_dirty_group = vars.group_change_set.len(); - let futs_of_all_groups = vars - .group_change_set - .drain() - .map(|key| { - // Get agg group of the key. - vars.agg_group_cache - .get_mut_unsafe(&key) - .expect("changed group must have corresponding AggGroup") - }) - .map(|mut agg_group| { - let storages = &this.storages; - let funcs = &this.agg_funcs; - // SAFETY: - // 1. `key`s in `keys_in_batch` are unique by nature, because they're - // from `group_change_set` which is a set. - // - // 2. `MutGuard` should not be sent to other tasks. - let mut agg_group = unsafe { agg_group.as_mut_guard() }; - async move { - // Build aggregate result change. - agg_group.build_change(storages, funcs).await - } - }); - - // TODO(rc): figure out a more reasonable concurrency limit. - const MAX_CONCURRENT_TASKS: usize = 100; - let mut futs_batches = IterChunks::chunks(futs_of_all_groups, MAX_CONCURRENT_TASKS); - while let Some(futs) = futs_batches.next() { - // Compute agg result changes for each group, and emit changes accordingly. - let changes = futures::future::try_join_all(futs).await?; - - // Emit from changes + // flush changed states into intermediate state table + for key in &vars.group_change_set { + let agg_group = vars.agg_group_cache.get_mut(key).unwrap(); + let encoded_states = agg_group.encode_states(&this.agg_funcs)?; if this.emit_on_window_close { - for change in changes.into_iter().flatten() { - // For EOWC, write change to the sort buffer. - vars.buffer.apply_change(change, &mut this.result_table); - } + vars.buffer + .update_without_old_value(encoded_states, &mut this.intermediate_state_table); } else { - for change in changes.into_iter().flatten() { - // For EOU, write change to result table and directly yield the change. - this.result_table.write_record(change.as_ref()); - if let Some(chunk) = vars.chunk_builder.append_record(change) { - yield chunk; - } - } + this.intermediate_state_table + .update_without_old_value(encoded_states); } } - // Emit remaining results from result table. if this.emit_on_window_close { + // remove all groups under watermark and emit their results if let Some(watermark) = window_watermark.as_ref() { #[for_await] for row in vars .buffer - .consume(watermark.clone(), &mut this.result_table) + .consume(watermark.clone(), &mut this.intermediate_state_table) { let row = row?; - if let Some(chunk) = vars.chunk_builder.append_row(Op::Insert, row) { + let group_key = row + .clone() + .into_iter() + .take(this.group_key_indices.len()) + .collect(); + let states = row.into_iter().skip(this.group_key_indices.len()).collect(); + + let mut agg_group = AggGroup::create_eowc( + Some(GroupKey::new( + group_key, + Some(this.group_key_table_pk_projection.clone()), + )), + &this.agg_calls, + &this.agg_funcs, + &this.storages, + &states, + &this.input_pk_indices, + this.row_count_index, + this.extreme_cache_size, + &this.input_schema, + )?; + + let change = agg_group + .build_change(&this.storages, &this.agg_funcs) + .await?; + if let Some(change) = change { + if let Some(chunk) = vars.chunk_builder.append_record(change) { + yield chunk; + } + } + } + } + } else { + // emit on update + // TODO(wrj,rc): we may need to parallelize it and set a reasonable concurrency limit. + for group_key in &vars.group_change_set { + let mut agg_group = vars.agg_group_cache.get_mut(group_key).unwrap(); + let change = agg_group + .build_change(&this.storages, &this.agg_funcs) + .await?; + if let Some(change) = change { + if let Some(chunk) = vars.chunk_builder.append_record(change) { yield chunk; } } } } + // clear the change set + vars.group_change_set.clear(); + // Yield the remaining rows in chunk builder. if let Some(chunk) = vars.chunk_builder.take() { yield chunk; @@ -537,14 +546,14 @@ impl HashAggExecutor { inner: mut this, } = self; - let window_col_idx_in_group_key = this.result_table.pk_indices()[0]; + let window_col_idx_in_group_key = this.intermediate_state_table.pk_indices()[0]; let window_col_idx = this.group_key_indices[window_col_idx_in_group_key]; let agg_group_cache_metrics_info = MetricsInfo::new( this.metrics.clone(), - this.result_table.table_id(), + this.intermediate_state_table.table_id(), this.actor_ctx.id, - "agg result table", + "agg intermediate state table", ); let mut vars = ExecutionVars { @@ -565,7 +574,7 @@ impl HashAggExecutor { buffered_watermarks: vec![None; this.group_key_indices.len()], window_watermark: None, chunk_builder: StreamChunkBuilder::new(this.chunk_size, this.info.schema.data_types()), - buffer: SortBuffer::new(window_col_idx_in_group_key, &this.result_table), + buffer: SortBuffer::new(window_col_idx_in_group_key, &this.intermediate_state_table), }; // TODO(rc): use something like a `ColumnMapping` type @@ -631,7 +640,7 @@ impl HashAggExecutor { // Update the vnode bitmap for state tables of all agg calls if asked. if let Some(vnode_bitmap) = barrier.as_update_vnode_bitmap(this.actor_ctx.id) { - let previous_vnode_bitmap = this.result_table.vnodes().clone(); + let previous_vnode_bitmap = this.intermediate_state_table.vnodes().clone(); this.all_state_tables_mut().for_each(|table| { let _ = table.update_vnode_bitmap(vnode_bitmap.clone()); }); diff --git a/src/stream/src/executor/integration_tests.rs b/src/stream/src/executor/integration_tests.rs index 34dc4c295fe1e..b0382aa12f342 100644 --- a/src/stream/src/executor/integration_tests.rs +++ b/src/stream/src/executor/integration_tests.rs @@ -14,7 +14,6 @@ use std::sync::{Arc, Mutex}; -use anyhow::Context; use futures::StreamExt; use futures_async_stream::try_stream; use multimap::MultiMap; @@ -259,7 +258,7 @@ impl StreamConsumer for SenderConsumer { let msg = item?; let barrier = msg.as_barrier().cloned(); - channel.send(msg).await.context("failed to send message")?; + channel.send(msg).await.expect("failed to send message"); if let Some(barrier) = barrier { yield barrier; diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs index a3109db69b4e3..b50f53977dc84 100644 --- a/src/stream/src/executor/simple_agg.rs +++ b/src/stream/src/executor/simple_agg.rs @@ -17,7 +17,7 @@ use futures_async_stream::try_stream; use risingwave_common::array::StreamChunk; use risingwave_common::catalog::Schema; use risingwave_common::util::iter_util::ZipEqFast; -use risingwave_expr::agg::{build, AggCall, BoxedAggregateFunction}; +use risingwave_expr::agg::{build_retractable, AggCall, BoxedAggregateFunction}; use risingwave_storage::StateStore; use super::agg_common::{AggExecutorArgs, SimpleAggExecutorExtraArgs}; @@ -73,10 +73,10 @@ struct ExecutorInner { /// State storage for each agg calls. storages: Vec>, - /// State table for the previous result of all agg calls. - /// The outputs of all managed agg states are collected and stored in this + /// Intermediate state table for value-state agg calls. + /// The state of all value-state aggregates are collected and stored in this /// table when `flush_data` is called. - result_table: StateTable, + intermediate_state_table: StateTable, /// State tables for deduplicating rows on distinct key for distinct agg calls. /// One table per distinct column (may be shared by multiple agg calls). @@ -95,11 +95,7 @@ impl ExecutorInner { fn all_state_tables_mut(&mut self) -> impl Iterator> { iter_table_storage(&mut self.storages) .chain(self.distinct_dedup_tables.values_mut()) - .chain(std::iter::once(&mut self.result_table)) - } - - fn all_state_tables_except_result_mut(&mut self) -> impl Iterator> { - iter_table_storage(&mut self.storages).chain(self.distinct_dedup_tables.values_mut()) + .chain(std::iter::once(&mut self.intermediate_state_table)) } } @@ -147,11 +143,11 @@ impl SimpleAggExecutor { }, input_pk_indices: input_info.pk_indices, input_schema: input_info.schema, - agg_funcs: args.agg_calls.iter().map(build).try_collect()?, + agg_funcs: args.agg_calls.iter().map(build_retractable).try_collect()?, agg_calls: args.agg_calls, row_count_index: args.row_count_index, storages: args.storages, - result_table: args.result_table, + intermediate_state_table: args.intermediate_state_table, distinct_dedup_tables: args.distinct_dedup_tables, watermark_epoch: args.watermark_epoch, extreme_cache_size: args.extreme_cache_size, @@ -220,30 +216,22 @@ impl SimpleAggExecutor { vars.distinct_dedup .flush(&mut this.distinct_dedup_tables, this.actor_ctx.clone())?; - // Commit all state tables except for result table. + // Flush states into intermediate state table. + let encoded_states = vars.agg_group.encode_states(&this.agg_funcs)?; + this.intermediate_state_table + .update_without_old_value(encoded_states); + + // Commit all state tables. futures::future::try_join_all( - this.all_state_tables_except_result_mut() - .map(|table| table.commit(epoch)), + this.all_state_tables_mut().map(|table| table.commit(epoch)), ) .await?; // Retrieve modified states and put the changes into the builders. - match vars - .agg_group + vars.agg_group .build_change(&this.storages, &this.agg_funcs) .await? - { - Some(change) => { - this.result_table.write_record(change.as_ref()); - this.result_table.commit(epoch).await?; - Some(change.to_stream_chunk(&this.info.schema.data_types())) - } - None => { - // Agg result is not changed. - this.result_table.commit_no_data_expected(epoch); - None - } - } + .map(|change| change.to_stream_chunk(&this.info.schema.data_types())) } else { // No state is changed. // Call commit on state table to increment the epoch. @@ -271,13 +259,13 @@ impl SimpleAggExecutor { }); let mut vars = ExecutionVars { - // Create `AggGroup`. This will fetch previous agg result from the result table. + // This will fetch previous agg states from the intermediate state table. agg_group: AggGroup::create( None, &this.agg_calls, &this.agg_funcs, &this.storages, - &this.result_table, + &this.intermediate_state_table, &this.input_pk_indices, this.row_count_index, this.extreme_cache_size, diff --git a/src/stream/src/executor/sort_buffer.rs b/src/stream/src/executor/sort_buffer.rs index 2a4cd4bf6ad78..709597109af14 100644 --- a/src/stream/src/executor/sort_buffer.rs +++ b/src/stream/src/executor/sort_buffer.rs @@ -118,6 +118,18 @@ impl SortBuffer { self.cache.insert(key, new_row.into_owned_row()); } + /// Update a row in the buffer without giving the old value. + pub fn update_without_old_value( + &mut self, + new_row: impl Row, + buffer_table: &mut StateTable, + ) { + buffer_table.update_without_old_value(&new_row); + let key = row_to_cache_key(self.sort_column_index, &new_row, buffer_table); + self.cache.delete(&key); + self.cache.insert(key, new_row.into_owned_row()); + } + /// Apply a change to the buffer, insert/delete/update. pub fn apply_change(&mut self, change: Record, buffer_table: &mut StateTable) { match change { diff --git a/src/stream/src/executor/stateless_simple_agg.rs b/src/stream/src/executor/stateless_simple_agg.rs index cf2abbebae8ee..150530e4ebb77 100644 --- a/src/stream/src/executor/stateless_simple_agg.rs +++ b/src/stream/src/executor/stateless_simple_agg.rs @@ -18,7 +18,7 @@ use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::catalog::Schema; use risingwave_common::util::iter_util::ZipEqFast; -use risingwave_expr::agg::{build, AggCall, AggregateState, BoxedAggregateFunction}; +use risingwave_expr::agg::{build_retractable, AggCall, AggregateState, BoxedAggregateFunction}; use super::aggregation::{agg_call_filter_res, generate_agg_schema}; use super::error::StreamExecutorError; @@ -136,7 +136,7 @@ impl StatelessSimpleAggExecutor { pk_indices, identity: format!("StatelessSimpleAggExecutor-{}", executor_id), }; - let aggs = agg_calls.iter().map(build).try_collect()?; + let aggs = agg_calls.iter().map(build_retractable).try_collect()?; Ok(StatelessSimpleAggExecutor { ctx, diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs index 95479f448b895..3c06c36e41b47 100644 --- a/src/stream/src/executor/test_utils.rs +++ b/src/stream/src/executor/test_utils.rs @@ -355,8 +355,8 @@ pub mod agg_executor { } } - /// Create result state table for agg executor. - pub async fn create_result_table( + /// Create intermediate state table for agg executor. + pub async fn create_intermediate_state_table( store: S, table_id: TableId, agg_calls: &[AggCall], @@ -386,7 +386,7 @@ pub mod agg_executor { add_column_desc(agg_call.return_type.clone()); }); - StateTable::new_without_distribution( + StateTable::new_without_distribution_inconsistent_op( store, table_id, column_descs, @@ -426,7 +426,7 @@ pub mod agg_executor { ) } - let result_table = create_result_table( + let intermediate_state_table = create_intermediate_state_table( store, TableId::new(agg_calls.len() as u32), &agg_calls, @@ -446,7 +446,7 @@ pub mod agg_executor { agg_calls, row_count_index, storages, - result_table, + intermediate_state_table, distinct_dedup_tables: Default::default(), watermark_epoch: Arc::new(AtomicU64::new(0)), metrics: Arc::new(StreamingMetrics::unused()), @@ -488,7 +488,7 @@ pub mod agg_executor { ) } - let result_table = create_result_table( + let intermediate_state_table = create_intermediate_state_table( store, TableId::new(agg_calls.len() as u32), &agg_calls, @@ -508,7 +508,7 @@ pub mod agg_executor { agg_calls, row_count_index, storages, - result_table, + intermediate_state_table, distinct_dedup_tables: Default::default(), watermark_epoch: Arc::new(AtomicU64::new(0)), metrics: Arc::new(StreamingMetrics::unused()), diff --git a/src/stream/src/from_proto/hash_agg.rs b/src/stream/src/from_proto/hash_agg.rs index c0f09d1c504f6..3d110624c2784 100644 --- a/src/stream/src/from_proto/hash_agg.rs +++ b/src/stream/src/from_proto/hash_agg.rs @@ -84,8 +84,9 @@ impl ExecutorBuilder for HashAggExecutorBuilder { vnodes.clone(), ) .await; - let result_table = StateTable::from_table_catalog( - node.get_result_table().unwrap(), + // disable sanity check so that old value is not required when updating states + let intermediate_state_table = StateTable::from_table_catalog_inconsistent_op( + node.get_intermediate_state_table().unwrap(), store.clone(), vnodes.clone(), ) @@ -106,7 +107,7 @@ impl ExecutorBuilder for HashAggExecutorBuilder { agg_calls, row_count_index: node.get_row_count_index() as usize, storages, - result_table, + intermediate_state_table, distinct_dedup_tables, watermark_epoch: stream.get_watermark_epoch(), metrics: params.executor_stats, diff --git a/src/stream/src/from_proto/simple_agg.rs b/src/stream/src/from_proto/simple_agg.rs index 78ab66df47ae0..403d82dc02e9a 100644 --- a/src/stream/src/from_proto/simple_agg.rs +++ b/src/stream/src/from_proto/simple_agg.rs @@ -46,9 +46,13 @@ impl ExecutorBuilder for SimpleAggExecutorBuilder { let storages = build_agg_state_storages_from_proto(node.get_agg_call_states(), store.clone(), None) .await; - let result_table = - StateTable::from_table_catalog(node.get_result_table().unwrap(), store.clone(), None) - .await; + // disable sanity check so that old value is not required when updating states + let intermediate_state_table = StateTable::from_table_catalog_inconsistent_op( + node.get_intermediate_state_table().unwrap(), + store.clone(), + None, + ) + .await; let distinct_dedup_tables = build_distinct_dedup_table_from_proto(node.get_distinct_dedup_tables(), store, None) .await; @@ -64,7 +68,7 @@ impl ExecutorBuilder for SimpleAggExecutorBuilder { agg_calls, row_count_index: node.get_row_count_index() as usize, storages, - result_table, + intermediate_state_table, distinct_dedup_tables, watermark_epoch: stream.get_watermark_epoch(), metrics: params.executor_stats, From c07633a0c0c28ee41b91f8210d35a4a2c99c8685 Mon Sep 17 00:00:00 2001 From: Noel Kwan <47273164+kwannoel@users.noreply.github.com> Date: Thu, 14 Sep 2023 13:44:53 +0800 Subject: [PATCH 05/14] fix(ci): fix regress test download of `locales` (#12296) --- ci/scripts/regress-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/regress-test.sh b/ci/scripts/regress-test.sh index e32eb2c9ad666..aa1ba7836dc57 100755 --- a/ci/scripts/regress-test.sh +++ b/ci/scripts/regress-test.sh @@ -30,6 +30,7 @@ mv target/debug/risingwave_regress_test-"$profile" target/debug/risingwave_regre chmod +x ./target/debug/risingwave_regress_test echo "--- Postgres regress test" +apt-get -y update apt-get -y install locales locale-gen C export LANGUAGE=C From 1aadfd0d6ebd533037da4ad6c47b7e437e0db494 Mon Sep 17 00:00:00 2001 From: xiangjinwu <17769960+xiangjinwu@users.noreply.github.com> Date: Thu, 14 Sep 2023 13:55:43 +0800 Subject: [PATCH 06/14] refactor(sink): impl RowEncoder for JsonEncoder (#12264) --- src/connector/src/sink/encoder/json.rs | 309 ++++++++++++++++++++++++ src/connector/src/sink/encoder/mod.rs | 80 ++++++ src/connector/src/sink/kafka.rs | 19 +- src/connector/src/sink/kinesis.rs | 21 +- src/connector/src/sink/mod.rs | 1 + src/connector/src/sink/remote.rs | 9 +- src/connector/src/sink/utils.rs | 322 ++----------------------- 7 files changed, 446 insertions(+), 315 deletions(-) create mode 100644 src/connector/src/sink/encoder/json.rs create mode 100644 src/connector/src/sink/encoder/mod.rs diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs new file mode 100644 index 0000000000000..e4fe775f6306c --- /dev/null +++ b/src/connector/src/sink/encoder/json.rs @@ -0,0 +1,309 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use base64::engine::general_purpose; +use base64::Engine as _; +use chrono::{Datelike, Timelike}; +use risingwave_common::array::{ArrayError, ArrayResult}; +use risingwave_common::catalog::{Field, Schema}; +use risingwave_common::row::Row; +use risingwave_common::types::{DataType, DatumRef, ScalarRefImpl, ToText}; +use risingwave_common::util::iter_util::ZipEqDebug; +use serde_json::{json, Map, Value}; + +use super::{Result, RowEncoder, SerTo, TimestampHandlingMode}; +use crate::sink::SinkError; + +pub struct JsonEncoder<'a> { + schema: &'a Schema, + col_indices: Option<&'a [usize]>, + timestamp_handling_mode: TimestampHandlingMode, +} + +impl<'a> JsonEncoder<'a> { + pub fn new( + schema: &'a Schema, + col_indices: Option<&'a [usize]>, + timestamp_handling_mode: TimestampHandlingMode, + ) -> Self { + Self { + schema, + col_indices, + timestamp_handling_mode, + } + } +} + +impl<'a> RowEncoder for JsonEncoder<'a> { + type Output = Map; + + fn schema(&self) -> &Schema { + self.schema + } + + fn col_indices(&self) -> Option<&[usize]> { + self.col_indices + } + + fn encode_cols( + &self, + row: impl Row, + col_indices: impl Iterator, + ) -> Result { + let mut mappings = Map::with_capacity(self.schema.len()); + for idx in col_indices { + let field = &self.schema[idx]; + let key = field.name.clone(); + let value = + datum_to_json_object(field, row.datum_at(idx), self.timestamp_handling_mode) + .map_err(|e| SinkError::JsonParse(e.to_string()))?; + mappings.insert(key, value); + } + Ok(mappings) + } +} + +impl SerTo for Map { + fn ser_to(self) -> Result { + Value::Object(self).ser_to() + } +} + +impl SerTo for Value { + fn ser_to(self) -> Result { + Ok(self.to_string()) + } +} + +fn datum_to_json_object( + field: &Field, + datum: DatumRef<'_>, + timestamp_handling_mode: TimestampHandlingMode, +) -> ArrayResult { + let scalar_ref = match datum { + None => return Ok(Value::Null), + Some(datum) => datum, + }; + + let data_type = field.data_type(); + + tracing::debug!("datum_to_json_object: {:?}, {:?}", data_type, scalar_ref); + + let value = match (data_type, scalar_ref) { + (DataType::Boolean, ScalarRefImpl::Bool(v)) => { + json!(v) + } + (DataType::Int16, ScalarRefImpl::Int16(v)) => { + json!(v) + } + (DataType::Int32, ScalarRefImpl::Int32(v)) => { + json!(v) + } + (DataType::Int64, ScalarRefImpl::Int64(v)) => { + json!(v) + } + (DataType::Float32, ScalarRefImpl::Float32(v)) => { + json!(f32::from(v)) + } + (DataType::Float64, ScalarRefImpl::Float64(v)) => { + json!(f64::from(v)) + } + (DataType::Varchar, ScalarRefImpl::Utf8(v)) => { + json!(v) + } + (DataType::Decimal, ScalarRefImpl::Decimal(v)) => { + json!(v.to_text()) + } + (DataType::Timestamptz, ScalarRefImpl::Timestamptz(v)) => { + // risingwave's timestamp with timezone is stored in UTC and does not maintain the + // timezone info and the time is in microsecond. + let parsed = v.to_datetime_utc().naive_utc(); + let v = parsed.format("%Y-%m-%d %H:%M:%S%.6f").to_string(); + json!(v) + } + (DataType::Time, ScalarRefImpl::Time(v)) => { + // todo: just ignore the nanos part to avoid leap second complex + json!(v.0.num_seconds_from_midnight() as i64 * 1000) + } + (DataType::Date, ScalarRefImpl::Date(v)) => { + json!(v.0.num_days_from_ce()) + } + (DataType::Timestamp, ScalarRefImpl::Timestamp(v)) => match timestamp_handling_mode { + TimestampHandlingMode::Milli => json!(v.0.timestamp_millis()), + TimestampHandlingMode::String => json!(v.0.format("%Y-%m-%d %H:%M:%S%.6f").to_string()), + }, + (DataType::Bytea, ScalarRefImpl::Bytea(v)) => { + json!(general_purpose::STANDARD_NO_PAD.encode(v)) + } + // PYMDTHMS + (DataType::Interval, ScalarRefImpl::Interval(v)) => { + json!(v.as_iso_8601()) + } + (DataType::Jsonb, ScalarRefImpl::Jsonb(jsonb_ref)) => { + json!(jsonb_ref.to_string()) + } + (DataType::List(datatype), ScalarRefImpl::List(list_ref)) => { + let elems = list_ref.iter(); + let mut vec = Vec::with_capacity(elems.len()); + let inner_field = Field::unnamed(Box::::into_inner(datatype)); + for sub_datum_ref in elems { + let value = + datum_to_json_object(&inner_field, sub_datum_ref, timestamp_handling_mode)?; + vec.push(value); + } + json!(vec) + } + (DataType::Struct(st), ScalarRefImpl::Struct(struct_ref)) => { + let mut map = Map::with_capacity(st.len()); + for (sub_datum_ref, sub_field) in struct_ref.iter_fields_ref().zip_eq_debug( + st.iter() + .map(|(name, dt)| Field::with_name(dt.clone(), name)), + ) { + let value = + datum_to_json_object(&sub_field, sub_datum_ref, timestamp_handling_mode)?; + map.insert(sub_field.name.clone(), value); + } + json!(map) + } + (data_type, scalar_ref) => { + return Err(ArrayError::internal( + format!("datum_to_json_object: unsupported data type: field name: {:?}, logical type: {:?}, physical type: {:?}", field.name, data_type, scalar_ref), + )); + } + }; + + Ok(value) +} + +#[cfg(test)] +mod tests { + + use risingwave_common::types::{DataType, Interval, ScalarImpl, Time, Timestamp}; + + use super::*; + #[test] + fn test_to_json_basic_type() { + let mock_field = Field { + data_type: DataType::Boolean, + name: Default::default(), + sub_fields: Default::default(), + type_name: Default::default(), + }; + let boolean_value = datum_to_json_object( + &Field { + data_type: DataType::Boolean, + ..mock_field.clone() + }, + Some(ScalarImpl::Bool(false).as_scalar_ref_impl()), + TimestampHandlingMode::String, + ) + .unwrap(); + assert_eq!(boolean_value, json!(false)); + + let int16_value = datum_to_json_object( + &Field { + data_type: DataType::Int16, + ..mock_field.clone() + }, + Some(ScalarImpl::Int16(16).as_scalar_ref_impl()), + TimestampHandlingMode::String, + ) + .unwrap(); + assert_eq!(int16_value, json!(16)); + + let int64_value = datum_to_json_object( + &Field { + data_type: DataType::Int64, + ..mock_field.clone() + }, + Some(ScalarImpl::Int64(std::i64::MAX).as_scalar_ref_impl()), + TimestampHandlingMode::String, + ) + .unwrap(); + assert_eq!( + serde_json::to_string(&int64_value).unwrap(), + std::i64::MAX.to_string() + ); + + // https://github.com/debezium/debezium/blob/main/debezium-core/src/main/java/io/debezium/time/ZonedTimestamp.java + let tstz_inner = "2018-01-26T18:30:09.453Z".parse().unwrap(); + let tstz_value = datum_to_json_object( + &Field { + data_type: DataType::Timestamptz, + ..mock_field.clone() + }, + Some(ScalarImpl::Timestamptz(tstz_inner).as_scalar_ref_impl()), + TimestampHandlingMode::String, + ) + .unwrap(); + assert_eq!(tstz_value, "2018-01-26 18:30:09.453000"); + + let ts_value = datum_to_json_object( + &Field { + data_type: DataType::Timestamp, + ..mock_field.clone() + }, + Some( + ScalarImpl::Timestamp(Timestamp::from_timestamp_uncheck(1000, 0)) + .as_scalar_ref_impl(), + ), + TimestampHandlingMode::Milli, + ) + .unwrap(); + assert_eq!(ts_value, json!(1000 * 1000)); + + let ts_value = datum_to_json_object( + &Field { + data_type: DataType::Timestamp, + ..mock_field.clone() + }, + Some( + ScalarImpl::Timestamp(Timestamp::from_timestamp_uncheck(1000, 0)) + .as_scalar_ref_impl(), + ), + TimestampHandlingMode::String, + ) + .unwrap(); + assert_eq!(ts_value, json!("1970-01-01 00:16:40.000000".to_string())); + + // Represents the number of microseconds past midnigh, io.debezium.time.Time + let time_value = datum_to_json_object( + &Field { + data_type: DataType::Time, + ..mock_field.clone() + }, + Some( + ScalarImpl::Time(Time::from_num_seconds_from_midnight_uncheck(1000, 0)) + .as_scalar_ref_impl(), + ), + TimestampHandlingMode::String, + ) + .unwrap(); + assert_eq!(time_value, json!(1000 * 1000)); + + let interval_value = datum_to_json_object( + &Field { + data_type: DataType::Interval, + ..mock_field + }, + Some( + ScalarImpl::Interval(Interval::from_month_day_usec(13, 2, 1000000)) + .as_scalar_ref_impl(), + ), + TimestampHandlingMode::String, + ) + .unwrap(); + assert_eq!(interval_value, json!("P1Y1M2DT0H0M1S")); + } +} diff --git a/src/connector/src/sink/encoder/mod.rs b/src/connector/src/sink/encoder/mod.rs new file mode 100644 index 0000000000000..83f185935a44e --- /dev/null +++ b/src/connector/src/sink/encoder/mod.rs @@ -0,0 +1,80 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_common::catalog::Schema; +use risingwave_common::row::Row; + +use crate::sink::Result; + +mod json; + +pub use json::JsonEncoder; + +/// Encode a row of a relation into +/// * an object in json +/// * a message in protobuf +/// * a record in avro +pub trait RowEncoder { + type Output: SerTo>; + + fn encode_cols( + &self, + row: impl Row, + col_indices: impl Iterator, + ) -> Result; + fn schema(&self) -> &Schema; + fn col_indices(&self) -> Option<&[usize]>; + + fn encode(&self, row: impl Row) -> Result { + assert_eq!(row.len(), self.schema().len()); + match self.col_indices() { + Some(col_indices) => self.encode_cols(row, col_indices.iter().copied()), + None => self.encode_cols(row, 0..self.schema().len()), + } + } +} + +/// Do the actual encoding from +/// * an json object +/// * a protobuf message +/// * an avro record +/// into +/// * string (required by kinesis key) +/// * bytes +/// +/// This is like `TryInto` but allows us to `impl> SerTo> for T`. +/// +/// Shall we consider `impl serde::Serialize` in the future? +pub trait SerTo { + fn ser_to(self) -> Result; +} + +impl> SerTo> for T { + fn ser_to(self) -> Result> { + self.ser_to().map(|s: String| s.into_bytes()) + } +} + +impl SerTo for T { + fn ser_to(self) -> Result { + Ok(self) + } +} + +/// Useful for both json and protobuf +#[derive(Clone, Copy)] +pub enum TimestampHandlingMode { + Milli, + String, +} diff --git a/src/connector/src/sink/kafka.rs b/src/connector/src/sink/kafka.rs index aee99da2b74bb..72234bcbbb907 100644 --- a/src/connector/src/sink/kafka.rs +++ b/src/connector/src/sink/kafka.rs @@ -33,6 +33,7 @@ use serde_derive::{Deserialize, Serialize}; use serde_json::Value; use serde_with::{serde_as, DisplayFromStr}; +use super::encoder::{JsonEncoder, TimestampHandlingMode}; use super::{ Sink, SinkError, SinkParam, SINK_TYPE_APPEND_ONLY, SINK_TYPE_DEBEZIUM, SINK_TYPE_OPTION, SINK_TYPE_UPSERT, @@ -546,10 +547,17 @@ impl KafkaSinkWriter { // TODO: Remove the clones here, only to satisfy borrow checker at present let schema = self.schema.clone(); let pk_indices = self.pk_indices.clone(); + let key_encoder = + JsonEncoder::new(&schema, Some(&pk_indices), TimestampHandlingMode::Milli); + let val_encoder = JsonEncoder::new(&schema, None, TimestampHandlingMode::Milli); // Initialize the upsert_stream - let upsert_stream = - gen_upsert_message_stream(&schema, &pk_indices, chunk, UpsertAdapterOpts::default()); + let upsert_stream = gen_upsert_message_stream( + chunk, + UpsertAdapterOpts::default(), + key_encoder, + val_encoder, + ); #[for_await] for msg in upsert_stream { @@ -564,13 +572,16 @@ impl KafkaSinkWriter { // TODO: Remove the clones here, only to satisfy borrow checker at present let schema = self.schema.clone(); let pk_indices = self.pk_indices.clone(); + let key_encoder = + JsonEncoder::new(&schema, Some(&pk_indices), TimestampHandlingMode::Milli); + let val_encoder = JsonEncoder::new(&schema, None, TimestampHandlingMode::Milli); // Initialize the append_only_stream let append_only_stream = gen_append_only_message_stream( - &schema, - &pk_indices, chunk, AppendOnlyAdapterOpts::default(), + key_encoder, + val_encoder, ); #[for_await] diff --git a/src/connector/src/sink/kinesis.rs b/src/connector/src/sink/kinesis.rs index 7ef608b6c6372..b73b48771c2bd 100644 --- a/src/connector/src/sink/kinesis.rs +++ b/src/connector/src/sink/kinesis.rs @@ -31,6 +31,7 @@ use tokio_retry::Retry; use super::SinkParam; use crate::common::KinesisCommon; +use crate::sink::encoder::{JsonEncoder, TimestampHandlingMode}; use crate::sink::utils::{ gen_append_only_message_stream, gen_debezium_message_stream, gen_upsert_message_stream, AppendOnlyAdapterOpts, DebeziumAdapterOpts, UpsertAdapterOpts, @@ -202,11 +203,17 @@ impl KinesisSinkWriter { } async fn upsert(&self, chunk: StreamChunk) -> Result<()> { - let upsert_stream = gen_upsert_message_stream( + let key_encoder = JsonEncoder::new( &self.schema, - &self.pk_indices, + Some(&self.pk_indices), + TimestampHandlingMode::Milli, + ); + let val_encoder = JsonEncoder::new(&self.schema, None, TimestampHandlingMode::Milli); + let upsert_stream = gen_upsert_message_stream( chunk, UpsertAdapterOpts::default(), + key_encoder, + val_encoder, ); crate::impl_load_stream_write_record!(upsert_stream, self.put_record); @@ -214,11 +221,17 @@ impl KinesisSinkWriter { } async fn append_only(&self, chunk: StreamChunk) -> Result<()> { - let append_only_stream = gen_append_only_message_stream( + let key_encoder = JsonEncoder::new( &self.schema, - &self.pk_indices, + Some(&self.pk_indices), + TimestampHandlingMode::Milli, + ); + let val_encoder = JsonEncoder::new(&self.schema, None, TimestampHandlingMode::Milli); + let append_only_stream = gen_append_only_message_stream( chunk, AppendOnlyAdapterOpts::default(), + key_encoder, + val_encoder, ); crate::impl_load_stream_write_record!(append_only_stream, self.put_record); diff --git a/src/connector/src/sink/mod.rs b/src/connector/src/sink/mod.rs index c2a0f3019c942..3e785edc7a9e9 100644 --- a/src/connector/src/sink/mod.rs +++ b/src/connector/src/sink/mod.rs @@ -16,6 +16,7 @@ pub mod boxed; pub mod catalog; pub mod clickhouse; pub mod coordinate; +pub mod encoder; pub mod iceberg; pub mod kafka; pub mod kinesis; diff --git a/src/connector/src/sink/remote.rs b/src/connector/src/sink/remote.rs index 403ee4c7b73e5..046d29bc687b5 100644 --- a/src/connector/src/sink/remote.rs +++ b/src/connector/src/sink/remote.rs @@ -38,9 +38,9 @@ use tokio::sync::mpsc::{Sender, UnboundedReceiver}; use tonic::Status; use tracing::{error, warn}; +use super::encoder::{JsonEncoder, RowEncoder, TimestampHandlingMode}; use crate::sink::coordinate::CoordinatedSinkWriter; use crate::sink::iceberg::REMOTE_ICEBERG_SINK; -use crate::sink::utils::{record_to_json, TimestampHandlingMode}; use crate::sink::SinkError::Remote; use crate::sink::{ DummySinkCommitCoordinator, Result, Sink, SinkCommitCoordinator, SinkError, SinkParam, @@ -351,12 +351,9 @@ where let payload = match self.payload_format { SinkPayloadFormat::Json => { let mut row_ops = vec![]; + let enc = JsonEncoder::new(&self.schema, None, TimestampHandlingMode::String); for (op, row_ref) in chunk.rows() { - let map = record_to_json( - row_ref, - &self.schema.fields, - TimestampHandlingMode::String, - )?; + let map = enc.encode(row_ref)?; let row_op = RowOp { op_type: op.to_protobuf() as i32, line: serde_json::to_string(&map) diff --git a/src/connector/src/sink/utils.rs b/src/connector/src/sink/utils.rs index 1eb2f91e9af22..cb3023b9e4f9b 100644 --- a/src/connector/src/sink/utils.rs +++ b/src/connector/src/sink/utils.rs @@ -12,19 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use base64::engine::general_purpose; -use base64::Engine as _; -use chrono::{Datelike, Timelike}; use futures_async_stream::try_stream; use risingwave_common::array::stream_chunk::Op; -use risingwave_common::array::{ArrayError, ArrayResult, RowRef, StreamChunk}; +use risingwave_common::array::StreamChunk; use risingwave_common::catalog::{Field, Schema}; -use risingwave_common::row::Row; -use risingwave_common::types::{DataType, DatumRef, ScalarRefImpl, ToText}; -use risingwave_common::util::iter_util::{ZipEqDebug, ZipEqFast}; use serde_json::{json, Map, Value}; use tracing::warn; +use super::encoder::{JsonEncoder, RowEncoder, TimestampHandlingMode}; use crate::sink::{Result, SinkError}; const DEBEZIUM_NAME_FIELD_PREFIX: &str = "RisingWave"; @@ -62,6 +57,9 @@ pub async fn gen_debezium_message_stream<'a>( let mut update_cache: Option> = None; + let key_encoder = JsonEncoder::new(schema, Some(pk_indices), TimestampHandlingMode::Milli); + let val_encoder = JsonEncoder::new(schema, None, TimestampHandlingMode::Milli); + for (op, row) in chunk.rows() { let event_key_object: Option = Some(json!({ "schema": json!({ @@ -70,14 +68,14 @@ pub async fn gen_debezium_message_stream<'a>( "optional": false, "name": concat_debezium_name_field(db_name, sink_from_name, "Key"), }), - "payload": pk_to_json(row, &schema.fields, pk_indices)?, + "payload": key_encoder.encode(row)?, })); let event_object: Option = match op { Op::Insert => Some(json!({ "schema": schema_to_json(schema, db_name, sink_from_name), "payload": { "before": null, - "after": record_to_json(row, &schema.fields, TimestampHandlingMode::Milli)?, + "after": val_encoder.encode(row)?, "op": "c", "ts_ms": ts_ms, "source": source_field, @@ -87,7 +85,7 @@ pub async fn gen_debezium_message_stream<'a>( let value_obj = Some(json!({ "schema": schema_to_json(schema, db_name, sink_from_name), "payload": { - "before": record_to_json(row, &schema.fields, TimestampHandlingMode::Milli)?, + "before": val_encoder.encode(row)?, "after": null, "op": "d", "ts_ms": ts_ms, @@ -105,11 +103,7 @@ pub async fn gen_debezium_message_stream<'a>( continue; } Op::UpdateDelete => { - update_cache = Some(record_to_json( - row, - &schema.fields, - TimestampHandlingMode::Milli, - )?); + update_cache = Some(val_encoder.encode(row)?); continue; } Op::UpdateInsert => { @@ -118,7 +112,7 @@ pub async fn gen_debezium_message_stream<'a>( "schema": schema_to_json(schema, db_name, sink_from_name), "payload": { "before": before, - "after": record_to_json(row, &schema.fields, TimestampHandlingMode::Milli)?, + "after": val_encoder.encode(row)?, "op": "u", "ts_ms": ts_ms, "source": source_field, @@ -245,186 +239,38 @@ pub(crate) fn field_to_json(field: &Field) -> Value { }) } -pub(crate) fn pk_to_json( - row: RowRef<'_>, - schema: &[Field], - pk_indices: &[usize], -) -> Result> { - let mut mappings = Map::with_capacity(schema.len()); - for idx in pk_indices { - let field = &schema[*idx]; - let key = field.name.clone(); - let value = datum_to_json_object(field, row.datum_at(*idx), TimestampHandlingMode::Milli) - .map_err(|e| SinkError::JsonParse(e.to_string()))?; - mappings.insert(key, value); - } - Ok(mappings) -} - pub fn chunk_to_json(chunk: StreamChunk, schema: &Schema) -> Result> { + let encoder = JsonEncoder::new(schema, None, TimestampHandlingMode::Milli); let mut records: Vec = Vec::with_capacity(chunk.capacity()); for (_, row) in chunk.rows() { - let record = Value::Object(record_to_json( - row, - &schema.fields, - TimestampHandlingMode::Milli, - )?); + let record = Value::Object(encoder.encode(row)?); records.push(record.to_string()); } Ok(records) } -#[derive(Clone, Copy)] -pub enum TimestampHandlingMode { - Milli, - String, -} - -pub fn record_to_json( - row: RowRef<'_>, - schema: &[Field], - timestamp_handling_mode: TimestampHandlingMode, -) -> Result> { - let mut mappings = Map::with_capacity(schema.len()); - for (field, datum_ref) in schema.iter().zip_eq_fast(row.iter()) { - let key = field.name.clone(); - let value = datum_to_json_object(field, datum_ref, timestamp_handling_mode) - .map_err(|e| SinkError::JsonParse(e.to_string()))?; - mappings.insert(key, value); - } - Ok(mappings) -} - -fn datum_to_json_object( - field: &Field, - datum: DatumRef<'_>, - timestamp_handling_mode: TimestampHandlingMode, -) -> ArrayResult { - let scalar_ref = match datum { - None => return Ok(Value::Null), - Some(datum) => datum, - }; - - let data_type = field.data_type(); - - tracing::debug!("datum_to_json_object: {:?}, {:?}", data_type, scalar_ref); - - let value = match (data_type, scalar_ref) { - (DataType::Boolean, ScalarRefImpl::Bool(v)) => { - json!(v) - } - (DataType::Int16, ScalarRefImpl::Int16(v)) => { - json!(v) - } - (DataType::Int32, ScalarRefImpl::Int32(v)) => { - json!(v) - } - (DataType::Int64, ScalarRefImpl::Int64(v)) => { - json!(v) - } - (DataType::Float32, ScalarRefImpl::Float32(v)) => { - json!(f32::from(v)) - } - (DataType::Float64, ScalarRefImpl::Float64(v)) => { - json!(f64::from(v)) - } - (DataType::Varchar, ScalarRefImpl::Utf8(v)) => { - json!(v) - } - (DataType::Decimal, ScalarRefImpl::Decimal(v)) => { - json!(v.to_text()) - } - (DataType::Timestamptz, ScalarRefImpl::Timestamptz(v)) => { - // risingwave's timestamp with timezone is stored in UTC and does not maintain the - // timezone info and the time is in microsecond. - let parsed = v.to_datetime_utc().naive_utc(); - let v = parsed.format("%Y-%m-%d %H:%M:%S%.6f").to_string(); - json!(v) - } - (DataType::Time, ScalarRefImpl::Time(v)) => { - // todo: just ignore the nanos part to avoid leap second complex - json!(v.0.num_seconds_from_midnight() as i64 * 1000) - } - (DataType::Date, ScalarRefImpl::Date(v)) => { - json!(v.0.num_days_from_ce()) - } - (DataType::Timestamp, ScalarRefImpl::Timestamp(v)) => match timestamp_handling_mode { - TimestampHandlingMode::Milli => json!(v.0.timestamp_millis()), - TimestampHandlingMode::String => json!(v.0.format("%Y-%m-%d %H:%M:%S%.6f").to_string()), - }, - (DataType::Bytea, ScalarRefImpl::Bytea(v)) => { - json!(general_purpose::STANDARD_NO_PAD.encode(v)) - } - // PYMDTHMS - (DataType::Interval, ScalarRefImpl::Interval(v)) => { - json!(v.as_iso_8601()) - } - (DataType::Jsonb, ScalarRefImpl::Jsonb(jsonb_ref)) => { - json!(jsonb_ref.to_string()) - } - (DataType::List(datatype), ScalarRefImpl::List(list_ref)) => { - let elems = list_ref.iter(); - let mut vec = Vec::with_capacity(elems.len()); - let inner_field = Field::unnamed(Box::::into_inner(datatype)); - for sub_datum_ref in elems { - let value = - datum_to_json_object(&inner_field, sub_datum_ref, timestamp_handling_mode)?; - vec.push(value); - } - json!(vec) - } - (DataType::Struct(st), ScalarRefImpl::Struct(struct_ref)) => { - let mut map = Map::with_capacity(st.len()); - for (sub_datum_ref, sub_field) in struct_ref.iter_fields_ref().zip_eq_debug( - st.iter() - .map(|(name, dt)| Field::with_name(dt.clone(), name)), - ) { - let value = - datum_to_json_object(&sub_field, sub_datum_ref, timestamp_handling_mode)?; - map.insert(sub_field.name.clone(), value); - } - json!(map) - } - (data_type, scalar_ref) => { - return Err(ArrayError::internal( - format!("datum_to_json_object: unsupported data type: field name: {:?}, logical type: {:?}, physical type: {:?}", field.name, data_type, scalar_ref), - )); - } - }; - - Ok(value) -} - #[derive(Debug, Clone, Default)] pub struct UpsertAdapterOpts {} #[try_stream(ok = (Option, Option), error = SinkError)] pub async fn gen_upsert_message_stream<'a>( - schema: &'a Schema, - pk_indices: &'a [usize], chunk: StreamChunk, _opts: UpsertAdapterOpts, + key_encoder: JsonEncoder<'a>, + val_encoder: JsonEncoder<'a>, ) { for (op, row) in chunk.rows() { - let event_key_object = Some(Value::Object(pk_to_json(row, &schema.fields, pk_indices)?)); + let event_key_object = Some(Value::Object(key_encoder.encode(row)?)); let event_object = match op { - Op::Insert => Some(Value::Object(record_to_json( - row, - &schema.fields, - TimestampHandlingMode::Milli, - )?)), + Op::Insert => Some(Value::Object(val_encoder.encode(row)?)), Op::Delete => Some(Value::Null), Op::UpdateDelete => { // upsert semantic does not require update delete event continue; } - Op::UpdateInsert => Some(Value::Object(record_to_json( - row, - &schema.fields, - TimestampHandlingMode::Milli, - )?)), + Op::UpdateInsert => Some(Value::Object(val_encoder.encode(row)?)), }; yield (event_key_object, event_object); @@ -436,144 +282,18 @@ pub struct AppendOnlyAdapterOpts {} #[try_stream(ok = (Option, Option), error = SinkError)] pub async fn gen_append_only_message_stream<'a>( - schema: &'a Schema, - pk_indices: &'a [usize], chunk: StreamChunk, _opts: AppendOnlyAdapterOpts, + key_encoder: JsonEncoder<'a>, + val_encoder: JsonEncoder<'a>, ) { for (op, row) in chunk.rows() { if op != Op::Insert { continue; } - let event_key_object = Some(Value::Object(pk_to_json(row, &schema.fields, pk_indices)?)); - let event_object = Some(Value::Object(record_to_json( - row, - &schema.fields, - TimestampHandlingMode::Milli, - )?)); + let event_key_object = Some(Value::Object(key_encoder.encode(row)?)); + let event_object = Some(Value::Object(val_encoder.encode(row)?)); yield (event_key_object, event_object); } } - -#[cfg(test)] -mod tests { - - use risingwave_common::types::{DataType, Interval, ScalarImpl, Time, Timestamp}; - - use super::*; - #[test] - fn test_to_json_basic_type() { - let mock_field = Field { - data_type: DataType::Boolean, - name: Default::default(), - sub_fields: Default::default(), - type_name: Default::default(), - }; - let boolean_value = datum_to_json_object( - &Field { - data_type: DataType::Boolean, - ..mock_field.clone() - }, - Some(ScalarImpl::Bool(false).as_scalar_ref_impl()), - TimestampHandlingMode::String, - ) - .unwrap(); - assert_eq!(boolean_value, json!(false)); - - let int16_value = datum_to_json_object( - &Field { - data_type: DataType::Int16, - ..mock_field.clone() - }, - Some(ScalarImpl::Int16(16).as_scalar_ref_impl()), - TimestampHandlingMode::String, - ) - .unwrap(); - assert_eq!(int16_value, json!(16)); - - let int64_value = datum_to_json_object( - &Field { - data_type: DataType::Int64, - ..mock_field.clone() - }, - Some(ScalarImpl::Int64(std::i64::MAX).as_scalar_ref_impl()), - TimestampHandlingMode::String, - ) - .unwrap(); - assert_eq!( - serde_json::to_string(&int64_value).unwrap(), - std::i64::MAX.to_string() - ); - - // https://github.com/debezium/debezium/blob/main/debezium-core/src/main/java/io/debezium/time/ZonedTimestamp.java - let tstz_inner = "2018-01-26T18:30:09.453Z".parse().unwrap(); - let tstz_value = datum_to_json_object( - &Field { - data_type: DataType::Timestamptz, - ..mock_field.clone() - }, - Some(ScalarImpl::Timestamptz(tstz_inner).as_scalar_ref_impl()), - TimestampHandlingMode::String, - ) - .unwrap(); - assert_eq!(tstz_value, "2018-01-26 18:30:09.453000"); - - let ts_value = datum_to_json_object( - &Field { - data_type: DataType::Timestamp, - ..mock_field.clone() - }, - Some( - ScalarImpl::Timestamp(Timestamp::from_timestamp_uncheck(1000, 0)) - .as_scalar_ref_impl(), - ), - TimestampHandlingMode::Milli, - ) - .unwrap(); - assert_eq!(ts_value, json!(1000 * 1000)); - - let ts_value = datum_to_json_object( - &Field { - data_type: DataType::Timestamp, - ..mock_field.clone() - }, - Some( - ScalarImpl::Timestamp(Timestamp::from_timestamp_uncheck(1000, 0)) - .as_scalar_ref_impl(), - ), - TimestampHandlingMode::String, - ) - .unwrap(); - assert_eq!(ts_value, json!("1970-01-01 00:16:40.000000".to_string())); - - // Represents the number of microseconds past midnigh, io.debezium.time.Time - let time_value = datum_to_json_object( - &Field { - data_type: DataType::Time, - ..mock_field.clone() - }, - Some( - ScalarImpl::Time(Time::from_num_seconds_from_midnight_uncheck(1000, 0)) - .as_scalar_ref_impl(), - ), - TimestampHandlingMode::String, - ) - .unwrap(); - assert_eq!(time_value, json!(1000 * 1000)); - - let interval_value = datum_to_json_object( - &Field { - data_type: DataType::Interval, - ..mock_field - }, - Some( - ScalarImpl::Interval(Interval::from_month_day_usec(13, 2, 1000000)) - .as_scalar_ref_impl(), - ), - TimestampHandlingMode::String, - ) - .unwrap(); - assert_eq!(interval_value, json!("P1Y1M2DT0H0M1S")); - } -} From 7b23eb28cbcbd777c11c6b89ff0069d1460d39e9 Mon Sep 17 00:00:00 2001 From: xxchan Date: Thu, 14 Sep 2023 13:57:51 +0800 Subject: [PATCH 07/14] feat(frontend): implement `WITH ORDINALITY` clause (#12273) --- e2e_test/batch/basic/unnest.slt.part | 35 +++ e2e_test/streaming/project_set.slt | 35 +++ .../tests/testdata/input/with_ordinality.yaml | 64 ++++++ .../testdata/output/with_ordinality.yaml | 210 ++++++++++++++++++ src/frontend/src/binder/relation/join.rs | 4 +- src/frontend/src/binder/relation/mod.rs | 24 +- .../src/binder/relation/table_function.rs | 54 ++++- .../plan_node/logical_table_function.rs | 38 ++-- .../table_function_to_project_set_rule.rs | 46 ++-- src/frontend/src/planner/relation.rs | 30 ++- src/sqlparser/src/ast/query.rs | 15 +- src/sqlparser/src/keywords.rs | 1 + src/sqlparser/src/parser.rs | 9 +- src/sqlparser/tests/testdata/select.yaml | 6 +- src/tests/sqlsmith/src/sql_gen/time_window.rs | 12 +- 15 files changed, 524 insertions(+), 59 deletions(-) create mode 100644 src/frontend/planner_test/tests/testdata/input/with_ordinality.yaml create mode 100644 src/frontend/planner_test/tests/testdata/output/with_ordinality.yaml diff --git a/e2e_test/batch/basic/unnest.slt.part b/e2e_test/batch/basic/unnest.slt.part index efcf9981e65c9..0807637c88e2d 100644 --- a/e2e_test/batch/basic/unnest.slt.part +++ b/e2e_test/batch/basic/unnest.slt.part @@ -83,3 +83,38 @@ select distinct unnest(array[1,1,2,3,1]) as x; 1 2 3 + +query I +select * from unnest(array[0,1,2]) with ordinality; +---- +0 1 +1 2 +2 3 + +query I +select * from unnest(array[0,1,2]) with ordinality, unnest(array[3,4]) with ordinality as unnest_2; +---- +0 1 3 1 +0 1 4 2 +1 2 3 1 +1 2 4 2 +2 3 3 1 +2 3 4 2 + +statement ok +create table t(arr varchar[]); + +statement ok +insert into t values (Array['a','b', 'c']), (Array['d','e']); + +query I rowsort +select * from t cross join unnest(t.arr) WITH ORDINALITY AS x(elts, num); +---- +{a,b,c} a 1 +{a,b,c} b 2 +{a,b,c} c 3 +{d,e} d 1 +{d,e} e 2 + +statement ok +drop table t; diff --git a/e2e_test/streaming/project_set.slt b/e2e_test/streaming/project_set.slt index 959c75ebebefc..f360663067e3e 100644 --- a/e2e_test/streaming/project_set.slt +++ b/e2e_test/streaming/project_set.slt @@ -107,3 +107,38 @@ with cte as (SELECT 1 as v1, unnest(array[1,2,3,4,5]) AS v2) select v1 from cte; 1 1 1 + +statement ok +create table t(arr varchar[]); + +statement ok +create materialized view mv as select * from t cross join unnest(t.arr) WITH ORDINALITY AS x(elts, num); + +statement ok +insert into t values (Array['a','b', 'c']), (Array['d','e']); + +query I rowsort +select * from mv; +---- +{a,b,c} a 1 +{a,b,c} b 2 +{a,b,c} c 3 +{d,e} d 1 +{d,e} e 2 + +statement ok +update t set arr = Array['a', 'c'] where arr = Array['a','b', 'c']; + +query I rowsort +select * from mv; +---- +{a,c} a 1 +{a,c} c 2 +{d,e} d 1 +{d,e} e 2 + +statement ok +drop materialized view mv; + +statement ok +drop table t; diff --git a/src/frontend/planner_test/tests/testdata/input/with_ordinality.yaml b/src/frontend/planner_test/tests/testdata/input/with_ordinality.yaml new file mode 100644 index 0000000000000..08dfafa2c7310 --- /dev/null +++ b/src/frontend/planner_test/tests/testdata/input/with_ordinality.yaml @@ -0,0 +1,64 @@ +# constant FROM +- sql: | + select * from unnest(array[1,2,3]) WITH ORDINALITY; + expected_outputs: + - batch_plan + - stream_plan +# lateral join +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY; + expected_outputs: + - batch_plan + - stream_plan +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY as foo; + expected_outputs: + - batch_plan + - stream_plan +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY as foo(a); + expected_outputs: + - batch_plan + - stream_plan +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY as foo(a,ord); + expected_outputs: + - batch_plan + - stream_plan +- name: use alias columns explicitlity + sql: | + create table t(x int , arr int[]); + select x, arr, a, ord from t cross join unnest(arr) WITH ORDINALITY as foo(a,ord); + expected_outputs: + - batch_plan + - stream_plan +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY as foo(a,ord,bar); + expected_outputs: + - binder_error +# multiple with ordinality +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY, unnest(arr) WITH ORDINALITY AS unnest_2(arr_2,ordinality_2); + expected_outputs: + - batch_plan + - stream_plan +# constant FROM (scalar function) +- sql: | + select * from abs(1) WITH ORDINALITY; + expected_outputs: + - batch_plan + - stream_plan +# lateral join (scalar function) +# FIXME: currently this panics due to CorrelatedInputRef in Values https://github.com/risingwavelabs/risingwave/issues/12231 +- sql: | + create table t(x int , arr int[]); + select * from t, abs(x) WITH ORDINALITY; + expected_outputs: + - batch_plan + - stream_error diff --git a/src/frontend/planner_test/tests/testdata/output/with_ordinality.yaml b/src/frontend/planner_test/tests/testdata/output/with_ordinality.yaml new file mode 100644 index 0000000000000..208eec58ecdba --- /dev/null +++ b/src/frontend/planner_test/tests/testdata/output/with_ordinality.yaml @@ -0,0 +1,210 @@ +# This file is automatically generated. See `src/frontend/planner_test/README.md` for more information. +- sql: | + select * from unnest(array[1,2,3]) WITH ORDINALITY; + batch_plan: |- + BatchProject { exprs: [Unnest(ARRAY[1, 2, 3]:List(Int32)), (projected_row_id + 1:Int64) as $expr1] } + └─BatchProjectSet { select_list: [Unnest(ARRAY[1, 2, 3]:List(Int32))] } + └─BatchValues { rows: [[]] } + stream_plan: |- + StreamMaterialize { columns: [unnest, ordinality, _row_id(hidden), projected_row_id(hidden)], stream_key: [_row_id, projected_row_id], pk_columns: [_row_id, projected_row_id], pk_conflict: NoCheck } + └─StreamProject { exprs: [Unnest(ARRAY[1, 2, 3]:List(Int32)), (projected_row_id + 1:Int64) as $expr1, _row_id, projected_row_id] } + └─StreamProjectSet { select_list: [Unnest(ARRAY[1, 2, 3]:List(Int32)), $0] } + └─StreamValues { rows: [[0:Int64]] } +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY; + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1] } + └─BatchHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: all } + ├─BatchExchange { order: [], dist: HashShard(t.arr) } + │ └─BatchScan { table: t, columns: [t.x, t.arr], distribution: SomeShard } + └─BatchProjectSet { select_list: [$0, Unnest($0)] } + └─BatchHashAgg { group_key: [t.arr], aggs: [] } + └─BatchExchange { order: [], dist: HashShard(t.arr) } + └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard } + stream_plan: |- + StreamMaterialize { columns: [x, arr, unnest, ordinality, t._row_id(hidden), t.arr(hidden), projected_row_id(hidden)], stream_key: [t._row_id, t.arr, projected_row_id, arr], pk_columns: [t._row_id, t.arr, projected_row_id, arr], pk_conflict: NoCheck } + └─StreamProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, t._row_id, t.arr, projected_row_id] } + └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), t._row_id] } + ├─StreamExchange { dist: HashShard(t.arr) } + │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } + └─StreamProjectSet { select_list: [$0, Unnest($0)] } + └─StreamProject { exprs: [t.arr] } + └─StreamHashAgg { group_key: [t.arr], aggs: [count] } + └─StreamExchange { dist: HashShard(t.arr) } + └─StreamTableScan { table: t, columns: [t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY as foo; + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1] } + └─BatchHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: all } + ├─BatchExchange { order: [], dist: HashShard(t.arr) } + │ └─BatchScan { table: t, columns: [t.x, t.arr], distribution: SomeShard } + └─BatchProjectSet { select_list: [$0, Unnest($0)] } + └─BatchHashAgg { group_key: [t.arr], aggs: [] } + └─BatchExchange { order: [], dist: HashShard(t.arr) } + └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard } + stream_plan: |- + StreamMaterialize { columns: [x, arr, foo, ordinality, t._row_id(hidden), t.arr(hidden), projected_row_id(hidden)], stream_key: [t._row_id, t.arr, projected_row_id, arr], pk_columns: [t._row_id, t.arr, projected_row_id, arr], pk_conflict: NoCheck } + └─StreamProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, t._row_id, t.arr, projected_row_id] } + └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), t._row_id] } + ├─StreamExchange { dist: HashShard(t.arr) } + │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } + └─StreamProjectSet { select_list: [$0, Unnest($0)] } + └─StreamProject { exprs: [t.arr] } + └─StreamHashAgg { group_key: [t.arr], aggs: [count] } + └─StreamExchange { dist: HashShard(t.arr) } + └─StreamTableScan { table: t, columns: [t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY as foo(a); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1] } + └─BatchHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: all } + ├─BatchExchange { order: [], dist: HashShard(t.arr) } + │ └─BatchScan { table: t, columns: [t.x, t.arr], distribution: SomeShard } + └─BatchProjectSet { select_list: [$0, Unnest($0)] } + └─BatchHashAgg { group_key: [t.arr], aggs: [] } + └─BatchExchange { order: [], dist: HashShard(t.arr) } + └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard } + stream_plan: |- + StreamMaterialize { columns: [x, arr, a, ordinality, t._row_id(hidden), t.arr(hidden), projected_row_id(hidden)], stream_key: [t._row_id, t.arr, projected_row_id, arr], pk_columns: [t._row_id, t.arr, projected_row_id, arr], pk_conflict: NoCheck } + └─StreamProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, t._row_id, t.arr, projected_row_id] } + └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), t._row_id] } + ├─StreamExchange { dist: HashShard(t.arr) } + │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } + └─StreamProjectSet { select_list: [$0, Unnest($0)] } + └─StreamProject { exprs: [t.arr] } + └─StreamHashAgg { group_key: [t.arr], aggs: [count] } + └─StreamExchange { dist: HashShard(t.arr) } + └─StreamTableScan { table: t, columns: [t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY as foo(a,ord); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1] } + └─BatchHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: all } + ├─BatchExchange { order: [], dist: HashShard(t.arr) } + │ └─BatchScan { table: t, columns: [t.x, t.arr], distribution: SomeShard } + └─BatchProjectSet { select_list: [$0, Unnest($0)] } + └─BatchHashAgg { group_key: [t.arr], aggs: [] } + └─BatchExchange { order: [], dist: HashShard(t.arr) } + └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard } + stream_plan: |- + StreamMaterialize { columns: [x, arr, a, ord, t._row_id(hidden), t.arr(hidden), projected_row_id(hidden)], stream_key: [t._row_id, t.arr, projected_row_id, arr], pk_columns: [t._row_id, t.arr, projected_row_id, arr], pk_conflict: NoCheck } + └─StreamProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, t._row_id, t.arr, projected_row_id] } + └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), t._row_id] } + ├─StreamExchange { dist: HashShard(t.arr) } + │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } + └─StreamProjectSet { select_list: [$0, Unnest($0)] } + └─StreamProject { exprs: [t.arr] } + └─StreamHashAgg { group_key: [t.arr], aggs: [count] } + └─StreamExchange { dist: HashShard(t.arr) } + └─StreamTableScan { table: t, columns: [t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } +- name: use alias columns explicitlity + sql: | + create table t(x int , arr int[]); + select x, arr, a, ord from t cross join unnest(arr) WITH ORDINALITY as foo(a,ord); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1] } + └─BatchHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: all } + ├─BatchExchange { order: [], dist: HashShard(t.arr) } + │ └─BatchScan { table: t, columns: [t.x, t.arr], distribution: SomeShard } + └─BatchProjectSet { select_list: [$0, Unnest($0)] } + └─BatchHashAgg { group_key: [t.arr], aggs: [] } + └─BatchExchange { order: [], dist: HashShard(t.arr) } + └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard } + stream_plan: |- + StreamMaterialize { columns: [x, arr, a, ord, t._row_id(hidden), t.arr(hidden), projected_row_id(hidden)], stream_key: [t._row_id, t.arr, projected_row_id, arr], pk_columns: [t._row_id, t.arr, projected_row_id, arr], pk_conflict: NoCheck } + └─StreamProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, t._row_id, t.arr, projected_row_id] } + └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), t._row_id] } + ├─StreamExchange { dist: HashShard(t.arr) } + │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } + └─StreamProjectSet { select_list: [$0, Unnest($0)] } + └─StreamProject { exprs: [t.arr] } + └─StreamHashAgg { group_key: [t.arr], aggs: [count] } + └─StreamExchange { dist: HashShard(t.arr) } + └─StreamTableScan { table: t, columns: [t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY as foo(a,ord,bar); + binder_error: 'Bind error: table "foo" has 2 columns available but 3 column aliases specified' +- sql: | + create table t(x int , arr int[]); + select * from t cross join unnest(arr) WITH ORDINALITY, unnest(arr) WITH ORDINALITY AS unnest_2(arr_2,ordinality_2); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, Unnest($0), (projected_row_id + 1:Int64) as $expr2] } + └─BatchHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: all } + ├─BatchHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: all } + │ ├─BatchExchange { order: [], dist: HashShard(t.arr) } + │ │ └─BatchScan { table: t, columns: [t.x, t.arr], distribution: SomeShard } + │ └─BatchProjectSet { select_list: [$0, Unnest($0)] } + │ └─BatchHashAgg { group_key: [t.arr], aggs: [] } + │ └─BatchExchange { order: [], dist: HashShard(t.arr) } + │ └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard } + └─BatchProjectSet { select_list: [$0, Unnest($0)] } + └─BatchHashAgg { group_key: [t.arr], aggs: [] } + └─BatchHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.arr] } + ├─BatchExchange { order: [], dist: HashShard(t.arr) } + │ └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard } + └─BatchProjectSet { select_list: [$0, Unnest($0)] } + └─BatchHashAgg { group_key: [t.arr], aggs: [] } + └─BatchExchange { order: [], dist: HashShard(t.arr) } + └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard } + stream_plan: |- + StreamMaterialize { columns: [x, arr, unnest, ordinality, arr_2, ordinality_2, t._row_id(hidden), t.arr(hidden), projected_row_id(hidden), t.arr#1(hidden), projected_row_id#1(hidden)], stream_key: [t._row_id, t.arr, projected_row_id, arr, t.arr#1, projected_row_id#1], pk_columns: [t._row_id, t.arr, projected_row_id, arr, t.arr#1, projected_row_id#1], pk_conflict: NoCheck } + └─StreamProject { exprs: [t.x, t.arr, Unnest($0), $expr1, Unnest($0), (projected_row_id + 1:Int64) as $expr2, t._row_id, t.arr, projected_row_id, t.arr, projected_row_id] } + └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, Unnest($0), $expr1, projected_row_id, t.arr, Unnest($0), t._row_id, t.arr, projected_row_id] } + ├─StreamShare { id: 8 } + │ └─StreamProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, t._row_id, t.arr, projected_row_id] } + │ └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), t._row_id] } + │ ├─StreamExchange { dist: HashShard(t.arr) } + │ │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } + │ └─StreamProjectSet { select_list: [$0, Unnest($0)] } + │ └─StreamProject { exprs: [t.arr] } + │ └─StreamHashAgg { group_key: [t.arr], aggs: [count] } + │ └─StreamExchange { dist: HashShard(t.arr) } + │ └─StreamTableScan { table: t, columns: [t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } + └─StreamProjectSet { select_list: [$0, Unnest($0)] } + └─StreamProject { exprs: [t.arr] } + └─StreamHashAgg { group_key: [t.arr], aggs: [count] } + └─StreamShare { id: 8 } + └─StreamProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, t._row_id, t.arr, projected_row_id] } + └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), t._row_id] } + ├─StreamExchange { dist: HashShard(t.arr) } + │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } + └─StreamProjectSet { select_list: [$0, Unnest($0)] } + └─StreamProject { exprs: [t.arr] } + └─StreamHashAgg { group_key: [t.arr], aggs: [count] } + └─StreamExchange { dist: HashShard(t.arr) } + └─StreamTableScan { table: t, columns: [t.arr, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } +- sql: | + select * from abs(1) WITH ORDINALITY; + batch_plan: 'BatchValues { rows: [[1:Int32, 1:Int64]] }' + stream_plan: |- + StreamMaterialize { columns: [abs, ordinality, _row_id(hidden)], stream_key: [_row_id], pk_columns: [_row_id], pk_conflict: NoCheck } + └─StreamValues { rows: [[Abs(1:Int32), 1:Int64, 0:Int64]] } +- sql: | + create table t(x int , arr int[]); + select * from t, abs(x) WITH ORDINALITY; + batch_plan: |- + BatchNestedLoopJoin { type: Inner, predicate: true, output: all } + ├─BatchExchange { order: [], dist: Single } + │ └─BatchHashJoin { type: Inner, predicate: t.x IS NOT DISTINCT FROM t.x, output: [t.x, t.arr] } + │ ├─BatchExchange { order: [], dist: HashShard(t.x) } + │ │ └─BatchScan { table: t, columns: [t.x, t.arr], distribution: SomeShard } + │ └─BatchHashAgg { group_key: [t.x], aggs: [] } + │ └─BatchExchange { order: [], dist: HashShard(t.x) } + │ └─BatchScan { table: t, columns: [t.x], distribution: SomeShard } + └─BatchValues { rows: [[Abs(CorrelatedInputRef { index: 0, correlated_id: 1 }), 1:Int64]] } + stream_error: |- + Not supported: streaming nested-loop join + HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible. + See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md diff --git a/src/frontend/src/binder/relation/join.rs b/src/frontend/src/binder/relation/join.rs index fed759d6eb83b..eb4ce96f9ab3f 100644 --- a/src/frontend/src/binder/relation/join.rs +++ b/src/frontend/src/binder/relation/join.rs @@ -59,7 +59,7 @@ impl Binder { let is_lateral = match &right { Relation::Subquery(subquery) if subquery.lateral => true, - Relation::TableFunction(_) => true, + Relation::TableFunction { .. } => true, _ => false, }; @@ -110,7 +110,7 @@ impl Binder { let is_lateral = match &right { Relation::Subquery(subquery) if subquery.lateral => true, - Relation::TableFunction(_) => true, + Relation::TableFunction { .. } => true, _ => false, }; diff --git a/src/frontend/src/binder/relation/mod.rs b/src/frontend/src/binder/relation/mod.rs index 2df943362e3f9..4ee4337c2f048 100644 --- a/src/frontend/src/binder/relation/mod.rs +++ b/src/frontend/src/binder/relation/mod.rs @@ -55,7 +55,11 @@ pub enum Relation { Join(Box), Apply(Box), WindowTableFunction(Box), - TableFunction(ExprImpl), + /// Table function or scalar function. + TableFunction { + expr: ExprImpl, + with_ordinality: bool, + }, Watermark(Box), Share(Box), } @@ -69,7 +73,9 @@ impl RewriteExprsRecursive for Relation { Relation::WindowTableFunction(inner) => inner.rewrite_exprs_recursive(rewriter), Relation::Watermark(inner) => inner.rewrite_exprs_recursive(rewriter), Relation::Share(inner) => inner.rewrite_exprs_recursive(rewriter), - Relation::TableFunction(inner) => *inner = rewriter.rewrite_expr(inner.take()), + Relation::TableFunction { expr: inner, .. } => { + *inner = rewriter.rewrite_expr(inner.take()) + } _ => {} } } @@ -113,7 +119,10 @@ impl Relation { ); correlated_indices } - Relation::TableFunction(table_function) => table_function + Relation::TableFunction { + expr: table_function, + with_ordinality: _, + } => table_function .collect_correlated_indices_by_depth_and_assign_id(depth + 1, correlated_id), _ => vec![], } @@ -437,9 +446,14 @@ impl Binder { alias, for_system_time_as_of_proctime, } => self.bind_relation_by_name(name, alias, for_system_time_as_of_proctime), - TableFactor::TableFunction { name, alias, args } => { + TableFactor::TableFunction { + name, + alias, + args, + with_ordinality, + } => { self.try_mark_lateral_as_visible(); - let result = self.bind_table_function(name, alias, args); + let result = self.bind_table_function(name, alias, args, with_ordinality); self.try_mark_lateral_as_invisible(); result } diff --git a/src/frontend/src/binder/relation/table_function.rs b/src/frontend/src/binder/relation/table_function.rs index 7441473316e33..988ea0561a860 100644 --- a/src/frontend/src/binder/relation/table_function.rs +++ b/src/frontend/src/binder/relation/table_function.rs @@ -18,6 +18,7 @@ use itertools::Itertools; use risingwave_common::catalog::{ Field, Schema, PG_CATALOG_SCHEMA_NAME, RW_INTERNAL_TABLE_FUNCTION_NAME, }; +use risingwave_common::error::ErrorCode; use risingwave_common::types::DataType; use risingwave_sqlparser::ast::{Function, FunctionArg, ObjectName, TableAlias}; @@ -34,16 +35,29 @@ impl Binder { /// /// Besides [`crate::expr::TableFunction`] expr, it can also be other things like window table /// functions, or scalar functions. + /// + /// `with_ordinality` is only supported for the `TableFunction` case now. pub(super) fn bind_table_function( &mut self, name: ObjectName, alias: Option, args: Vec, + with_ordinality: bool, ) -> Result { let func_name = &name.0[0].real_value(); // internal/system table functions { if func_name.eq_ignore_ascii_case(RW_INTERNAL_TABLE_FUNCTION_NAME) { + if with_ordinality { + return Err(ErrorCode::NotImplemented( + format!( + "WITH ORDINALITY for internal/system table function {}", + func_name + ), + None.into(), + ) + .into()); + } return self.bind_internal_table(args, alias); } if func_name.eq_ignore_ascii_case(PG_GET_KEYWORDS_FUNC_NAME) @@ -51,6 +65,16 @@ impl Binder { format!("{}.{}", PG_CATALOG_SCHEMA_NAME, PG_GET_KEYWORDS_FUNC_NAME).as_str(), ) { + if with_ordinality { + return Err(ErrorCode::NotImplemented( + format!( + "WITH ORDINALITY for internal/system table function {}", + func_name + ), + None.into(), + ) + .into()); + } return self.bind_relation_by_name_inner( Some(PG_CATALOG_SCHEMA_NAME), PG_KEYWORDS_TABLE_NAME, @@ -61,12 +85,25 @@ impl Binder { } // window table functions (tumble/hop) if let Ok(kind) = WindowTableFunctionKind::from_str(func_name) { + if with_ordinality { + return Err(ErrorCode::InvalidInputSyntax(format!( + "WITH ORDINALITY for window table function {}", + func_name + )) + .into()); + } return Ok(Relation::WindowTableFunction(Box::new( self.bind_window_table_function(alias, kind, args)?, ))); } // watermark if is_watermark_func(func_name) { + if with_ordinality { + return Err(ErrorCode::InvalidInputSyntax( + "WITH ORDINALITY for watermark".to_string(), + ) + .into()); + } return Ok(Relation::Watermark(Box::new( self.bind_watermark(alias, args)?, ))); @@ -88,13 +125,14 @@ impl Binder { self.pop_context()?; let func = func?; - let columns = if let DataType::Struct(s) = func.return_type() { - // If the table function returns a struct, it's fields can be accessed just - // like a table's columns. + // bool indicates if the field is hidden + let mut columns = if let DataType::Struct(s) = func.return_type() { + // If the table function returns a struct, it will be flattened into multiple columns. let schema = Schema::from(&s); schema.fields.into_iter().map(|f| (false, f)).collect_vec() } else { - // If there is an table alias, we should use the alias as the table function's + // If there is an table alias (and it doesn't return a struct), + // we should use the alias as the table function's // column name. If column aliases are also provided, they // are handled in bind_table_to_context. // @@ -112,9 +150,15 @@ impl Binder { }; vec![(false, Field::with_name(func.return_type(), col_name))] }; + if with_ordinality { + columns.push((false, Field::with_name(DataType::Int64, "ordinality"))); + } self.bind_table_to_context(columns, func_name.clone(), alias)?; - Ok(Relation::TableFunction(func)) + Ok(Relation::TableFunction { + expr: func, + with_ordinality, + }) } } diff --git a/src/frontend/src/optimizer/plan_node/logical_table_function.rs b/src/frontend/src/optimizer/plan_node/logical_table_function.rs index ee60a624be3ba..c42a51aeb9024 100644 --- a/src/frontend/src/optimizer/plan_node/logical_table_function.rs +++ b/src/frontend/src/optimizer/plan_node/logical_table_function.rs @@ -14,7 +14,7 @@ use pretty_xmlish::{Pretty, XmlNode}; use risingwave_common::catalog::{Field, Schema}; -use risingwave_common::error::{ErrorCode, Result}; +use risingwave_common::error::Result; use risingwave_common::types::DataType; use super::utils::{childless_record, Distill}; @@ -25,23 +25,30 @@ use super::{ use crate::expr::{Expr, ExprRewriter, TableFunction}; use crate::optimizer::optimizer_context::OptimizerContextRef; use crate::optimizer::plan_node::{ - BatchTableFunction, ColumnPruningContext, PredicatePushdownContext, RewriteStreamContext, - ToStreamContext, + ColumnPruningContext, PredicatePushdownContext, RewriteStreamContext, ToStreamContext, }; use crate::optimizer::property::FunctionalDependencySet; use crate::utils::{ColIndexMapping, Condition}; -/// `LogicalGenerateSeries` implements Hop Table Function. +/// `LogicalTableFunction` is a scalar/table function used as a relation (in the `FROM` clause). +/// +/// If the function returns a struct, it will be flattened into multiple columns. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct LogicalTableFunction { pub base: PlanBase, pub table_function: TableFunction, + pub with_ordinality: bool, } impl LogicalTableFunction { /// Create a [`LogicalTableFunction`] node. Used internally by optimizer. - pub fn new(table_function: TableFunction, ctx: OptimizerContextRef) -> Self { - let schema = if let DataType::Struct(s) = table_function.return_type() { + pub fn new( + table_function: TableFunction, + with_ordinality: bool, + ctx: OptimizerContextRef, + ) -> Self { + let mut schema = if let DataType::Struct(s) = table_function.return_type() { + // If the function returns a struct, it will be flattened into multiple columns. Schema::from(&s) } else { Schema { @@ -51,11 +58,17 @@ impl LogicalTableFunction { )], } }; + if with_ordinality { + schema + .fields + .push(Field::with_name(DataType::Int64, "ordinality")); + } let functional_dependency = FunctionalDependencySet::new(schema.len()); let base = PlanBase::new_logical(ctx, schema, vec![], functional_dependency); Self { base, table_function, + with_ordinality, } } @@ -110,26 +123,19 @@ impl PredicatePushdown for LogicalTableFunction { impl ToBatch for LogicalTableFunction { fn to_batch(&self) -> Result { - Ok(BatchTableFunction::new(self.clone()).into()) + unreachable!("TableFunction should be converted to ProjectSet") } } impl ToStream for LogicalTableFunction { fn to_stream(&self, _ctx: &mut ToStreamContext) -> Result { - Err( - ErrorCode::NotImplemented("LogicalTableFunction::to_stream".to_string(), None.into()) - .into(), - ) + unreachable!("TableFunction should be converted to ProjectSet") } fn logical_rewrite_for_stream( &self, _ctx: &mut RewriteStreamContext, ) -> Result<(PlanRef, ColIndexMapping)> { - Err(ErrorCode::NotImplemented( - "LogicalTableFunction::logical_rewrite_for_stream".to_string(), - None.into(), - ) - .into()) + unreachable!("TableFunction should be converted to ProjectSet") } } diff --git a/src/frontend/src/optimizer/rule/table_function_to_project_set_rule.rs b/src/frontend/src/optimizer/rule/table_function_to_project_set_rule.rs index 095e08664e1c4..5a6f1187fdd02 100644 --- a/src/frontend/src/optimizer/rule/table_function_to_project_set_rule.rs +++ b/src/frontend/src/optimizer/rule/table_function_to_project_set_rule.rs @@ -19,11 +19,11 @@ use risingwave_common::types::DataType; use super::{BoxedRule, Rule}; use crate::expr::{Expr, ExprImpl, ExprType, FunctionCall, InputRef}; use crate::optimizer::plan_node::{ - LogicalProject, LogicalProjectSet, LogicalTableFunction, LogicalValues, + LogicalProject, LogicalProjectSet, LogicalTableFunction, LogicalValues, PlanTreeNodeUnary, }; use crate::optimizer::PlanRef; -/// Transform a table function into a project set +/// Transform a `TableFunction` (used in FROM clause) into a `ProjectSet` so that it can be unnested later if it contains `CorrelatedInputRef`. /// /// Before: /// @@ -54,11 +54,11 @@ impl Rule for TableFunctionToProjectSetRule { logical_table_function.base.ctx.clone(), ); let logical_project_set = LogicalProjectSet::create(logical_values, vec![table_function]); - // We need a project to align schema type because `LogicalProjectSet` has a hidden column - // `projected_row_id` and table function could return multiple columns, while project set - // return only one column with struct type. + // We need a project to align schema type because + // 1. `LogicalProjectSet` has a hidden column `projected_row_id` (0-th col) + // 2. When the function returns a struct type, TableFunction will return flatten it into multiple columns, while ProjectSet still returns a single column. let table_function_col_idx = 1; - if let DataType::Struct(st) = table_function_return_type.clone() { + let logical_project = if let DataType::Struct(st) = table_function_return_type.clone() { let exprs = st .types() .enumerate() @@ -66,13 +66,11 @@ impl Rule for TableFunctionToProjectSetRule { let field_access = FunctionCall::new_unchecked( ExprType::Field, vec![ - ExprImpl::InputRef( - InputRef::new( - table_function_col_idx, - table_function_return_type.clone(), - ) - .into(), - ), + InputRef::new( + table_function_col_idx, + table_function_return_type.clone(), + ) + .into(), ExprImpl::literal_int(i as i32), ], data_type.clone(), @@ -80,13 +78,27 @@ impl Rule for TableFunctionToProjectSetRule { ExprImpl::FunctionCall(field_access.into()) }) .collect_vec(); - let logical_project = LogicalProject::new(logical_project_set, exprs); - Some(logical_project.into()) + LogicalProject::new(logical_project_set, exprs) } else { - let logical_project = LogicalProject::with_out_col_idx( + LogicalProject::with_out_col_idx( logical_project_set, std::iter::once(table_function_col_idx), - ); + ) + }; + + if logical_table_function.with_ordinality { + let projected_row_id = InputRef::new(0, DataType::Int64).into(); + let ordinality = FunctionCall::new( + ExprType::Add, + vec![projected_row_id, ExprImpl::literal_bigint(1)], + ) + .unwrap() // i64 + i64 is ok + .into(); + let mut exprs = logical_project.exprs().clone(); + exprs.push(ordinality); + let logical_project = LogicalProject::new(logical_project.input(), exprs); + Some(logical_project.into()) + } else { Some(logical_project.into()) } } diff --git a/src/frontend/src/planner/relation.rs b/src/frontend/src/planner/relation.rs index db4bb0233f077..f4085f6ffa42e 100644 --- a/src/frontend/src/planner/relation.rs +++ b/src/frontend/src/planner/relation.rs @@ -46,7 +46,10 @@ impl Planner { Relation::Apply(join) => self.plan_apply(*join), Relation::WindowTableFunction(tf) => self.plan_window_table_function(*tf), Relation::Source(s) => self.plan_source(*s), - Relation::TableFunction(tf) => self.plan_table_function(tf), + Relation::TableFunction { + expr: tf, + with_ordinality, + } => self.plan_table_function(tf, with_ordinality), Relation::Watermark(tf) => self.plan_watermark(*tf), Relation::Share(share) => self.plan_share(*share), } @@ -150,16 +153,33 @@ impl Planner { } } - pub(super) fn plan_table_function(&mut self, table_function: ExprImpl) -> Result { + pub(super) fn plan_table_function( + &mut self, + table_function: ExprImpl, + with_ordinality: bool, + ) -> Result { // TODO: maybe we can unify LogicalTableFunction with LogicalValues match table_function { - ExprImpl::TableFunction(tf) => Ok(LogicalTableFunction::new(*tf, self.ctx()).into()), + ExprImpl::TableFunction(tf) => { + Ok(LogicalTableFunction::new(*tf, with_ordinality, self.ctx()).into()) + } expr => { - let schema = Schema { + let mut schema = Schema { // TODO: should be named fields: vec![Field::unnamed(expr.return_type())], }; - Ok(LogicalValues::create(vec![vec![expr]], schema, self.ctx())) + if with_ordinality { + schema + .fields + .push(Field::with_name(DataType::Int64, "ordinality")); + Ok(LogicalValues::create( + vec![vec![expr, ExprImpl::literal_bigint(1)]], + schema, + self.ctx(), + )) + } else { + Ok(LogicalValues::create(vec![vec![expr]], schema, self.ctx())) + } } } } diff --git a/src/sqlparser/src/ast/query.rs b/src/sqlparser/src/ast/query.rs index cc703e5b81a38..f018b853f3330 100644 --- a/src/sqlparser/src/ast/query.rs +++ b/src/sqlparser/src/ast/query.rs @@ -387,11 +387,14 @@ pub enum TableFactor { subquery: Box, alias: Option, }, - /// `[ AS ]` + /// `(args)[ AS ]` + /// + /// Note that scalar functions can also be used in this way. TableFunction { name: ObjectName, alias: Option, args: Vec, + with_ordinality: bool, }, /// Represents a parenthesized table factor. The SQL spec only allows a /// join expression (`(foo bar [ baz ... ])`) to be nested, @@ -433,8 +436,16 @@ impl fmt::Display for TableFactor { } Ok(()) } - TableFactor::TableFunction { name, alias, args } => { + TableFactor::TableFunction { + name, + alias, + args, + with_ordinality, + } => { write!(f, "{}({})", name, display_comma_separated(args))?; + if *with_ordinality { + write!(f, " WITH ORDINALITY")?; + } if let Some(alias) = alias { write!(f, " AS {}", alias)?; } diff --git a/src/sqlparser/src/keywords.rs b/src/sqlparser/src/keywords.rs index 283a0a409512c..5c2fedb0ea547 100644 --- a/src/sqlparser/src/keywords.rs +++ b/src/sqlparser/src/keywords.rs @@ -350,6 +350,7 @@ define_keywords!( OPTION, OR, ORDER, + ORDINALITY, OTHERS, OUT, OUTER, diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs index 4302a4ea12ab1..7cb8a099436c9 100644 --- a/src/sqlparser/src/parser.rs +++ b/src/sqlparser/src/parser.rs @@ -4275,8 +4275,15 @@ impl Parser { if !order_by.is_empty() { return parser_err!("Table-valued functions do not support ORDER BY clauses"); } + let with_ordinality = self.parse_keywords(&[Keyword::WITH, Keyword::ORDINALITY]); + let alias = self.parse_optional_table_alias(keywords::RESERVED_FOR_TABLE_ALIAS)?; - Ok(TableFactor::TableFunction { name, alias, args }) + Ok(TableFactor::TableFunction { + name, + alias, + args, + with_ordinality, + }) } else { let for_system_time_as_of_proctime = self.parse_for_system_time_as_of_proctime()?; let alias = self.parse_optional_table_alias(keywords::RESERVED_FOR_TABLE_ALIAS)?; diff --git a/src/sqlparser/tests/testdata/select.yaml b/src/sqlparser/tests/testdata/select.yaml index bbbb5a72bbdab..6aed3d2a4dc4c 100644 --- a/src/sqlparser/tests/testdata/select.yaml +++ b/src/sqlparser/tests/testdata/select.yaml @@ -29,10 +29,10 @@ formatted_sql: SELECT (CAST(ROW(1, 2, 3) AS foo)).v1.* - input: SELECT * FROM generate_series('2'::INT,'10'::INT,'2'::INT) formatted_sql: SELECT * FROM generate_series(CAST('2' AS INT), CAST('10' AS INT), CAST('2' AS INT)) - formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [Wildcard(None)], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "generate_series", quote_style: None }]), alias: None, args: [Unnamed(Expr(Cast { expr: Value(SingleQuotedString("2")), data_type: Int })), Unnamed(Expr(Cast { expr: Value(SingleQuotedString("10")), data_type: Int })), Unnamed(Expr(Cast { expr: Value(SingleQuotedString("2")), data_type: Int }))] }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' + formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [Wildcard(None)], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "generate_series", quote_style: None }]), alias: None, args: [Unnamed(Expr(Cast { expr: Value(SingleQuotedString("2")), data_type: Int })), Unnamed(Expr(Cast { expr: Value(SingleQuotedString("10")), data_type: Int })), Unnamed(Expr(Cast { expr: Value(SingleQuotedString("2")), data_type: Int }))], with_ordinality: false }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' - input: SELECT * FROM unnest(Array[1,2,3]); formatted_sql: SELECT * FROM unnest(ARRAY[1, 2, 3]) - formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [Wildcard(None)], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "unnest", quote_style: None }]), alias: None, args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("3"))], named: true })))] }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' + formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [Wildcard(None)], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "unnest", quote_style: None }]), alias: None, args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("3"))], named: true })))], with_ordinality: false }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' - input: SELECT id, fname, lname FROM customer WHERE salary <> 'Not Provided' AND salary <> '' formatted_sql: SELECT id, fname, lname FROM customer WHERE salary <> 'Not Provided' AND salary <> '' - input: SELECT id FROM customer WHERE NOT salary = '' @@ -102,7 +102,7 @@ formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Identifier(Ident { value: "id1", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "a1", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "id2", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "a2", quote_style: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "stream", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "S", quote_style: None }, columns: [] }), for_system_time_as_of_proctime: false }, joins: [Join { relation: Table { name: ObjectName([Ident { value: "version", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "V", quote_style: None }, columns: [] }), for_system_time_as_of_proctime: true }, join_operator: Inner(On(BinaryOp { left: Identifier(Ident { value: "id1", quote_style: None }), op: Eq, right: Identifier(Ident { value: "id2", quote_style: None }) })) }] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' - input: select percentile_cont(0.3) within group (order by x desc) from unnest(array[1,2,4,5,10]) as x formatted_sql: SELECT percentile_cont(0.3) FROM unnest(ARRAY[1, 2, 4, 5, 10]) AS x - formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "percentile_cont", quote_style: None }]), args: [Unnamed(Expr(Value(Number("0.3"))))], over: None, distinct: false, order_by: [], filter: None, within_group: Some(OrderByExpr { expr: Identifier(Ident { value: "x", quote_style: None }), asc: Some(false), nulls_first: None }) }))], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "unnest", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "x", quote_style: None }, columns: [] }), args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("4")), Value(Number("5")), Value(Number("10"))], named: true })))] }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' + formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "percentile_cont", quote_style: None }]), args: [Unnamed(Expr(Value(Number("0.3"))))], over: None, distinct: false, order_by: [], filter: None, within_group: Some(OrderByExpr { expr: Identifier(Ident { value: "x", quote_style: None }), asc: Some(false), nulls_first: None }) }))], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "unnest", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "x", quote_style: None }, columns: [] }), args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("4")), Value(Number("5")), Value(Number("10"))], named: true })))], with_ordinality: false }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' - input: select percentile_cont(0.3) within group (order by x, y desc) from t error_msg: 'sql parser error: only one arg in order by is expected here' - input: select 'apple' ~~ 'app%' diff --git a/src/tests/sqlsmith/src/sql_gen/time_window.rs b/src/tests/sqlsmith/src/sql_gen/time_window.rs index d5fd7c8936b22..053af729ee526 100644 --- a/src/tests/sqlsmith/src/sql_gen/time_window.rs +++ b/src/tests/sqlsmith/src/sql_gen/time_window.rs @@ -46,7 +46,7 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { let time_col = time_cols.choose(&mut self.rng).unwrap(); let time_col = Expr::Identifier(time_col.name.as_str().into()); let args = create_args(vec![name, time_col, size]); - let relation = create_tvf("tumble", alias, args); + let relation = create_tvf("tumble", alias, args, false); let table = Table::new(table_name, schema.clone()); @@ -72,7 +72,7 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { let time_col = Expr::Identifier(time_col.name.as_str().into()); let args = create_args(vec![name, time_col, slide, size]); - let relation = create_tvf("hop", alias, args); + let relation = create_tvf("hop", alias, args, false); let table = Table::new(table_name, schema.clone()); @@ -120,11 +120,17 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { } /// Create a table view function. -fn create_tvf(name: &str, alias: TableAlias, args: Vec) -> TableFactor { +fn create_tvf( + name: &str, + alias: TableAlias, + args: Vec, + with_ordinality: bool, +) -> TableFactor { TableFactor::TableFunction { name: ObjectName(vec![name.into()]), alias: Some(alias), args, + with_ordinality, } } From f25c625bdc7cdee4d43a5f4551f513f804f1c29e Mon Sep 17 00:00:00 2001 From: "Zhanxiang (Patrick) Huang" Date: Wed, 13 Sep 2023 23:11:57 -0700 Subject: [PATCH 08/14] feat(sink): support elasticsearch 8 sink (#12269) --- .../sink/elasticsearch/elasticsearch_sink.slt | 2 +- .../elasticsearch-sink/README.md | 41 +++++++++++ .../elasticsearch-sink/create_es7_sink.sql | 9 +++ .../elasticsearch-sink/create_es8_sink.sql | 9 +++ .../elasticsearch-sink/create_mv.sql | 7 ++ .../elasticsearch-sink/create_source.sql | 18 +++++ .../elasticsearch-sink/docker-compose.yml | 73 +++++++++++++++++++ .../python-client/integration_tests.py | 2 +- .../com/risingwave/connector/SinkUtils.java | 4 +- .../{EsSink7Test.java => EsSinkTest.java} | 12 +-- .../connector/{EsSink7.java => EsSink.java} | 31 +++++--- .../{EsSink7Config.java => EsSinkConfig.java} | 10 +-- ...EsSink7Factory.java => EsSinkFactory.java} | 10 +-- src/connector/src/sink/remote.rs | 2 +- 14 files changed, 200 insertions(+), 30 deletions(-) create mode 100644 integration_tests/elasticsearch-sink/README.md create mode 100644 integration_tests/elasticsearch-sink/create_es7_sink.sql create mode 100644 integration_tests/elasticsearch-sink/create_es8_sink.sql create mode 100644 integration_tests/elasticsearch-sink/create_mv.sql create mode 100644 integration_tests/elasticsearch-sink/create_source.sql create mode 100644 integration_tests/elasticsearch-sink/docker-compose.yml rename java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/elasticsearch/{EsSink7Test.java => EsSinkTest.java} (94%) rename java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/{EsSink7.java => EsSink.java} (89%) rename java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/{EsSink7Config.java => EsSinkConfig.java} (87%) rename java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/{EsSink7Factory.java => EsSinkFactory.java} (91%) diff --git a/e2e_test/sink/elasticsearch/elasticsearch_sink.slt b/e2e_test/sink/elasticsearch/elasticsearch_sink.slt index 2728f33da04e5..cecf9a47cf94a 100644 --- a/e2e_test/sink/elasticsearch/elasticsearch_sink.slt +++ b/e2e_test/sink/elasticsearch/elasticsearch_sink.slt @@ -3,7 +3,7 @@ CREATE TABLE t7 (v1 int primary key, v2 bigint, v3 varchar); statement ok CREATE SINK s7 AS select t7.v1 as v1, t7.v2 as v2, t7.v3 as v3 from t7 WITH ( - connector = 'elasticsearch-7', + connector = 'elasticsearch', index = 'test', url = 'http://elasticsearch:9200', username = 'elastic', diff --git a/integration_tests/elasticsearch-sink/README.md b/integration_tests/elasticsearch-sink/README.md new file mode 100644 index 0000000000000..b114e8132024a --- /dev/null +++ b/integration_tests/elasticsearch-sink/README.md @@ -0,0 +1,41 @@ +# Demo: Sinking to ElasticSearch + +In this demo, we want to showcase how RisingWave is able to sink data to ElasticSearch. + +1. Set the compose profile accordingly: +Demo with elasticsearch 7: +``` +export COMPOSE_PROFILES=es7 +``` + +Demo with elasticsearch 8 +``` +export COMPOSE_PROFILES=es8 +``` + +2. Launch the cluster: + +```sh +docker-compose up -d +``` + +The cluster contains a RisingWave cluster and its necessary dependencies, a datagen that generates the data, a single-node elasticsearch for sink. + +3. Execute the SQL queries in sequence: + +- create_source.sql +- create_mv.sql +- create_es[7/8]_sink.sql + +4. Check the contents in ES: + +```sh +# Check the document counts +curl -XGET -u elastic:risingwave "http://localhost:9200/test/_count" -H 'Content-Type: application/json' + +# Check the content of a document by user_id +curl -XGET -u elastic:risingwave "http://localhost:9200/test/_search" -H 'Content-Type: application/json' -d '{"query":{"term": {"user_id":2}}' | jq + +# Get the first 10 documents sort by user_id +curl -XGET -u elastic:risingwave "http://localhost:9200/test/_search?size=10" -H 'Content-Type: application/json' -d'{"query":{"match_all":{}}, "sort": ["user_id"]}' | jq +``` \ No newline at end of file diff --git a/integration_tests/elasticsearch-sink/create_es7_sink.sql b/integration_tests/elasticsearch-sink/create_es7_sink.sql new file mode 100644 index 0000000000000..997c238b90344 --- /dev/null +++ b/integration_tests/elasticsearch-sink/create_es7_sink.sql @@ -0,0 +1,9 @@ +CREATE SINK bhv_es_sink +FROM + bhv_mv WITH ( + connector = 'elasticsearch', + index = 'test', + url = 'http://elasticsearch8:9200', + username = 'elastic', + password = 'risingwave' +); \ No newline at end of file diff --git a/integration_tests/elasticsearch-sink/create_es8_sink.sql b/integration_tests/elasticsearch-sink/create_es8_sink.sql new file mode 100644 index 0000000000000..997c238b90344 --- /dev/null +++ b/integration_tests/elasticsearch-sink/create_es8_sink.sql @@ -0,0 +1,9 @@ +CREATE SINK bhv_es_sink +FROM + bhv_mv WITH ( + connector = 'elasticsearch', + index = 'test', + url = 'http://elasticsearch8:9200', + username = 'elastic', + password = 'risingwave' +); \ No newline at end of file diff --git a/integration_tests/elasticsearch-sink/create_mv.sql b/integration_tests/elasticsearch-sink/create_mv.sql new file mode 100644 index 0000000000000..0a803f8a2762d --- /dev/null +++ b/integration_tests/elasticsearch-sink/create_mv.sql @@ -0,0 +1,7 @@ +CREATE MATERIALIZED VIEW bhv_mv AS +SELECT + user_id, + target_id, + event_timestamp +FROM + user_behaviors; \ No newline at end of file diff --git a/integration_tests/elasticsearch-sink/create_source.sql b/integration_tests/elasticsearch-sink/create_source.sql new file mode 100644 index 0000000000000..c28c10f3616da --- /dev/null +++ b/integration_tests/elasticsearch-sink/create_source.sql @@ -0,0 +1,18 @@ +CREATE table user_behaviors ( + user_id int, + target_id VARCHAR, + target_type VARCHAR, + event_timestamp TIMESTAMP, + behavior_type VARCHAR, + parent_target_type VARCHAR, + parent_target_id VARCHAR, + PRIMARY KEY(user_id) +) WITH ( + connector = 'datagen', + fields.user_id.kind = 'sequence', + fields.user_id.start = '1', + fields.user_id.end = '1000', + fields.user_name.kind = 'random', + fields.user_name.length = '10', + datagen.rows.per.second = '10' +) FORMAT PLAIN ENCODE JSON; \ No newline at end of file diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml new file mode 100644 index 0000000000000..47d314d1f57e2 --- /dev/null +++ b/integration_tests/elasticsearch-sink/docker-compose.yml @@ -0,0 +1,73 @@ +--- +version: "3" +services: + elasticsearch7: + image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 + environment: + - xpack.security.enabled=true + - discovery.type=single-node + - ELASTIC_PASSWORD=risingwave + ports: + - 9200:9200 + profiles: + - es7 + elasticsearch8: + image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0 + environment: + - xpack.security.enabled=true + - discovery.type=single-node + - ELASTIC_PASSWORD=risingwave + ports: + - 9200:9200 + profiles: + - es8 + compactor-0: + extends: + file: ../../docker/docker-compose.yml + service: compactor-0 + compute-node-0: + extends: + file: ../../docker/docker-compose.yml + service: compute-node-0 + etcd-0: + extends: + file: ../../docker/docker-compose.yml + service: etcd-0 + frontend-node-0: + extends: + file: ../../docker/docker-compose.yml + service: frontend-node-0 + grafana-0: + extends: + file: ../../docker/docker-compose.yml + service: grafana-0 + meta-node-0: + extends: + file: ../../docker/docker-compose.yml + service: meta-node-0 + connector-node: + extends: + file: ../../docker/docker-compose.yml + service: connector-node + minio-0: + extends: + file: ../../docker/docker-compose.yml + service: minio-0 + prometheus-0: + extends: + file: ../../docker/docker-compose.yml + service: prometheus-0 +volumes: + compute-node-0: + external: false + etcd-0: + external: false + grafana-0: + external: false + minio-0: + external: false + prometheus-0: + external: false + message_queue: + external: false +name: risingwave-compose diff --git a/java/connector-node/python-client/integration_tests.py b/java/connector-node/python-client/integration_tests.py index 962f2a658018a..64fa949f48ce2 100644 --- a/java/connector-node/python-client/integration_tests.py +++ b/java/connector-node/python-client/integration_tests.py @@ -272,7 +272,7 @@ def test_jdbc_sink(input_file, param): def test_elasticsearch_sink(param): prop = { - "connector": "elasticsearch-7", + "connector": "elasticsearch", "url": "http://127.0.0.1:9200", "index": "test", } diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkUtils.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkUtils.java index ab3ac84346fa6..944d529a02d8d 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkUtils.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkUtils.java @@ -41,8 +41,8 @@ public static SinkFactory getSinkFactory(String sinkName) { return new IcebergSinkFactory(); case "deltalake": return new DeltaLakeSinkFactory(); - case "elasticsearch-7": - return new EsSink7Factory(); + case "elasticsearch": + return new EsSinkFactory(); case "cassandra": return new CassandraFactory(); default: diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/elasticsearch/EsSink7Test.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/elasticsearch/EsSinkTest.java similarity index 94% rename from java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/elasticsearch/EsSink7Test.java rename to java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/elasticsearch/EsSinkTest.java index e3024ff09b26e..af0ea7190f946 100644 --- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/elasticsearch/EsSink7Test.java +++ b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/elasticsearch/EsSinkTest.java @@ -19,8 +19,8 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import com.risingwave.connector.EsSink7; -import com.risingwave.connector.EsSink7Config; +import com.risingwave.connector.EsSink; +import com.risingwave.connector.EsSinkConfig; import com.risingwave.connector.api.TableSchema; import com.risingwave.connector.api.sink.ArraySinkRow; import com.risingwave.proto.Data; @@ -39,7 +39,7 @@ import org.junit.Test; import org.testcontainers.elasticsearch.ElasticsearchContainer; -public class EsSink7Test { +public class EsSinkTest { static TableSchema getTestTableSchema() { return new TableSchema( @@ -52,9 +52,9 @@ static TableSchema getTestTableSchema() { public void testEsSink(ElasticsearchContainer container, String username, String password) throws IOException { - EsSink7 sink = - new EsSink7( - new EsSink7Config(container.getHttpHostAddress(), "test") + EsSink sink = + new EsSink( + new EsSinkConfig(container.getHttpHostAddress(), "test") .withDelimiter("$") .withUsername(username) .withPassword(password), diff --git a/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink7.java b/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink.java similarity index 89% rename from java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink7.java rename to java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink.java index 44f1610ceaaba..7c1727f4a82f3 100644 --- a/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink7.java +++ b/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink.java @@ -36,6 +36,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.RestHighLevelClientBuilder; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; @@ -55,17 +56,17 @@ * * 4. bulkprocessor and high-level-client are deprecated in es 8 java api. */ -public class EsSink7 extends SinkWriterBase { - private static final Logger LOG = LoggerFactory.getLogger(EsSink7.class); +public class EsSink extends SinkWriterBase { + private static final Logger LOG = LoggerFactory.getLogger(EsSink.class); private static final String ERROR_REPORT_TEMPLATE = "Error when exec %s, message %s"; - private final EsSink7Config config; + private final EsSinkConfig config; private final BulkProcessor bulkProcessor; private final RestHighLevelClient client; // For bulk listener private final List primaryKeyIndexes; - public EsSink7(EsSink7Config config, TableSchema tableSchema) { + public EsSink(EsSinkConfig config, TableSchema tableSchema) { super(tableSchema); HttpHost host; try { @@ -75,9 +76,14 @@ public EsSink7(EsSink7Config config, TableSchema tableSchema) { } this.config = config; + + // ApiCompatibilityMode is enabled to ensure the client can talk to newer version es sever. this.client = - new RestHighLevelClient( - configureRestClientBuilder(RestClient.builder(host), config)); + new RestHighLevelClientBuilder( + configureRestClientBuilder(RestClient.builder(host), config) + .build()) + .setApiCompatibilityMode(true) + .build(); // Test connection try { boolean isConnected = this.client.ping(RequestOptions.DEFAULT); @@ -98,7 +104,7 @@ public EsSink7(EsSink7Config config, TableSchema tableSchema) { } private static RestClientBuilder configureRestClientBuilder( - RestClientBuilder builder, EsSink7Config config) { + RestClientBuilder builder, EsSinkConfig config) { // Possible config: // 1. Connection path prefix // 2. Username and password @@ -116,7 +122,7 @@ private static RestClientBuilder configureRestClientBuilder( } private BulkProcessor.Builder applyBulkConfig( - RestHighLevelClient client, EsSink7Config config, BulkProcessor.Listener listener) { + RestHighLevelClient client, EsSinkConfig config, BulkProcessor.Listener listener) { BulkProcessor.Builder builder = BulkProcessor.builder( (BulkRequestConsumerFactory) @@ -181,7 +187,14 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) private Map buildDoc(SinkRow row) { Map doc = new HashMap(); for (int i = 0; i < getTableSchema().getNumColumns(); i++) { - doc.put(getTableSchema().getColumnDesc(i).getName(), row.get(i)); + Object col = row.get(i); + if (col instanceof Date) { + // es client doesn't natively support java.sql.Timestamp/Time/Date + // so we need to convert Date type into a string as suggested in + // https://github.com/elastic/elasticsearch/issues/31377#issuecomment-398102292 + col = col.toString(); + } + doc.put(getTableSchema().getColumnDesc(i).getName(), col); } return doc; } diff --git a/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink7Config.java b/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSinkConfig.java similarity index 87% rename from java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink7Config.java rename to java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSinkConfig.java index 6d45b3ee52d87..e053dfed77b63 100644 --- a/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink7Config.java +++ b/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSinkConfig.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.risingwave.connector.api.sink.CommonSinkConfig; -public class EsSink7Config extends CommonSinkConfig { +public class EsSinkConfig extends CommonSinkConfig { /** Required */ private String url; @@ -38,7 +38,7 @@ public class EsSink7Config extends CommonSinkConfig { private String password; @JsonCreator - public EsSink7Config( + public EsSinkConfig( @JsonProperty(value = "url") String url, @JsonProperty(value = "index") String index) { this.url = url; this.index = index; @@ -64,17 +64,17 @@ public String getPassword() { return password; } - public EsSink7Config withDelimiter(String delimiter) { + public EsSinkConfig withDelimiter(String delimiter) { this.delimiter = delimiter; return this; } - public EsSink7Config withUsername(String username) { + public EsSinkConfig withUsername(String username) { this.username = username; return this; } - public EsSink7Config withPassword(String password) { + public EsSinkConfig withPassword(String password) { this.password = password; return this; } diff --git a/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink7Factory.java b/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSinkFactory.java similarity index 91% rename from java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink7Factory.java rename to java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSinkFactory.java index 4bc6fa6cc1990..a31826a45a5ab 100644 --- a/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink7Factory.java +++ b/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSinkFactory.java @@ -36,13 +36,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class EsSink7Factory implements SinkFactory { - private static final Logger LOG = LoggerFactory.getLogger(EsSink7Factory.class); +public class EsSinkFactory implements SinkFactory { + private static final Logger LOG = LoggerFactory.getLogger(EsSinkFactory.class); public SinkWriter createWriter(TableSchema tableSchema, Map tableProperties) { ObjectMapper mapper = new ObjectMapper(); - EsSink7Config config = mapper.convertValue(tableProperties, EsSink7Config.class); - return new SinkWriterV1.Adapter(new EsSink7(config, tableSchema)); + EsSinkConfig config = mapper.convertValue(tableProperties, EsSinkConfig.class); + return new SinkWriterV1.Adapter(new EsSink(config, tableSchema)); } @Override @@ -52,7 +52,7 @@ public void validate( Catalog.SinkType sinkType) { ObjectMapper mapper = new ObjectMapper(); mapper.configure(DeserializationFeature.FAIL_ON_MISSING_CREATOR_PROPERTIES, true); - EsSink7Config config = mapper.convertValue(tableProperties, EsSink7Config.class); + EsSinkConfig config = mapper.convertValue(tableProperties, EsSinkConfig.class); // 1. check url HttpHost host; diff --git a/src/connector/src/sink/remote.rs b/src/connector/src/sink/remote.rs index 046d29bc687b5..851c81b8916d0 100644 --- a/src/connector/src/sink/remote.rs +++ b/src/connector/src/sink/remote.rs @@ -52,7 +52,7 @@ pub const VALID_REMOTE_SINKS: [&str; 5] = [ "jdbc", REMOTE_ICEBERG_SINK, "deltalake", - "elasticsearch-7", + "elasticsearch", "cassandra", ]; From c7fb909188e441086cb167d0655f2ae2f5c3f648 Mon Sep 17 00:00:00 2001 From: Li0k Date: Thu, 14 Sep 2023 14:28:33 +0800 Subject: [PATCH 09/14] fix(storage): remove directly convert of tier compaction picker (#12280) --- src/meta/src/hummock/compaction/picker/mod.rs | 2 +- .../picker/tier_compaction_picker.rs | 34 +------- src/meta/src/rpc/metrics.rs | 2 +- .../hummock_test/src/compactor_tests.rs | 85 ++++++++----------- 4 files changed, 41 insertions(+), 82 deletions(-) diff --git a/src/meta/src/hummock/compaction/picker/mod.rs b/src/meta/src/hummock/compaction/picker/mod.rs index 15e7a61f548ee..cf3a4555e18e1 100644 --- a/src/meta/src/hummock/compaction/picker/mod.rs +++ b/src/meta/src/hummock/compaction/picker/mod.rs @@ -43,7 +43,7 @@ use crate::hummock::level_handler::LevelHandler; pub const MAX_COMPACT_LEVEL_COUNT: usize = 42; -#[derive(Default)] +#[derive(Default, Debug)] pub struct LocalPickerStatistic { pub skip_by_write_amp_limit: u64, pub skip_by_count_limit: u64, diff --git a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs index 99b17694f528e..5b3058317a4b0 100644 --- a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs +++ b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs @@ -14,8 +14,6 @@ use std::sync::Arc; -use risingwave_hummock_sdk::can_concat; -use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_pb::hummock::hummock_version::Levels; use risingwave_pb::hummock::{CompactionConfig, InputLevel, LevelType, OverlappingLevel}; @@ -69,33 +67,11 @@ impl TierCompactionPicker { continue; } - let mut input_level = InputLevel { + let input_level = InputLevel { level_idx: 0, level_type: level.level_type, table_infos: level.table_infos.clone(), }; - // Since the level is overlapping, we can change the order of origin sstable infos in - // task. - input_level.table_infos.sort_by(|sst1, sst2| { - let a = sst1.key_range.as_ref().unwrap(); - let b = sst2.key_range.as_ref().unwrap(); - a.compare(b) - }); - - if can_concat(&input_level.table_infos) { - return Some(CompactionInput { - select_input_size: input_level - .table_infos - .iter() - .map(|sst| sst.file_size) - .sum(), - total_file_count: input_level.table_infos.len() as u64, - input_levels: vec![input_level], - target_level: 0, - target_sub_level_id: level.sub_level_id, - ..Default::default() - }); - } let mut select_level_inputs = vec![input_level]; @@ -182,7 +158,6 @@ impl CompactionPicker for TierCompactionPicker { pub mod tests { use std::sync::Arc; - use risingwave_hummock_sdk::can_concat; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::new_sub_level; use risingwave_pb::hummock::hummock_version::Levels; use risingwave_pb::hummock::{LevelType, OverlappingLevel}; @@ -281,11 +256,8 @@ pub mod tests { // sub-level 0 is excluded because it's nonoverlapping and violating // sub_level_max_compaction_bytes. let mut picker = TierCompactionPicker::new(config); - let ret = picker - .pick_compaction(&levels, &levels_handler, &mut local_stats) - .unwrap(); - assert_eq!(ret.input_levels.len(), 1); - assert!(can_concat(&ret.input_levels[0].table_infos)); + let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats); + assert!(ret.is_none()) } #[test] diff --git a/src/meta/src/rpc/metrics.rs b/src/meta/src/rpc/metrics.rs index c19c06d9ab2cd..1518495df0f7c 100644 --- a/src/meta/src/rpc/metrics.rs +++ b/src/meta/src/rpc/metrics.rs @@ -552,7 +552,7 @@ impl MetaMetrics { let opts = histogram_opts!( "storage_compact_task_size", "Total size of compact that have been issued to state store", - exponential_buckets(4096.0, 1.6, 28).unwrap() + exponential_buckets(1048576.0, 2.0, 16).unwrap() ); let compact_task_size = diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs index ca5418cbe6a0f..5864fa9c0a484 100644 --- a/src/storage/hummock_test/src/compactor_tests.rs +++ b/src/storage/hummock_test/src/compactor_tests.rs @@ -422,67 +422,54 @@ pub(crate) mod tests { .await; // 2. get compact task - let mut compact_task = hummock_manager_ref + + // 3. compact + while let Some(compact_task) = hummock_manager_ref .get_compact_task( StaticCompactionGroupId::StateDefault.into(), &mut default_level_selector(), ) .await .unwrap() - .unwrap(); - let compaction_filter_flag = CompactionFilterFlag::NONE; - compact_task.compaction_filter_mask = compaction_filter_flag.bits(); - compact_task.current_epoch_time = 0; - - // assert compact_task - assert_eq!( - compact_task - .input_ssts - .iter() - .map(|level| level.table_infos.len()) - .sum::(), - SST_COUNT as usize / 2 + 1, - ); - compact_task.target_level = 6; - - // 3. compact - let (_tx, rx) = tokio::sync::oneshot::channel(); - let (mut result_task, task_stats) = compact( - compact_ctx, - compact_task.clone(), - rx, - Box::new(sstable_object_id_manager.clone()), - ) - .await; + { + // 3. compact + let (_tx, rx) = tokio::sync::oneshot::channel(); + let (mut result_task, task_stats) = compact( + compact_ctx.clone(), + compact_task.clone(), + rx, + Box::new(sstable_object_id_manager.clone()), + ) + .await; - hummock_manager_ref - .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats))) - .await - .unwrap(); + hummock_manager_ref + .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats))) + .await + .unwrap(); + } // 4. get the latest version and check let version = hummock_manager_ref.get_current_version().await; - let output_table = version + let output_tables = version .get_compaction_group_levels(StaticCompactionGroupId::StateDefault.into()) .levels - .last() - .unwrap() - .table_infos - .first() - .unwrap(); - let table = storage - .sstable_store() - .sstable(output_table, &mut StoreLocalStatistic::default()) - .await - .unwrap(); - let target_table_size = storage.storage_opts().sstable_size_mb * (1 << 20); - - assert!( - table.value().meta.estimated_size > target_table_size, - "table.meta.estimated_size {} <= target_table_size {}", - table.value().meta.estimated_size, - target_table_size - ); + .iter() + .flat_map(|level| level.table_infos.clone()) + .collect_vec(); + for output_table in &output_tables { + let table = storage + .sstable_store() + .sstable(output_table, &mut StoreLocalStatistic::default()) + .await + .unwrap(); + let target_table_size = storage.storage_opts().sstable_size_mb * (1 << 20); + assert!( + table.value().meta.estimated_size > target_table_size, + "table.meta.estimated_size {} <= target_table_size {}", + table.value().meta.estimated_size, + target_table_size + ); + } // 5. storage get back the correct kv after compaction storage.wait_version(version).await; From 888f2dda736a99f294487c5935b81fbbfdd59d7b Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Thu, 14 Sep 2023 14:32:59 +0800 Subject: [PATCH 10/14] fix: panic when dumping memory profile (#12276) --- Cargo.lock | 6 +++--- Cargo.toml | 4 ++++ src/batch/Cargo.toml | 2 +- src/cmd/Cargo.toml | 6 ++---- src/cmd_all/Cargo.toml | 12 +++++++----- src/compute/Cargo.toml | 2 +- src/compute/src/memory_management/policy.rs | 5 ++--- src/tests/simulation/Cargo.toml | 4 +--- 8 files changed, 21 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f062a0bb9c98a..e9edb2360ef31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8927,7 +8927,7 @@ dependencies = [ [[package]] name = "tikv-jemalloc-ctl" version = "0.5.4" -source = "git+https://github.com/risingwavelabs/jemallocator.git?rev=b7f9f3#b7f9f34664dcfea190e64bef64587e23f9f2710c" +source = "git+https://github.com/risingwavelabs/jemallocator.git?rev=64a2d9#64a2d988d687a94cd859855e19241cd8b0705466" dependencies = [ "libc", "paste", @@ -8937,7 +8937,7 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" version = "0.5.4+5.3.0-patched" -source = "git+https://github.com/risingwavelabs/jemallocator.git?rev=b7f9f3#b7f9f34664dcfea190e64bef64587e23f9f2710c" +source = "git+https://github.com/risingwavelabs/jemallocator.git?rev=64a2d9#64a2d988d687a94cd859855e19241cd8b0705466" dependencies = [ "cc", "libc", @@ -8946,7 +8946,7 @@ dependencies = [ [[package]] name = "tikv-jemallocator" version = "0.5.4" -source = "git+https://github.com/risingwavelabs/jemallocator.git?rev=b7f9f3#b7f9f34664dcfea190e64bef64587e23f9f2710c" +source = "git+https://github.com/risingwavelabs/jemallocator.git?rev=64a2d9#64a2d988d687a94cd859855e19241cd8b0705466" dependencies = [ "libc", "tikv-jemalloc-sys", diff --git a/Cargo.toml b/Cargo.toml index e33c09ddc5734..332bc5f8e7acc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -112,6 +112,10 @@ arrow-schema = "46" arrow-buffer = "46" arrow-flight = "46" arrow-select = "46" +tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [ + "profiling", + "stats", +], rev = "64a2d9" } risingwave_backup = { path = "./src/storage/backup" } risingwave_batch = { path = "./src/batch" } diff --git a/src/batch/Cargo.toml b/src/batch/Cargo.toml index 5f90151a400ab..35ede881bc47f 100644 --- a/src/batch/Cargo.toml +++ b/src/batch/Cargo.toml @@ -67,7 +67,7 @@ rand = "0.8" tempfile = "3" [target.'cfg(unix)'.dev-dependencies] -tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "b7f9f3" } +tikv-jemallocator = { workspace = true } [[bench]] name = "filter" diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index 46a33654d3545..894d19c9f969a 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -46,11 +46,9 @@ workspace-hack = { path = "../workspace-hack" } task_stats_alloc = { path = "../utils/task_stats_alloc" } [target.'cfg(unix)'.dependencies] -tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [ - "profiling", - "stats", +tikv-jemallocator = { workspace = true, features = [ "unprefixed_malloc_on_supported_platforms", -], rev = "b7f9f3" } +] } [[bin]] name = "frontend" diff --git a/src/cmd_all/Cargo.toml b/src/cmd_all/Cargo.toml index b6907cdebaeff..f22f0b59ea2db 100644 --- a/src/cmd_all/Cargo.toml +++ b/src/cmd_all/Cargo.toml @@ -53,17 +53,19 @@ workspace-hack = { path = "../workspace-hack" } expect-test = "1" [build-dependencies] -vergen = { version = "8", default-features = false, features = ["build", "git", "gitcl"] } +vergen = { version = "8", default-features = false, features = [ + "build", + "git", + "gitcl", +] } [target.'cfg(enable_task_local_alloc)'.dependencies] task_stats_alloc = { path = "../utils/task_stats_alloc" } [target.'cfg(unix)'.dependencies] -tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [ - "profiling", - "stats", +tikv-jemallocator = { workspace = true, features = [ "unprefixed_malloc_on_supported_platforms", -], rev = "b7f9f3" } +] } [[bin]] name = "risingwave" diff --git a/src/compute/Cargo.toml b/src/compute/Cargo.toml index 7774a2d917731..e0e2a4ab0528b 100644 --- a/src/compute/Cargo.toml +++ b/src/compute/Cargo.toml @@ -55,7 +55,7 @@ tower = { version = "0.4", features = ["util", "load-shed"] } tracing = "0.1" [target.'cfg(target_os = "linux")'.dependencies] -tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "b7f9f3" } +tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" } [target.'cfg(not(madsim))'.dependencies] workspace-hack = { path = "../workspace-hack" } diff --git a/src/compute/src/memory_management/policy.rs b/src/compute/src/memory_management/policy.rs index 5c3602a46fe68..005edb813f823 100644 --- a/src/compute/src/memory_management/policy.rs +++ b/src/compute/src/memory_management/policy.rs @@ -128,9 +128,8 @@ impl JemallocMemoryControl { .unwrap() .to_string() } else { - let prof_prefix_mib = jemalloc_prof::prefix::mib().unwrap(); - let prof_prefix = prof_prefix_mib.read().unwrap(); - let mut file_path = prof_prefix.to_string_lossy().to_string(); + let prof_prefix = jemalloc_opt::prof_prefix::read().unwrap(); + let mut file_path = prof_prefix.to_str().unwrap().to_string(); file_path.push_str(&file_name); file_path }; diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml index 594f06b3b61c3..0e7b4e0d8be8f 100644 --- a/src/tests/simulation/Cargo.toml +++ b/src/tests/simulation/Cargo.toml @@ -46,9 +46,7 @@ serde_derive = "1.0.188" serde_json = "1.0.106" sqllogictest = "0.15.3" tempfile = "3" -tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [ - "profiling", -], rev = "b7f9f3" } +tikv-jemallocator = { workspace = true } tokio = { version = "0.2.23", package = "madsim-tokio" } tokio-postgres = "0.7" tracing = "0.1" From 5ffd58db7d018989b28ec492e7c85c3625dd1d23 Mon Sep 17 00:00:00 2001 From: Dylan Date: Thu, 14 Sep 2023 14:35:03 +0800 Subject: [PATCH 11/14] refactor(connector): replace validate source rpc with jni (#12270) --- .../source/JniSourceValidateHandler.java | 49 +++++++++++++++++ .../source/SourceValidateHandler.java | 6 +-- .../src/source/cdc/enumerator/mod.rs | 53 ++++++++++++++----- src/jni_core/src/lib.rs | 4 +- 4 files changed, 94 insertions(+), 18 deletions(-) create mode 100644 java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/JniSourceValidateHandler.java diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/JniSourceValidateHandler.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/JniSourceValidateHandler.java new file mode 100644 index 0000000000000..a25bf0dee065a --- /dev/null +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/JniSourceValidateHandler.java @@ -0,0 +1,49 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.risingwave.connector.source; + +import static com.risingwave.connector.source.SourceValidateHandler.validateResponse; +import static com.risingwave.connector.source.SourceValidateHandler.validateSource; + +import com.risingwave.proto.ConnectorServiceProto; +import io.grpc.StatusRuntimeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JniSourceValidateHandler { + static final Logger LOG = LoggerFactory.getLogger(JniSourceValidateHandler.class); + + public static byte[] validate(byte[] validateSourceRequestBytes) + throws com.google.protobuf.InvalidProtocolBufferException { + try { + var request = + ConnectorServiceProto.ValidateSourceRequest.parseFrom( + validateSourceRequestBytes); + + // For jni.rs + java.lang.Thread.currentThread() + .setContextClassLoader(java.lang.ClassLoader.getSystemClassLoader()); + validateSource(request); + // validate pass + return ConnectorServiceProto.ValidateSourceResponse.newBuilder().build().toByteArray(); + } catch (StatusRuntimeException e) { + LOG.warn("Source validation failed", e); + return validateResponse(e.getMessage()).toByteArray(); + } catch (Exception e) { + LOG.error("Internal error on source validation", e); + return validateResponse("Internal error: " + e.getMessage()).toByteArray(); + } + } +} diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java index 2611d1cca676b..38fd23ae1c1aa 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java @@ -56,7 +56,7 @@ public void handle(ConnectorServiceProto.ValidateSourceRequest request) { } } - private ConnectorServiceProto.ValidateSourceResponse validateResponse(String message) { + public static ConnectorServiceProto.ValidateSourceResponse validateResponse(String message) { return ConnectorServiceProto.ValidateSourceResponse.newBuilder() .setError( ConnectorServiceProto.ValidationError.newBuilder() @@ -65,14 +65,14 @@ private ConnectorServiceProto.ValidateSourceResponse validateResponse(String mes .build(); } - private void ensurePropNotNull(Map props, String name) { + public static void ensurePropNotNull(Map props, String name) { if (!props.containsKey(name)) { throw ValidatorUtils.invalidArgument( String.format("'%s' not found, please check the WITH properties", name)); } } - private void validateSource(ConnectorServiceProto.ValidateSourceRequest request) + public static void validateSource(ConnectorServiceProto.ValidateSourceRequest request) throws Exception { var props = request.getPropertiesMap(); diff --git a/src/connector/src/source/cdc/enumerator/mod.rs b/src/connector/src/source/cdc/enumerator/mod.rs index 1c689026e568b..0b98005b270f4 100644 --- a/src/connector/src/source/cdc/enumerator/mod.rs +++ b/src/connector/src/source/cdc/enumerator/mod.rs @@ -13,13 +13,17 @@ // limitations under the License. use std::marker::PhantomData; +use std::ops::Deref; use std::str::FromStr; use anyhow::anyhow; use async_trait::async_trait; use itertools::Itertools; +use jni::objects::{JByteArray, JValue, JValueOwned}; +use prost::Message; use risingwave_common::util::addr::HostAddr; -use risingwave_pb::connector_service::SourceType; +use risingwave_jni_core::jvm_runtime::JVM; +use risingwave_pb::connector_service::{SourceType, ValidateSourceRequest, ValidateSourceResponse}; use crate::source::cdc::{ CdcProperties, CdcSourceTypeTrait, CdcSplitBase, Citus, DebeziumCdcSplit, MySqlCdcSplit, Mysql, @@ -49,10 +53,6 @@ where props: CdcProperties, context: SourceEnumeratorContextRef, ) -> anyhow::Result { - let connector_client = context.connector_client.clone().ok_or_else(|| { - anyhow!("connector node endpoint not specified or unable to connect to connector node") - })?; - let server_addrs = props .props .get(DATABASE_SERVERS_KEY) @@ -69,15 +69,42 @@ where SourceType::from(T::source_type()) ); + let mut env = JVM.as_ref()?.attach_current_thread()?; + + let validate_source_request = ValidateSourceRequest { + source_id: context.info.source_id as u64, + source_type: props.get_source_type_pb() as _, + properties: props.props, + table_schema: Some(props.table_schema), + }; + + let validate_source_request_bytes = + env.byte_array_from_slice(&Message::encode_to_vec(&validate_source_request))?; + // validate connector properties - connector_client - .validate_source_properties( - context.info.source_id as u64, - props.get_source_type_pb(), - props.props, - Some(props.table_schema), - ) - .await?; + let response = env.call_static_method( + "com/risingwave/connector/source/JniSourceValidateHandler", + "validate", + "([B)[B", + &[JValue::Object(&validate_source_request_bytes)], + )?; + + let validate_source_response_bytes = match response { + JValueOwned::Object(o) => unsafe { JByteArray::from_raw(o.into_raw()) }, + _ => unreachable!(), + }; + + let validate_source_response: ValidateSourceResponse = Message::decode( + risingwave_jni_core::to_guarded_slice(&validate_source_response_bytes, &mut env)? + .deref(), + )?; + + validate_source_response.error.map_or(Ok(()), |err| { + Err(anyhow!(format!( + "source cannot pass validation: {}", + err.error_message + ))) + })?; tracing::debug!("validate cdc source properties success"); Ok(Self { diff --git a/src/jni_core/src/lib.rs b/src/jni_core/src/lib.rs index a8f0e5a683e35..6fa6f2f10e991 100644 --- a/src/jni_core/src/lib.rs +++ b/src/jni_core/src/lib.rs @@ -57,7 +57,7 @@ pub type GetEventStreamJniSender = Sender; static RUNTIME: LazyLock = LazyLock::new(|| tokio::runtime::Runtime::new().unwrap()); #[derive(Error, Debug)] -enum BindingError { +pub enum BindingError { #[error("JniError {error}")] Jni { #[from] @@ -89,7 +89,7 @@ enum BindingError { type Result = std::result::Result; -fn to_guarded_slice<'array, 'env>( +pub fn to_guarded_slice<'array, 'env>( array: &'array JByteArray<'env>, env: &'array mut JNIEnv<'env>, ) -> Result> { From 4525e67bf73a98304350c46c4bea72786074a893 Mon Sep 17 00:00:00 2001 From: Noel Kwan <47273164+kwannoel@users.noreply.github.com> Date: Thu, 14 Sep 2023 14:38:03 +0800 Subject: [PATCH 12/14] feat(stream): support source throttling (#12295) --- e2e_test/streaming/rate_limit.slt | 2 +- proto/stream_plan.proto | 2 + src/common/src/session_config/mod.rs | 6 +- .../src/optimizer/plan_node/stream.rs | 3 + .../src/optimizer/plan_node/stream_source.rs | 7 + src/stream/src/executor/flow_control.rs | 12 +- src/stream/src/from_proto/source.rs | 274 +++++++++--------- 7 files changed, 164 insertions(+), 142 deletions(-) diff --git a/e2e_test/streaming/rate_limit.slt b/e2e_test/streaming/rate_limit.slt index 95e87b9fd681c..539a9736754cf 100644 --- a/e2e_test/streaming/rate_limit.slt +++ b/e2e_test/streaming/rate_limit.slt @@ -2,7 +2,7 @@ statement ok CREATE TABLE t1(v1 int, v2 int); statement ok -SET RW_STREAMING_RATE_LIMIT TO 10000; +SET STREAMING_RATE_LIMIT TO 10000; statement ok CREATE MATERIALIZED VIEW m AS SELECT * FROM t1; diff --git a/proto/stream_plan.proto b/proto/stream_plan.proto index de8cfe14746cf..bd54c0f924900 100644 --- a/proto/stream_plan.proto +++ b/proto/stream_plan.proto @@ -150,6 +150,8 @@ message StreamSource { map properties = 6; catalog.StreamSourceInfo info = 7; string source_name = 8; + // Streaming rate limit + optional uint32 rate_limit = 9; } // The executor only for receiving barrier from the meta service. It always resides in the leaves diff --git a/src/common/src/session_config/mod.rs b/src/common/src/session_config/mod.rs index 7e8ad82665bab..2643f9e0643d4 100644 --- a/src/common/src/session_config/mod.rs +++ b/src/common/src/session_config/mod.rs @@ -71,7 +71,7 @@ const CONFIG_KEYS: [&str; 37] = [ "LOCK_TIMEOUT", "ROW_SECURITY", "STANDARD_CONFORMING_STRINGS", - "RW_STREAMING_RATE_LIMIT", + "STREAMING_RATE_LIMIT", "CDC_BACKFILL", "RW_STREAMING_OVER_WINDOW_CACHE_POLICY", ]; @@ -112,7 +112,7 @@ const STATEMENT_TIMEOUT: usize = 30; const LOCK_TIMEOUT: usize = 31; const ROW_SECURITY: usize = 32; const STANDARD_CONFORMING_STRINGS: usize = 33; -const RW_STREAMING_RATE_LIMIT: usize = 34; +const STREAMING_RATE_LIMIT: usize = 34; const CDC_BACKFILL: usize = 35; const STREAMING_OVER_WINDOW_CACHE_POLICY: usize = 36; @@ -337,7 +337,7 @@ type StatementTimeout = ConfigI32; type LockTimeout = ConfigI32; type RowSecurity = ConfigBool; type StandardConformingStrings = ConfigString; -type StreamingRateLimit = ConfigU64; +type StreamingRateLimit = ConfigU64; type CdcBackfill = ConfigBool; /// Report status or notice to caller. diff --git a/src/frontend/src/optimizer/plan_node/stream.rs b/src/frontend/src/optimizer/plan_node/stream.rs index 3fa4f5aa026e9..828f509351b37 100644 --- a/src/frontend/src/optimizer/plan_node/stream.rs +++ b/src/frontend/src/optimizer/plan_node/stream.rs @@ -735,6 +735,8 @@ pub fn to_stream_prost_body( log_store_type: SinkLogStoreType::InMemoryLogStore as i32, }), Node::Source(me) => { + // TODO(kwannoel): Is branch used, seems to be a duplicate of stream_source? + let rate_limit = me.ctx().session_ctx().config().get_streaming_rate_limit(); let me = &me.core.catalog; let source_inner = me.as_ref().map(|me| StreamSource { source_id: me.id, @@ -748,6 +750,7 @@ pub fn to_stream_prost_body( row_id_index: me.row_id_index.map(|index| index as _), columns: me.columns.iter().map(|c| c.to_protobuf()).collect(), properties: me.properties.clone().into_iter().collect(), + rate_limit, }); PbNodeBody::Source(SourceNode { source_inner }) } diff --git a/src/frontend/src/optimizer/plan_node/stream_source.rs b/src/frontend/src/optimizer/plan_node/stream_source.rs index 3172c4c06f80c..f188889189464 100644 --- a/src/frontend/src/optimizer/plan_node/stream_source.rs +++ b/src/frontend/src/optimizer/plan_node/stream_source.rs @@ -23,6 +23,7 @@ use risingwave_pb::stream_plan::{PbStreamSource, SourceNode}; use super::utils::{childless_record, Distill}; use super::{generic, ExprRewritable, PlanBase, StreamNode}; use crate::catalog::source_catalog::SourceCatalog; +use crate::optimizer::plan_node::generic::GenericPlanRef; use crate::optimizer::plan_node::utils::column_names_pretty; use crate::optimizer::property::Distribution; use crate::stream_fragmenter::BuildFragmentGraphState; @@ -86,6 +87,12 @@ impl StreamNode for StreamSource { .map(|c| c.to_protobuf()) .collect_vec(), properties: source_catalog.properties.clone().into_iter().collect(), + rate_limit: self + .base + .ctx() + .session_ctx() + .config() + .get_streaming_rate_limit(), }); PbNodeBody::Source(SourceNode { source_inner }) } diff --git a/src/stream/src/executor/flow_control.rs b/src/stream/src/executor/flow_control.rs index 1790f212566b7..45e04717e2a9d 100644 --- a/src/stream/src/executor/flow_control.rs +++ b/src/stream/src/executor/flow_control.rs @@ -16,7 +16,7 @@ use std::fmt::{Debug, Formatter}; use std::num::NonZeroU32; use governor::clock::MonotonicClock; -use governor::{Quota, RateLimiter}; +use governor::{InsufficientCapacity, Quota, RateLimiter}; use risingwave_common::catalog::Schema; use super::*; @@ -58,10 +58,12 @@ impl FlowControlExecutor { let result = rate_limiter .until_n_ready(NonZeroU32::new(chunk.cardinality() as u32).unwrap()) .await; - assert!( - result.is_ok(), - "the capacity of rate_limiter must be larger than the cardinality of chunk" - ); + if let Err(InsufficientCapacity(n)) = result { + tracing::error!( + "Rate Limit {} smaller than chunk cardinality {n}", + self.rate_limit, + ); + } } yield Message::Chunk(chunk); } diff --git a/src/stream/src/from_proto/source.rs b/src/stream/src/from_proto/source.rs index f8487b98dc6a8..77bbcc53e69c5 100644 --- a/src/stream/src/from_proto/source.rs +++ b/src/stream/src/from_proto/source.rs @@ -27,7 +27,7 @@ use crate::executor::external::ExternalStorageTable; use crate::executor::source::StreamSourceCore; use crate::executor::source_executor::SourceExecutor; use crate::executor::state_table_handler::SourceStateTableHandler; -use crate::executor::{CdcBackfillExecutor, FsSourceExecutor}; +use crate::executor::{CdcBackfillExecutor, FlowControlExecutor, FsSourceExecutor}; const FS_CONNECTORS: &[&str] = &["s3"]; pub struct SourceExecutorBuilder; @@ -50,150 +50,157 @@ impl ExecutorBuilder for SourceExecutorBuilder { let system_params = params.env.system_params_manager_ref().get_params(); if let Some(source) = &node.source_inner { - let source_id = TableId::new(source.source_id); - let source_name = source.source_name.clone(); - let source_info = source.get_info()?; - - let source_desc_builder = SourceDescBuilder::new( - source.columns.clone(), - params.env.source_metrics(), - source.row_id_index.map(|x| x as _), - source.properties.clone(), - source_info.clone(), - params.env.connector_params(), - params.env.config().developer.connector_message_buffer_size, - // `pk_indices` is used to ensure that a message will be skipped instead of parsed - // with null pk when the pk column is missing. - // - // Currently pk_indices for source is always empty since pk information is not - // passed via `StreamSource` so null pk may be emitted to downstream. - // - // TODO: use the correct information to fill in pk_dicies. - // We should consdier add back the "pk_column_ids" field removed by #8841 in - // StreamSource - params.pk_indices.clone(), - ); - - let source_ctrl_opts = SourceCtrlOpts { - chunk_size: params.env.config().developer.chunk_size, - }; - - let column_ids: Vec<_> = source - .columns - .iter() - .map(|column| ColumnId::from(column.get_column_desc().unwrap().column_id)) - .collect(); - let fields = source - .columns - .iter() - .map(|prost| { - let column_desc = prost.column_desc.as_ref().unwrap(); - let data_type = DataType::from(column_desc.column_type.as_ref().unwrap()); - let name = column_desc.name.clone(); - Field::with_name(data_type, name) - }) - .collect(); - let schema = Schema::new(fields); - - let state_table_handler = SourceStateTableHandler::from_table_catalog( - source.state_table.as_ref().unwrap(), - store.clone(), - ) - .await; - let stream_source_core = StreamSourceCore::new( - source_id, - source_name, - column_ids, - source_desc_builder, - state_table_handler, - ); - - let connector = source - .properties - .get("connector") - .map(|c| c.to_ascii_lowercase()) - .unwrap_or_default(); - let is_fs_connector = FS_CONNECTORS.contains(&connector.as_str()); - - if is_fs_connector { - Ok(Box::new(FsSourceExecutor::new( - params.actor_context, - schema, - params.pk_indices, - stream_source_core, - params.executor_stats, - barrier_receiver, - system_params, - params.executor_id, - source_ctrl_opts, - )?)) - } else { - let source_exec = SourceExecutor::new( - params.actor_context.clone(), - schema.clone(), - params.pk_indices.clone(), - Some(stream_source_core), - params.executor_stats.clone(), - barrier_receiver, - system_params, - params.executor_id, - source_ctrl_opts.clone(), + let executor = { + let source_id = TableId::new(source.source_id); + let source_name = source.source_name.clone(); + let source_info = source.get_info()?; + + let source_desc_builder = SourceDescBuilder::new( + source.columns.clone(), + params.env.source_metrics(), + source.row_id_index.map(|x| x as _), + source.properties.clone(), + source_info.clone(), params.env.connector_params(), + params.env.config().developer.connector_message_buffer_size, + // `pk_indices` is used to ensure that a message will be skipped instead of parsed + // with null pk when the pk column is missing. + // + // Currently pk_indices for source is always empty since pk information is not + // passed via `StreamSource` so null pk may be emitted to downstream. + // + // TODO: use the correct information to fill in pk_dicies. + // We should consdier add back the "pk_column_ids" field removed by #8841 in + // StreamSource + params.pk_indices.clone(), ); - let table_type = ExternalTableType::from_properties(&source.properties); - if table_type.can_backfill() && let Some(table_desc) = source_info.upstream_table.clone() { - let upstream_table_name = SchemaTableName::from_properties(&source.properties); - let pk_indices = table_desc - .pk - .iter() - .map(|k| k.column_index as usize) - .collect_vec(); - - let order_types = table_desc - .pk - .iter() - .map(|desc| OrderType::from_protobuf(desc.get_order_type().unwrap())) - .collect_vec(); - - let table_reader = table_type.create_table_reader(source.properties.clone(), schema.clone())?; - let external_table = ExternalStorageTable::new( - TableId::new(source.source_id), - upstream_table_name, - table_reader, - schema.clone(), - order_types, - pk_indices.clone(), - (0..table_desc.columns.len()).collect_vec(), - ); + let source_ctrl_opts = SourceCtrlOpts { + chunk_size: params.env.config().developer.chunk_size, + }; + + let column_ids: Vec<_> = source + .columns + .iter() + .map(|column| ColumnId::from(column.get_column_desc().unwrap().column_id)) + .collect(); + let fields = source + .columns + .iter() + .map(|prost| { + let column_desc = prost.column_desc.as_ref().unwrap(); + let data_type = DataType::from(column_desc.column_type.as_ref().unwrap()); + let name = column_desc.name.clone(); + Field::with_name(data_type, name) + }) + .collect(); + let schema = Schema::new(fields); + + let state_table_handler = SourceStateTableHandler::from_table_catalog( + source.state_table.as_ref().unwrap(), + store.clone(), + ) + .await; + let stream_source_core = StreamSourceCore::new( + source_id, + source_name, + column_ids, + source_desc_builder, + state_table_handler, + ); - // use the state table from source to store the backfill state (may refactor in future) - let source_state_handler = SourceStateTableHandler::from_table_catalog( - source.state_table.as_ref().unwrap(), - store.clone(), - ).await; - let cdc_backfill = CdcBackfillExecutor::new( + let connector = source + .properties + .get("connector") + .map(|c| c.to_ascii_lowercase()) + .unwrap_or_default(); + let is_fs_connector = FS_CONNECTORS.contains(&connector.as_str()); + + if is_fs_connector { + FsSourceExecutor::new( + params.actor_context, + schema, + params.pk_indices, + stream_source_core, + params.executor_stats, + barrier_receiver, + system_params, + params.executor_id, + source_ctrl_opts, + )? + .boxed() + } else { + let source_exec = SourceExecutor::new( params.actor_context.clone(), - external_table, - Box::new(source_exec), - (0..source.columns.len()).collect_vec(), // eliminate the last column (_rw_offset) - None, schema.clone(), - pk_indices, - params.executor_stats, - source_state_handler, - source_ctrl_opts.chunk_size + params.pk_indices.clone(), + Some(stream_source_core), + params.executor_stats.clone(), + barrier_receiver, + system_params, + params.executor_id, + source_ctrl_opts.clone(), + params.env.connector_params(), ); - Ok(Box::new(cdc_backfill)) - } else { - Ok(Box::new(source_exec)) + let table_type = ExternalTableType::from_properties(&source.properties); + if table_type.can_backfill() && let Some(table_desc) = source_info.upstream_table.clone() { + let upstream_table_name = SchemaTableName::from_properties(&source.properties); + let pk_indices = table_desc + .pk + .iter() + .map(|k| k.column_index as usize) + .collect_vec(); + + let order_types = table_desc + .pk + .iter() + .map(|desc| OrderType::from_protobuf(desc.get_order_type().unwrap())) + .collect_vec(); + + let table_reader = table_type.create_table_reader(source.properties.clone(), schema.clone())?; + let external_table = ExternalStorageTable::new( + TableId::new(source.source_id), + upstream_table_name, + table_reader, + schema.clone(), + order_types, + pk_indices.clone(), + (0..table_desc.columns.len()).collect_vec(), + ); + + // use the state table from source to store the backfill state (may refactor in future) + let source_state_handler = SourceStateTableHandler::from_table_catalog( + source.state_table.as_ref().unwrap(), + store.clone(), + ).await; + let cdc_backfill = CdcBackfillExecutor::new( + params.actor_context.clone(), + external_table, + Box::new(source_exec), + (0..source.columns.len()).collect_vec(), // eliminate the last column (_rw_offset) + None, + schema.clone(), + pk_indices, + params.executor_stats, + source_state_handler, + source_ctrl_opts.chunk_size + ); + cdc_backfill.boxed() + } else { + source_exec.boxed() + } } + }; + if let Ok(rate_limit) = source.get_rate_limit() { + Ok(FlowControlExecutor::new(executor, *rate_limit).boxed()) + } else { + Ok(executor) } } else { // If there is no external stream source, then no data should be persisted. We pass a // `PanicStateStore` type here for indication. - Ok(Box::new(SourceExecutor::::new( + Ok(SourceExecutor::::new( params.actor_context, params.schema, params.pk_indices, @@ -205,7 +212,8 @@ impl ExecutorBuilder for SourceExecutorBuilder { // we don't expect any data in, so no need to set chunk_sizes SourceCtrlOpts::default(), params.env.connector_params(), - ))) + ) + .boxed()) } } } From 9814af8e4f86ada5ef051583a24acd5e56e30357 Mon Sep 17 00:00:00 2001 From: Runji Wang Date: Thu, 14 Sep 2023 14:45:14 +0800 Subject: [PATCH 13/14] feat(expr): add `pg_sleep` function (#12294) Signed-off-by: Runji Wang --- proto/expr.proto | 3 ++ src/expr/macro/src/gen.rs | 6 ++- src/expr/macro/src/lib.rs | 34 +++++++++++++--- src/expr/macro/src/parse.rs | 10 +++-- src/expr/macro/src/types.rs | 4 ++ src/expr/src/vector_op/delay.rs | 52 ++++++++++++++++++++++++ src/expr/src/vector_op/mod.rs | 1 + src/expr/src/vector_op/proctime.rs | 2 +- src/frontend/src/binder/expr/function.rs | 6 ++- src/frontend/src/expr/pure.rs | 6 ++- 10 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 src/expr/src/vector_op/delay.rs diff --git a/proto/expr.proto b/proto/expr.proto index c4779deafacb4..e81036f92190d 100644 --- a/proto/expr.proto +++ b/proto/expr.proto @@ -220,6 +220,9 @@ message ExprNode { VNODE = 1101; // Non-deterministic functions PROCTIME = 2023; + PG_SLEEP = 2024; + PG_SLEEP_FOR = 2025; + PG_SLEEP_UNTIL = 2026; } Type function_type = 1; data.DataType return_type = 3; diff --git a/src/expr/macro/src/gen.rs b/src/expr/macro/src/gen.rs index 6b7d33ac74680..9e8b4d9d3c1b2 100644 --- a/src/expr/macro/src/gen.rs +++ b/src/expr/macro/src/gen.rs @@ -235,11 +235,13 @@ impl FunctionAttr { true => quote! { &mut writer, }, false => quote! {}, }; + let await_ = user_fn.async_.then(|| quote! { .await }); // call the user defined function // inputs: [ Option ] - let mut output = - quote! { #fn_name #generic(#(#non_prebuilt_inputs,)* #prebuilt_arg #context #writer) }; + let mut output = quote! { #fn_name #generic(#(#non_prebuilt_inputs,)* #prebuilt_arg #context #writer) #await_ }; output = match user_fn.return_type_kind { + // XXX: we don't support void type yet. return null::int for now. + _ if self.ret == "void" => quote! { { #output; Option::::None } }, ReturnTypeKind::T => quote! { Some(#output) }, ReturnTypeKind::Option => output, ReturnTypeKind::Result => quote! { Some(#output?) }, diff --git a/src/expr/macro/src/lib.rs b/src/expr/macro/src/lib.rs index 82e7c8014220a..4d8c48ca9ccac 100644 --- a/src/expr/macro/src/lib.rs +++ b/src/expr/macro/src/lib.rs @@ -30,18 +30,19 @@ mod utils; /// /// # Table of Contents /// -/// - [Function Signature](#function-signature) +/// - [SQL Function Signature](#sql-function-signature) /// - [Multiple Function Definitions](#multiple-function-definitions) /// - [Type Expansion](#type-expansion) /// - [Automatic Type Inference](#automatic-type-inference) /// - [Custom Type Inference Function](#custom-type-inference-function) -/// - [Rust Function Requirements](#rust-function-requirements) +/// - [Rust Function Signature](#rust-function-signature) /// - [Nullable Arguments](#nullable-arguments) /// - [Return Value](#return-value) /// - [Optimization](#optimization) /// - [Functions Returning Strings](#functions-returning-strings) /// - [Preprocessing Constant Arguments](#preprocessing-constant-arguments) /// - [Context](#context) +/// - [Async Function](#async-function) /// - [Table Function](#table-function) /// - [Registration and Invocation](#registration-and-invocation) /// - [Appendix: Type Matrix](#appendix-type-matrix) @@ -55,13 +56,13 @@ mod utils; /// } /// ``` /// -/// # Function Signature +/// # SQL Function Signature /// /// Each function must have a signature, specified in the `function("...")` part of the macro /// invocation. The signature follows this pattern: /// /// ```text -/// name([arg_types],*) -> [setof] return_type +/// name ( [arg_types],* ) [ -> [setof] return_type ] /// ``` /// /// Where `name` is the function name, which must match the function name defined in `prost`. @@ -73,6 +74,9 @@ mod utils; /// function (table function), meaning it can return multiple values instead of just one. For more /// details, see the section on table functions. /// +/// If no return type is specified, the function returns `void`. However, the void type is not +/// supported in our type system, so it now returns a null value of type int. +/// /// ## Multiple Function Definitions /// /// Multiple `#[function]` macros can be applied to a single generic Rust function to define @@ -154,7 +158,7 @@ mod utils; /// /// This type inference function will be invoked at the frontend. /// -/// # Rust Function Requirements +/// # Rust Function Signature /// /// The `#[function]` macro can handle various types of Rust functions. /// @@ -277,6 +281,19 @@ mod utils; /// } /// ``` /// +/// ## Async Function +/// +/// Functions can be asynchronous. +/// +/// ```ignore +/// #[function("pg_sleep(float64)")] +/// async fn pg_sleep(second: F64) { +/// tokio::time::sleep(Duration::from_secs_f64(second.0)).await; +/// } +/// ``` +/// +/// Asynchronous functions will be evaluated on rows sequentially. +/// /// # Table Function /// /// A table function is a special kind of function that can return multiple values instead of just @@ -460,6 +477,8 @@ struct FunctionAttr { prebuild: Option, /// Type inference function. type_infer: Option, + /// Whether the function is volatile. + volatile: bool, /// Whether the function is deprecated. deprecated: bool, } @@ -469,6 +488,8 @@ struct FunctionAttr { struct UserFunctionAttr { /// Function name name: String, + /// Whether the function is async. + async_: bool, /// Whether contains argument `&Context`. context: bool, /// The last argument type is `&mut dyn Write`. @@ -556,7 +577,8 @@ impl FunctionAttr { impl UserFunctionAttr { /// Returns true if the function is like `fn(T1, T2, .., Tn) -> T`. fn is_pure(&self) -> bool { - !self.write + !self.async_ + && !self.write && !self.context && !self.arg_option && self.return_type_kind == ReturnTypeKind::T diff --git a/src/expr/macro/src/parse.rs b/src/expr/macro/src/parse.rs index dfcd08aafd3f9..be7a6c86df624 100644 --- a/src/expr/macro/src/parse.rs +++ b/src/expr/macro/src/parse.rs @@ -28,9 +28,10 @@ impl Parse for FunctionAttr { let sig = input.parse::()?; let sig_str = sig.value(); - let (name_args, ret) = sig_str - .split_once("->") - .ok_or_else(|| Error::new_spanned(&sig, "expected '->'"))?; + let (name_args, ret) = match sig_str.split_once("->") { + Some((name_args, ret)) => (name_args, ret), + None => (sig_str.as_str(), "void"), + }; let (name, args) = name_args .split_once('(') .ok_or_else(|| Error::new_spanned(&sig, "expected '('"))?; @@ -74,6 +75,8 @@ impl Parse for FunctionAttr { parsed.prebuild = Some(get_value()?); } else if meta.path().is_ident("type_infer") { parsed.type_infer = Some(get_value()?); + } else if meta.path().is_ident("volatile") { + parsed.volatile = true; } else if meta.path().is_ident("deprecated") { parsed.deprecated = true; } else if meta.path().is_ident("append_only") { @@ -113,6 +116,7 @@ impl From<&syn::Signature> for UserFunctionAttr { }; UserFunctionAttr { name: sig.ident.to_string(), + async_: sig.asyncness.is_some(), write: sig.inputs.iter().any(arg_is_write), context: sig.inputs.iter().any(arg_is_context), retract: last_arg_is_retract(sig), diff --git a/src/expr/macro/src/types.rs b/src/expr/macro/src/types.rs index b9e4e9b6ccef5..53f224b79a773 100644 --- a/src/expr/macro/src/types.rs +++ b/src/expr/macro/src/types.rs @@ -81,6 +81,10 @@ fn lookup_matrix(mut ty: &str, idx: usize) -> &str { ty = "list"; } else if ty.starts_with("struct") { ty = "struct"; + } else if ty == "void" { + // XXX: we don't support void type yet. + // replace it with int32 for now. + ty = "int32"; } let s = TYPE_MATRIX.trim().lines().find_map(|line| { let mut parts = line.split_whitespace(); diff --git a/src/expr/src/vector_op/delay.rs b/src/expr/src/vector_op/delay.rs new file mode 100644 index 0000000000000..b1661e56e75b4 --- /dev/null +++ b/src/expr/src/vector_op/delay.rs @@ -0,0 +1,52 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +use risingwave_common::types::{Interval, F64}; +use risingwave_expr_macro::function; + +/// Makes the current session's process sleep until the given number of seconds have elapsed. +/// +/// ```slt +/// query I +/// SELECT pg_sleep(1.5); +/// ---- +/// NULL +/// ``` +#[function("pg_sleep(float64)", volatile)] +async fn pg_sleep(second: F64) { + tokio::time::sleep(Duration::from_secs_f64(second.0)).await; +} + +/// Makes the current session's process sleep until the given interval has elapsed. +/// +/// ```slt +/// query I +/// SELECT pg_sleep_for('1 second'); +/// ---- +/// NULL +/// ``` +#[function("pg_sleep_for(interval)", volatile)] +async fn pg_sleep_for(interval: Interval) { + // we only use the microsecond part of the interval + let usecs = if interval.is_positive() { + interval.usecs() as u64 + } else { + // return if the interval is not positive + return; + }; + let duration = Duration::from_micros(usecs); + tokio::time::sleep(duration).await; +} diff --git a/src/expr/src/vector_op/mod.rs b/src/expr/src/vector_op/mod.rs index aa0f99a505d68..acad4f8e71686 100644 --- a/src/expr/src/vector_op/mod.rs +++ b/src/expr/src/vector_op/mod.rs @@ -32,6 +32,7 @@ pub mod cmp; pub mod concat_op; pub mod conjunction; pub mod date_trunc; +pub mod delay; pub mod encdec; pub mod exp; pub mod extract; diff --git a/src/expr/src/vector_op/proctime.rs b/src/expr/src/vector_op/proctime.rs index f0ad130cd6d3d..12b0aa6e5ef51 100644 --- a/src/expr/src/vector_op/proctime.rs +++ b/src/expr/src/vector_op/proctime.rs @@ -19,7 +19,7 @@ use risingwave_expr_macro::function; use crate::{ExprError, Result}; /// Get the processing time in Timestamptz scalar from the task-local epoch. -#[function("proctime() -> timestamptz")] +#[function("proctime() -> timestamptz", volatile)] fn proctime() -> Result { let epoch = epoch::task_local::curr_epoch().ok_or(ExprError::Context)?; Ok(epoch.as_timestamptz()) diff --git a/src/frontend/src/binder/expr/function.rs b/src/frontend/src/binder/expr/function.rs index 6bfa4883c6a5e..9cb17c5e39250 100644 --- a/src/frontend/src/binder/expr/function.rs +++ b/src/frontend/src/binder/expr/function.rs @@ -1142,7 +1142,11 @@ impl Binder { // non-deterministic ("now", now()), ("current_timestamp", now()), - ("proctime", proctime()) + ("proctime", proctime()), + ("pg_sleep", raw_call(ExprType::PgSleep)), + ("pg_sleep_for", raw_call(ExprType::PgSleepFor)), + // TODO: implement pg_sleep_until + // ("pg_sleep_until", raw_call(ExprType::PgSleepUntil)), ] .into_iter() .collect() diff --git a/src/frontend/src/expr/pure.rs b/src/frontend/src/expr/pure.rs index a229fed79b4f0..3c48aa6561821 100644 --- a/src/frontend/src/expr/pure.rs +++ b/src/frontend/src/expr/pure.rs @@ -207,7 +207,11 @@ impl ExprVisitor for ImpureAnalyzer { x } // expression output is not deterministic - expr_node::Type::Vnode | expr_node::Type::Proctime => true, + expr_node::Type::Vnode + | expr_node::Type::Proctime + | expr_node::Type::PgSleep + | expr_node::Type::PgSleepFor + | expr_node::Type::PgSleepUntil => true, } } } From ec129b65f8a3645532d070f3cc4c4ee8f4f67406 Mon Sep 17 00:00:00 2001 From: Yuhao Su <31772373+yuhao-su@users.noreply.github.com> Date: Thu, 14 Sep 2023 16:04:37 +0800 Subject: [PATCH 14/14] chore: use cfg! to instead of #cfg[] for jemalloc control policy (#12307) --- Cargo.lock | 1 + src/compute/Cargo.toml | 4 +-- .../src/memory_management/memory_manager.rs | 3 +- src/compute/src/memory_management/mod.rs | 31 +++++++------------ .../src/rpc/service/monitor_service.rs | 18 ++++------- src/workspace-hack/Cargo.toml | 2 ++ 6 files changed, 23 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9edb2360ef31..9a481abea2090 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10094,6 +10094,7 @@ dependencies = [ "subtle", "syn 1.0.109", "syn 2.0.33", + "tikv-jemalloc-sys", "time", "time-macros", "tinyvec", diff --git a/src/compute/Cargo.toml b/src/compute/Cargo.toml index e0e2a4ab0528b..24d12151c04b7 100644 --- a/src/compute/Cargo.toml +++ b/src/compute/Cargo.toml @@ -39,7 +39,7 @@ risingwave_storage = { workspace = true } risingwave_stream = { workspace = true } serde = { version = "1", features = ["derive"] } serde_json = "1" - +tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" } tokio = { version = "0.2", package = "madsim-tokio", features = [ "rt", "rt-multi-thread", @@ -54,8 +54,6 @@ tonic = { workspace = true } tower = { version = "0.4", features = ["util", "load-shed"] } tracing = "0.1" -[target.'cfg(target_os = "linux")'.dependencies] -tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" } [target.'cfg(not(madsim))'.dependencies] workspace-hack = { path = "../workspace-hack" } diff --git a/src/compute/src/memory_management/memory_manager.rs b/src/compute/src/memory_management/memory_manager.rs index 9481ef0ebef71..656d42ed54422 100644 --- a/src/compute/src/memory_management/memory_manager.rs +++ b/src/compute/src/memory_management/memory_manager.rs @@ -50,8 +50,7 @@ impl GlobalMemoryManager { auto_dump_heap_profile_config: AutoDumpHeapProfileConfig, ) -> Arc { let memory_control_policy = - build_memory_control_policy(total_memory_bytes, auto_dump_heap_profile_config.clone()) - .unwrap(); + build_memory_control_policy(total_memory_bytes, auto_dump_heap_profile_config.clone()); tracing::info!("memory control policy: {:?}", &memory_control_policy); if auto_dump_heap_profile_config.enabled { diff --git a/src/compute/src/memory_management/mod.rs b/src/compute/src/memory_management/mod.rs index f9553f860ae41..0b45f92bfb4a1 100644 --- a/src/compute/src/memory_management/mod.rs +++ b/src/compute/src/memory_management/mod.rs @@ -16,7 +16,6 @@ pub mod memory_manager; // Only enable the non-trivial policies on Linux as it relies on statistics from `jemalloc-ctl` // which might be inaccurate on other platforms. -#[cfg(target_os = "linux")] pub mod policy; use std::sync::atomic::AtomicU64; @@ -24,7 +23,6 @@ use std::sync::Arc; use risingwave_batch::task::BatchManager; use risingwave_common::config::{AutoDumpHeapProfileConfig, StorageConfig, StorageMemoryConfig}; -use risingwave_common::error::Result; use risingwave_common::util::pretty_bytes::convert; use risingwave_stream::task::LocalStreamManager; @@ -69,28 +67,23 @@ pub trait MemoryControl: Send + Sync + std::fmt::Debug { ) -> MemoryControlStats; } -#[cfg(target_os = "linux")] pub fn build_memory_control_policy( total_memory_bytes: usize, auto_dump_heap_profile_config: AutoDumpHeapProfileConfig, -) -> Result { +) -> MemoryControlRef { use self::policy::JemallocMemoryControl; - Ok(Box::new(JemallocMemoryControl::new( - total_memory_bytes, - auto_dump_heap_profile_config, - ))) -} - -#[cfg(not(target_os = "linux"))] -pub fn build_memory_control_policy( - _total_memory_bytes: usize, - _auto_dump_heap_profile_config: AutoDumpHeapProfileConfig, -) -> Result { - // We disable memory control on operating systems other than Linux now because jemalloc - // stats do not work well. - tracing::warn!("memory control is only enabled on Linux now"); - Ok(Box::new(DummyPolicy)) + if cfg!(target_os = "linux") { + Box::new(JemallocMemoryControl::new( + total_memory_bytes, + auto_dump_heap_profile_config, + )) + } else { + // We disable memory control on operating systems other than Linux now because jemalloc + // stats do not work well. + tracing::warn!("memory control is only enabled on Linux now"); + Box::new(DummyPolicy) + } } /// `DummyPolicy` is used for operarting systems other than Linux. It does nothing as memory control diff --git a/src/compute/src/rpc/service/monitor_service.rs b/src/compute/src/rpc/service/monitor_service.rs index 640eb03be7415..01f1ed6d1c4cf 100644 --- a/src/compute/src/rpc/service/monitor_service.rs +++ b/src/compute/src/rpc/service/monitor_service.rs @@ -105,7 +105,6 @@ impl MonitorService for MonitorServiceImpl { } } - #[cfg(target_os = "linux")] #[cfg_attr(coverage, no_coverage)] async fn heap_profiling( &self, @@ -117,6 +116,12 @@ impl MonitorService for MonitorServiceImpl { use tikv_jemalloc_ctl; + if !cfg!(target_os = "linux") { + return Err(Status::unimplemented( + "heap profiling is only implemented on Linux", + )); + } + if !tikv_jemalloc_ctl::opt::prof::read().unwrap() { return Err(Status::failed_precondition( "Jemalloc profiling is not enabled on the node. Try start the node with `MALLOC_CONF=prof:true`", @@ -147,17 +152,6 @@ impl MonitorService for MonitorServiceImpl { let _ = unsafe { Box::from_raw(file_path_ptr) }; response } - - #[cfg(not(target_os = "linux"))] - #[cfg_attr(coverage, no_coverage)] - async fn heap_profiling( - &self, - _request: Request, - ) -> Result, Status> { - Err(Status::unimplemented( - "heap profiling is only implemented on Linux", - )) - } } pub use grpc_middleware::*; diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml index a539527690fce..c84d4dcb26edf 100644 --- a/src/workspace-hack/Cargo.toml +++ b/src/workspace-hack/Cargo.toml @@ -94,6 +94,7 @@ serde_json = { version = "1", features = ["alloc"] } serde_with = { version = "3", features = ["json"] } smallvec = { version = "1", default-features = false, features = ["serde", "union", "write"] } subtle = { version = "2" } +tikv-jemalloc-sys = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9", features = ["profiling", "stats"] } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } tinyvec = { version = "1", features = ["alloc", "grab_spare_slice", "rustc_1_55"] } tokio = { version = "1", features = ["full", "stats", "tracing"] } @@ -195,6 +196,7 @@ smallvec = { version = "1", default-features = false, features = ["serde", "unio subtle = { version = "2" } syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +tikv-jemalloc-sys = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9", features = ["profiling", "stats"] } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] } tinyvec = { version = "1", features = ["alloc", "grab_spare_slice", "rustc_1_55"] }