From 9884181182d01cbac9c321ab99463037144a1eff Mon Sep 17 00:00:00 2001 From: August Date: Mon, 11 Sep 2023 14:08:17 +0800 Subject: [PATCH 01/30] feat: init relation and migration --- Cargo.lock | 663 +++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/meta/Cargo.toml | 1 + src/meta/src/lib.rs | 1 + src/meta/src/manager/metadata.rs | 253 ++++++++++++ src/meta/src/manager/mod.rs | 4 + 6 files changed, 923 insertions(+) create mode 100644 src/meta/src/manager/metadata.rs diff --git a/Cargo.lock b/Cargo.lock index 03962be7e7072..5e22d83c3fbb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,6 +56,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -389,6 +395,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" +[[package]] +name = "async-attributes" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" +dependencies = [ + "quote", + "syn 1.0.109", +] + [[package]] name = "async-channel" version = "1.9.0" @@ -413,6 +429,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-executor" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fa3dc5f2a8564f07759c008b9109dc0d39de92a88d5588b8a5036d286383afb" +dependencies = [ + "async-lock", + "async-task", + "concurrent-queue", + "fastrand 1.9.0", + "futures-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" +dependencies = [ + "async-channel", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "once_cell", + "tokio", +] + [[package]] name = "async-io" version = "1.13.0" @@ -487,6 +533,33 @@ dependencies = [ "syn 2.0.31", ] +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-attributes", + "async-channel", + "async-global-executor", + "async-io", + "async-lock", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -526,12 +599,27 @@ dependencies = [ "syn 2.0.31", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic" version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" +[[package]] +name = "atomic-waker" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" + [[package]] name = "atty" version = "0.2.14" @@ -1197,6 +1285,9 @@ name = "bitflags" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +dependencies = [ + "serde", +] [[package]] name = "bitmaps" @@ -1244,6 +1335,21 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77231a1c8f801696fc0123ec6150ce92cffb8e164a02afb9c8ddee0e9b65ad65" +dependencies = [ + "async-channel", + "async-lock", + "async-task", + "atomic-waker", + "fastrand 1.9.0", + "futures-lite", + "log", +] + [[package]] name = "borsh" version = "0.10.3" @@ -2346,6 +2452,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_builder" version = "0.12.0" @@ -2448,6 +2565,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "downcast" version = "0.11.0" @@ -2536,6 +2659,9 @@ name = "either" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +dependencies = [ + "serde", +] [[package]] name = "encode_unicode" @@ -2661,6 +2787,17 @@ dependencies = [ "tower-service", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "ethnum" version = "1.4.0" @@ -2805,6 +2942,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "pin-project", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3114,6 +3263,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot 0.12.1", +] + [[package]] name = "futures-io" version = "0.3.28" @@ -3238,6 +3398,18 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "google-cloud-auth" version = "0.12.0" @@ -3417,6 +3589,15 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.0", +] + [[package]] name = "hdrhistogram" version = "7.5.2" @@ -3445,6 +3626,9 @@ name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -3467,6 +3651,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac", +] + [[package]] name = "hmac" version = "0.12.1" @@ -3775,6 +3968,17 @@ dependencies = [ "str_stack", ] +[[package]] +name = "inherent" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce243b1bfa62ffc028f1cc3b6034ec63d649f3031bc8a4fbbb004e1ac17d1f68" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] + [[package]] name = "inquire" version = "0.6.2" @@ -3943,6 +4147,15 @@ dependencies = [ "duct", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -4073,6 +4286,17 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +[[package]] +name = "libsqlite3-sys" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libtest-mimic" version = "0.6.1" @@ -4151,6 +4375,9 @@ name = "log" version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +dependencies = [ + "value-bag", +] [[package]] name = "loom" @@ -4450,6 +4677,14 @@ dependencies = [ "autocfg", ] +[[package]] +name = "migration" +version = "0.1.0" +dependencies = [ + "async-std", + "sea-orm-migration", +] + [[package]] name = "mime" version = "0.3.17" @@ -5227,6 +5462,30 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ouroboros" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ba07320d39dfea882faa70554b4bd342a5f273ed59ba7c1c6b4c840492c954" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4c6225c69b4ca778c0aea097321a64c421cf4577b331c61b229267edabb6f8" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.31", +] + [[package]] name = "outref" version = "0.5.1" @@ -7274,6 +7533,7 @@ dependencies = [ "risingwave_sqlparser", "risingwave_test_runner", "scopeguard", + "sea-orm", "serde", "serde_json", "static_assertions", @@ -8033,6 +8293,165 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sea-bae" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd3534a9978d0aa7edd2808dc1f8f31c4d0ecd31ddf71d997b3c98e9f3c9114" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.31", +] + +[[package]] +name = "sea-orm" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61f6c7daef05dde3476d97001e11fca7a52b655aa3bf4fd610ab2da1176a2ed5" +dependencies = [ + "async-stream", + "async-trait", + "bigdecimal", + "chrono", + "futures", + "log", + "ouroboros", + "rust_decimal", + "sea-orm-macros", + "sea-query", + "sea-query-binder", + "serde", + "serde_json", + "sqlx", + "strum 0.25.0", + "thiserror", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sea-orm-cli" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e3f0ff2fa5672e2e7314d107c6498a18e469beeb340a0ed84e3075fce73c2cd" +dependencies = [ + "chrono", + "clap 4.4.2", + "dotenvy", + "glob", + "regex", + "sea-schema", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "sea-orm-macros" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd90e73d5f5b184bad525767da29fbfec132b4e62ebd6f60d2f2737ec6468f62" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "sea-bae", + "syn 2.0.31", + "unicode-ident", +] + +[[package]] +name = "sea-orm-migration" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21f673fcefb3a7e7b89a12b6c0e854ec0be14367635ac3435369c8ad7f11e09e" +dependencies = [ + "async-trait", + "clap 4.4.2", + "dotenvy", + "futures", + "sea-orm", + "sea-orm-cli", + "sea-schema", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "sea-query" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28c05a5bf6403834be253489bbe95fa9b1e5486bc843b61f60d26b5c9c1e244b" +dependencies = [ + "bigdecimal", + "chrono", + "derivative", + "inherent", + "ordered-float 3.9.1", + "rust_decimal", + "sea-query-derive", + "serde_json", + "time", + "uuid", +] + +[[package]] +name = "sea-query-binder" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36bbb68df92e820e4d5aeb17b4acd5cc8b5d18b2c36a4dd6f4626aabfa7ab1b9" +dependencies = [ + "bigdecimal", + "chrono", + "rust_decimal", + "sea-query", + "serde_json", + "sqlx", + "time", + "uuid", +] + +[[package]] +name = "sea-query-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd78f2e0ee8e537e9195d1049b752e0433e2cac125426bccb7b5c3e508096117" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", + "thiserror", +] + +[[package]] +name = "sea-schema" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e09eb40c78cee8fef8dfbb648036a26b7ad1f618499203ad0e8b6f97593f7f" +dependencies = [ + "futures", + "sea-query", + "sea-schema-derive", +] + +[[package]] +name = "sea-schema-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f686050f76bffc4f635cda8aea6df5548666b830b52387e8bc7de11056d11e" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "seahash" version = "4.1.0" @@ -8653,6 +9072,17 @@ dependencies = [ "der 0.7.8", ] +[[package]] +name = "sqlformat" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" +dependencies = [ + "itertools 0.11.0", + "nom", + "unicode_categories", +] + [[package]] name = "sqllogictest" version = "0.15.3" @@ -8676,6 +9106,221 @@ dependencies = [ "tracing", ] +[[package]] +name = "sqlx" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" +dependencies = [ + "ahash 0.8.3", + "atoi", + "bigdecimal", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "dotenvy", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap 2.0.0", + "log", + "memchr", + "native-tls", + "once_cell", + "paste", + "percent-encoding", + "rust_decimal", + "serde", + "serde_json", + "sha2 0.10.7", + "smallvec", + "sqlformat", + "thiserror", + "time", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" +dependencies = [ + "dotenvy", + "either", + "heck 0.4.1", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.7", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" +dependencies = [ + "atoi", + "base64 0.21.3", + "bigdecimal", + "bitflags 2.4.0", + "byteorder", + "bytes", + "chrono", + "crc", + "digest 0.10.7", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "rust_decimal", + "serde", + "sha1", + "sha2 0.10.7", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "time", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" +dependencies = [ + "atoi", + "base64 0.21.3", + "bigdecimal", + "bitflags 2.4.0", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "num-bigint", + "once_cell", + "rand", + "rust_decimal", + "serde", + "serde_json", + "sha1", + "sha2 0.10.7", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "time", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "time", + "tracing", + "url", + "uuid", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -9607,6 +10252,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "unsafe-libyaml" version = "0.2.9" @@ -9660,6 +10311,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "value-bag" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" + [[package]] name = "value-trait" version = "0.6.1" @@ -9879,6 +10536,12 @@ dependencies = [ "rustix 0.38.11", ] +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 83be53f1d7439..fee5f6fafcd29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "src/java_binding", "src/jni_core", "src/meta", + "src/meta/src/model_v2/migration", "src/object_store", "src/prost", "src/prost/helpers", diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml index d8401cc5a7f71..f84d14f3fc2dd 100644 --- a/src/meta/Cargo.toml +++ b/src/meta/Cargo.toml @@ -54,6 +54,7 @@ risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } risingwave_sqlparser = { workspace = true } scopeguard = "1.2.0" +sea-orm = { version = "0.12.0", features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "mock", "runtime-tokio-native-tls", "macros" ] } serde = { version = "1", features = ["derive"] } serde_json = "1" sync-point = { path = "../utils/sync-point" } diff --git a/src/meta/src/lib.rs b/src/meta/src/lib.rs index 92d3c571f57c5..0f727d2f9aa51 100644 --- a/src/meta/src/lib.rs +++ b/src/meta/src/lib.rs @@ -42,6 +42,7 @@ mod error; pub mod hummock; pub mod manager; pub mod model; +pub mod model_v2; mod rpc; pub(crate) mod serving; pub mod storage; diff --git a/src/meta/src/manager/metadata.rs b/src/meta/src/manager/metadata.rs new file mode 100644 index 0000000000000..66b3a6558265a --- /dev/null +++ b/src/meta/src/manager/metadata.rs @@ -0,0 +1,253 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::iter; + +use anyhow::anyhow; +use prometheus::core::{Atomic, AtomicI64}; +use risingwave_common::catalog::{ + DEFAULT_DATABASE_NAME, DEFAULT_SCHEMA_NAME, DEFAULT_SUPER_USER, DEFAULT_SUPER_USER_FOR_PG, + DEFAULT_SUPER_USER_FOR_PG_ID, DEFAULT_SUPER_USER_ID, SYSTEM_SCHEMAS, +}; +use risingwave_pb::catalog::PbDatabase; +use risingwave_pb::meta::subscribe_response::{Info, Operation}; +use sea_orm::{ + ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, Database, DatabaseConnection, + EntityTrait, QueryFilter, TransactionTrait, +}; + +use crate::manager::{DatabaseId, MetaSrvEnv, NotificationVersion}; +use crate::model_v2::prelude; +use crate::{model_v2, MetaError, MetaResult}; + +// todo: refine the error transform. +impl From for MetaError { + fn from(err: sea_orm::DbErr) -> Self { + anyhow!(err).into() + } +} + +// enum Object { +// Database(database::Model), +// Schema(schema::Model), +// Table(schema::Model), +// User(user::Model), +// } +// +// impl Object { +// fn obj_type(&self) -> &str { +// match self { +// Object::Database(_) => "database", +// Object::Schema(_) => "schema", +// Object::Table(_) => "table", +// Object::User(_) => "user", +// } +// } +// } + +pub struct MetadataManager { + env: MetaSrvEnv, + conn: DatabaseConnection, +} + +impl MetadataManager { + pub async fn new(env: MetaSrvEnv, conn: DatabaseConnection) -> MetaResult { + //FIXME: for test only, to remove. + // let dbs: Vec = prelude::Database::find().all(&conn).await.unwrap(); + // dbs.into_iter().for_each(|db| { + // println!("db: {:?}", db); + // }); + let mgr = Self { env, conn }; + mgr.init_user().await?; + mgr.init_database().await?; + Ok(mgr) + } + + // FIXME: should be done in init.sql scripts. + async fn init_user(&self) -> MetaResult<()> { + for (id, name) in [ + (DEFAULT_SUPER_USER_ID, DEFAULT_SUPER_USER), + (DEFAULT_SUPER_USER_FOR_PG_ID, DEFAULT_SUPER_USER_FOR_PG), + ] { + if prelude::User::find_by_id(id as i32) + .one(&self.conn) + .await? + .is_some() + { + continue; + } + let user = model_v2::user::ActiveModel { + user_id: ActiveValue::Set(id as _), + name: ActiveValue::Set(name.into()), + is_super: ActiveValue::Set(Some(true)), + can_create_db: ActiveValue::Set(Some(true)), + can_create_user: ActiveValue::Set(Some(true)), + can_login: ActiveValue::Set(Some(true)), + ..Default::default() + }; + user.insert(&self.conn).await?; + } + Ok(()) + } + + // FIXME: should be done in init.sql scripts. + async fn init_database(&self) -> MetaResult<()> { + if prelude::Database::find() + .filter(model_v2::database::Column::Name.eq(DEFAULT_DATABASE_NAME)) + .one(&self.conn) + .await? + .is_some() + { + return Ok(()); + } + let mut db = model_v2::database::ActiveModel::new(); + db.name = ActiveValue::Set(DEFAULT_DATABASE_NAME.into()); + db.owner_id = ActiveValue::Set(DEFAULT_SUPER_USER_ID as _); + self.create_database(db).await?; + Ok(()) + } +} + +impl MetadataManager { + async fn notify_frontend(&self, operation: Operation, info: Info) -> NotificationVersion { + self.env + .notification_manager() + .notify_frontend(operation, info) + .await + } +} + +impl MetadataManager { + // pub async fn snapshot() -> MetaResult { + // //d,s,tdd + self.revision + // // [1,2,3] 5 + // // [1,2,3] 6 + // // 4 6 + // } + pub async fn create_database( + &self, + db: model_v2::database::ActiveModel, + ) -> MetaResult { + let txn = self.conn.begin().await?; + let db = db.insert(&txn).await?; + let mut schemas = vec![]; + for schema_name in iter::once(DEFAULT_SCHEMA_NAME).chain(SYSTEM_SCHEMAS) { + let mut schema = model_v2::schema::ActiveModel::new(); + schema.database_id = ActiveValue::Set(db.database_id); + schema.name = ActiveValue::Set(schema_name.into()); + schema.owner_id = ActiveValue::Set(db.owner_id); + schemas.push(schema.insert(&txn).await?); + } + txn.commit().await?; + + let mut version = self + .notify_frontend(Operation::Add, Info::Database(db.into())) + .await; + for schema in schemas { + version = self + .notify_frontend(Operation::Add, Info::Schema(schema.into())) + .await; + } + + Ok(version) + } + + // todo: return all streaming jobs. + pub async fn drop_database(&self, database_id: DatabaseId) -> MetaResult<()> { + let tables = prelude::Table::find() + .filter(model_v2::table::Column::DatabaseId.eq(database_id as i32)) + .all(&self.conn) + .await?; + // 1. unregister source. + // 2. fragments + actors, streaming manager drop streaming job. + // 3. connection to drop. + + // let fragments = prelude::Fragment:: + // + + let txn = self.conn.begin().await?; + let db: model_v2::database::ActiveModel = prelude::Database::find_by_id(database_id as i32) + .one(&txn) + .await? + .ok_or_else(|| anyhow!("database not found"))? + .into(); + + prelude::Table::delete_many() + .filter(model_v2::table::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + prelude::Source::delete_many() + .filter(model_v2::source::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + prelude::Sink::delete_many() + .filter(model_v2::sink::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + prelude::Index::delete_many() + .filter(model_v2::index::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + prelude::Function::delete_many() + .filter(model_v2::function::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + prelude::Connection::delete_many() + .filter(model_v2::connection::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + prelude::View::delete_many() + .filter(model_v2::view::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + prelude::Schema::delete_many() + .filter(model_v2::schema::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + prelude::Database::delete(db).exec(&txn).await?; + + txn.commit().await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use sea_orm::{DbBackend, MockDatabase}; + use super::*; + + #[tokio::test] + async fn test_create_database() { + let conn = Database::connect("postgres://postgres:@localhost:5432/postgres") + .await + .unwrap(); + // let conn = MockDatabase::new(DbBackend::Postgres).into_connection(); + let mgr = MetadataManager::new(MetaSrvEnv::for_test().await, conn) + .await + .unwrap(); + let db = model_v2::database::ActiveModel { + name: ActiveValue::Set("test".into()), + owner_id: ActiveValue::Set(1), + ..Default::default() + }; + mgr.create_database(db).await.unwrap(); + let db = prelude::Database::find() + .filter(model_v2::database::Column::Name.eq("test")) + .one(&mgr.conn) + .await + .unwrap() + .unwrap(); + mgr.drop_database(db.database_id as _).await.unwrap(); + } +} diff --git a/src/meta/src/manager/mod.rs b/src/meta/src/manager/mod.rs index 6f787dba23d09..77e1c3fce6669 100644 --- a/src/meta/src/manager/mod.rs +++ b/src/meta/src/manager/mod.rs @@ -17,6 +17,7 @@ mod cluster; mod env; mod id; mod idle; +mod metadata; mod notification; pub(crate) mod sink_coordination; mod streaming_job; @@ -29,7 +30,10 @@ pub use env::MetaSrvEnv; pub(crate) use env::*; pub(crate) use id::*; pub(crate) use idle::*; +pub use metadata::*; pub(crate) use notification::*; pub use notification::{LocalNotification, MessageStatus, NotificationManagerRef}; pub(crate) use streaming_job::*; pub(crate) use system_param::*; + +pub use super::model_v2::prelude; From e1537f5836edce284b2ca0560c2a40232be8180e Mon Sep 17 00:00:00 2001 From: August Date: Mon, 11 Sep 2023 14:09:03 +0800 Subject: [PATCH 02/30] add model --- src/meta/src/model_v2/actor.rs | 39 ++++++ src/meta/src/model_v2/connection.rs | 88 +++++++++++++ src/meta/src/model_v2/database.rs | 97 ++++++++++++++ src/meta/src/model_v2/fragment.rs | 50 +++++++ src/meta/src/model_v2/function.rs | 78 +++++++++++ src/meta/src/model_v2/index.rs | 93 +++++++++++++ src/meta/src/model_v2/migration/Cargo.toml | 16 +++ src/meta/src/model_v2/migration/README.md | 41 ++++++ src/meta/src/model_v2/migration/src/lib.rs | 14 ++ .../migration/src/m20230908_072257_init.rs | 51 +++++++ src/meta/src/model_v2/migration/src/main.rs | 6 + src/meta/src/model_v2/mod.rs | 60 +++++++++ src/meta/src/model_v2/object.rs | 123 +++++++++++++++++ src/meta/src/model_v2/object_dependency.rs | 34 +++++ src/meta/src/model_v2/prelude.rs | 19 +++ src/meta/src/model_v2/schema.rs | 103 +++++++++++++++ src/meta/src/model_v2/sink.rs | 97 ++++++++++++++ src/meta/src/model_v2/source.rs | 114 ++++++++++++++++ src/meta/src/model_v2/table.rs | 124 ++++++++++++++++++ src/meta/src/model_v2/user.rs | 94 +++++++++++++ src/meta/src/model_v2/user_privilege.rs | 51 +++++++ src/meta/src/model_v2/view.rs | 75 +++++++++++ src/meta/src/model_v2/worker.rs | 28 ++++ src/meta/src/model_v2/worker_property.rs | 34 +++++ 24 files changed, 1529 insertions(+) create mode 100644 src/meta/src/model_v2/actor.rs create mode 100644 src/meta/src/model_v2/connection.rs create mode 100644 src/meta/src/model_v2/database.rs create mode 100644 src/meta/src/model_v2/fragment.rs create mode 100644 src/meta/src/model_v2/function.rs create mode 100644 src/meta/src/model_v2/index.rs create mode 100644 src/meta/src/model_v2/migration/Cargo.toml create mode 100644 src/meta/src/model_v2/migration/README.md create mode 100644 src/meta/src/model_v2/migration/src/lib.rs create mode 100644 src/meta/src/model_v2/migration/src/m20230908_072257_init.rs create mode 100644 src/meta/src/model_v2/migration/src/main.rs create mode 100644 src/meta/src/model_v2/mod.rs create mode 100644 src/meta/src/model_v2/object.rs create mode 100644 src/meta/src/model_v2/object_dependency.rs create mode 100644 src/meta/src/model_v2/prelude.rs create mode 100644 src/meta/src/model_v2/schema.rs create mode 100644 src/meta/src/model_v2/sink.rs create mode 100644 src/meta/src/model_v2/source.rs create mode 100644 src/meta/src/model_v2/table.rs create mode 100644 src/meta/src/model_v2/user.rs create mode 100644 src/meta/src/model_v2/user_privilege.rs create mode 100644 src/meta/src/model_v2/view.rs create mode 100644 src/meta/src/model_v2/worker.rs create mode 100644 src/meta/src/model_v2/worker_property.rs diff --git a/src/meta/src/model_v2/actor.rs b/src/meta/src/model_v2/actor.rs new file mode 100644 index 0000000000000..ae3601eb1ec2b --- /dev/null +++ b/src/meta/src/model_v2/actor.rs @@ -0,0 +1,39 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "actor")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub actor_id: i32, + pub fragment_id: i32, + pub status: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub splits: Option, + pub parallel_unit_id: i32, + pub upstream_actor_ids: Option>, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub dispatchers: Option, + pub vnode_bitmap: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::fragment::Entity", + from = "Column::FragmentId", + to = "super::fragment::Column::FragmentId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Fragment, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Fragment.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/connection.rs b/src/meta/src/model_v2/connection.rs new file mode 100644 index 0000000000000..5a3ad066cd1b9 --- /dev/null +++ b/src/meta/src/model_v2/connection.rs @@ -0,0 +1,88 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "connection")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub connection_id: i32, + pub name: String, + pub schema_id: i32, + pub database_id: i32, + pub owner_id: i32, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub info: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::database::Entity", + from = "Column::DatabaseId", + to = "super::database::Column::DatabaseId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Database, + #[sea_orm(has_many = "super::object::Entity")] + Object, + #[sea_orm( + belongs_to = "super::schema::Entity", + from = "Column::SchemaId", + to = "super::schema::Column::SchemaId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Schema, + #[sea_orm(has_many = "super::sink::Entity")] + Sink, + #[sea_orm(has_many = "super::source::Entity")] + Source, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::OwnerId", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "NoAction" + )] + User, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Database.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Object.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Schema.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Sink.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Source.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/database.rs b/src/meta/src/model_v2/database.rs new file mode 100644 index 0000000000000..2cb31cd5f4002 --- /dev/null +++ b/src/meta/src/model_v2/database.rs @@ -0,0 +1,97 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "database")] +pub struct Model { + #[sea_orm(primary_key)] + pub database_id: i32, + #[sea_orm(unique)] + pub name: String, + pub owner_id: i32, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::connection::Entity")] + Connection, + #[sea_orm(has_many = "super::function::Entity")] + Function, + #[sea_orm(has_many = "super::index::Entity")] + Index, + #[sea_orm(has_many = "super::schema::Entity")] + Schema, + #[sea_orm(has_many = "super::sink::Entity")] + Sink, + #[sea_orm(has_many = "super::source::Entity")] + Source, + #[sea_orm(has_many = "super::table::Entity")] + Table, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::OwnerId", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "NoAction" + )] + User, + #[sea_orm(has_many = "super::view::Entity")] + View, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Connection.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Function.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Index.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Schema.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Sink.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Source.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Table.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::View.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/fragment.rs b/src/meta/src/model_v2/fragment.rs new file mode 100644 index 0000000000000..cbe691b8e0f27 --- /dev/null +++ b/src/meta/src/model_v2/fragment.rs @@ -0,0 +1,50 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "fragment")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub fragment_id: i32, + pub table_id: i32, + pub fragment_type_mask: Option, + pub distribution_type: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub stream_node: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub vnode_mapping: Option, + pub state_table_ids: Option>, + pub upstream_fragment_id: Option>, + pub dispatcher_type: Option, + pub dist_key_indices: Option>, + pub output_indices: Option>, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::actor::Entity")] + Actor, + #[sea_orm( + belongs_to = "super::table::Entity", + from = "Column::TableId", + to = "super::table::Column::TableId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Table, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Actor.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Table.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/function.rs b/src/meta/src/model_v2/function.rs new file mode 100644 index 0000000000000..abec747fa97f3 --- /dev/null +++ b/src/meta/src/model_v2/function.rs @@ -0,0 +1,78 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "function")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub function_id: i32, + pub name: String, + pub schema_id: i32, + pub database_id: i32, + pub owner_id: i32, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub arg_types: Option, + pub return_type: Option, + pub language: Option, + pub link: Option, + pub identifier: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub kind: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::database::Entity", + from = "Column::DatabaseId", + to = "super::database::Column::DatabaseId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Database, + #[sea_orm(has_many = "super::object::Entity")] + Object, + #[sea_orm( + belongs_to = "super::schema::Entity", + from = "Column::SchemaId", + to = "super::schema::Column::SchemaId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Schema, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::OwnerId", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "NoAction" + )] + User, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Database.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Object.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Schema.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/index.rs b/src/meta/src/model_v2/index.rs new file mode 100644 index 0000000000000..88268e47cdb07 --- /dev/null +++ b/src/meta/src/model_v2/index.rs @@ -0,0 +1,93 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "index")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub index_id: i32, + pub name: String, + pub schema_id: i32, + pub database_id: i32, + pub owner_id: i32, + pub index_table_id: i32, + pub primary_table_id: i32, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub index_items: Option, + pub original_columns: Option>, + pub initialized_at_epoch: Option, + pub created_at_epoch: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::database::Entity", + from = "Column::DatabaseId", + to = "super::database::Column::DatabaseId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Database, + #[sea_orm(has_many = "super::object::Entity")] + Object, + #[sea_orm( + belongs_to = "super::schema::Entity", + from = "Column::SchemaId", + to = "super::schema::Column::SchemaId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Schema, + #[sea_orm( + belongs_to = "super::table::Entity", + from = "Column::IndexTableId", + to = "super::table::Column::TableId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Table2, + #[sea_orm( + belongs_to = "super::table::Entity", + from = "Column::PrimaryTableId", + to = "super::table::Column::TableId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Table1, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::OwnerId", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "NoAction" + )] + User, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Database.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Object.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Schema.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/migration/Cargo.toml b/src/meta/src/model_v2/migration/Cargo.toml new file mode 100644 index 0000000000000..722f8ae0ba225 --- /dev/null +++ b/src/meta/src/model_v2/migration/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "migration" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +name = "migration" +path = "src/lib.rs" + +[dependencies] +async-std = { version = "1", features = ["attributes", "tokio1"] } + +[dependencies.sea-orm-migration] +version = "0.12.0" +features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls" ] diff --git a/src/meta/src/model_v2/migration/README.md b/src/meta/src/model_v2/migration/README.md new file mode 100644 index 0000000000000..3b438d89e31c5 --- /dev/null +++ b/src/meta/src/model_v2/migration/README.md @@ -0,0 +1,41 @@ +# Running Migrator CLI + +- Generate a new migration file + ```sh + cargo run -- generate MIGRATION_NAME + ``` +- Apply all pending migrations + ```sh + cargo run + ``` + ```sh + cargo run -- up + ``` +- Apply first 10 pending migrations + ```sh + cargo run -- up -n 10 + ``` +- Rollback last applied migrations + ```sh + cargo run -- down + ``` +- Rollback last 10 applied migrations + ```sh + cargo run -- down -n 10 + ``` +- Drop all tables from the database, then reapply all migrations + ```sh + cargo run -- fresh + ``` +- Rollback all applied migrations, then reapply all migrations + ```sh + cargo run -- refresh + ``` +- Rollback all applied migrations + ```sh + cargo run -- reset + ``` +- Check the status of all migrations + ```sh + cargo run -- status + ``` diff --git a/src/meta/src/model_v2/migration/src/lib.rs b/src/meta/src/model_v2/migration/src/lib.rs new file mode 100644 index 0000000000000..3abf4a994c17d --- /dev/null +++ b/src/meta/src/model_v2/migration/src/lib.rs @@ -0,0 +1,14 @@ +pub use sea_orm_migration::prelude::*; + +mod m20230908_072257_init; + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![ + Box::new(m20230908_072257_init::Migration), + ] + } +} diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs new file mode 100644 index 0000000000000..5180e77baa95c --- /dev/null +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -0,0 +1,51 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +const INIT_SQL: &str = r#" + +"#; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Replace the sample below with your own migration scripts + todo!(); + + manager + .create_table( + Table::create() + .table(Post::Table) + .if_not_exists() + .col( + ColumnDef::new(Post::Id) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(Post::Title).string().not_null()) + .col(ColumnDef::new(Post::Text).string().not_null()) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Replace the sample below with your own migration scripts + todo!(); + + manager + .drop_table(Table::drop().table(Post::Table).to_owned()) + .await + } +} + +#[derive(DeriveIden)] +enum Post { + Table, + Id, + Title, + Text, +} diff --git a/src/meta/src/model_v2/migration/src/main.rs b/src/meta/src/model_v2/migration/src/main.rs new file mode 100644 index 0000000000000..c6b6e48dbc06b --- /dev/null +++ b/src/meta/src/model_v2/migration/src/main.rs @@ -0,0 +1,6 @@ +use sea_orm_migration::prelude::*; + +#[async_std::main] +async fn main() { + cli::run_cli(migration::Migrator).await; +} diff --git a/src/meta/src/model_v2/mod.rs b/src/meta/src/model_v2/mod.rs new file mode 100644 index 0000000000000..76277e8722c16 --- /dev/null +++ b/src/meta/src/model_v2/mod.rs @@ -0,0 +1,60 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use risingwave_pb::catalog::{PbDatabase, PbSchema}; +use risingwave_pb::user::PbUserInfo; + +pub mod prelude; + +pub mod actor; +pub mod connection; +pub mod database; +pub mod fragment; +pub mod function; +pub mod index; +pub mod object; +pub mod object_dependency; +pub mod schema; +pub mod sink; +pub mod source; +pub mod table; +pub mod user; +pub mod user_privilege; +pub mod view; +pub mod worker; +pub mod worker_property; + +impl From for PbDatabase { + fn from(model: database::Model) -> Self { + Self { + id: model.database_id as _, + name: model.name, + owner: model.owner_id as _, + } + } +} + +impl From for PbSchema { + fn from(model: schema::Model) -> Self { + Self { + id: model.schema_id as _, + name: model.name, + database_id: model.database_id as _, + owner: model.owner_id as _, + } + } +} + +// impl From for PbUserInfo { +// fn from(model: user::Model) -> Self { +// Self { +// id: model.user_id as _, +// name: model.name, +// is_super: model.is_super.unwrap_or_default(), +// can_create_db: model.can_create_db.unwrap_or_default(), +// can_create_user: model.can_create_user.unwrap_or_default(), +// can_login: model.can_login.unwrap_or_default(), +// auth_info: None, +// grant_privileges: vec![], +// } +// } +// } diff --git a/src/meta/src/model_v2/object.rs b/src/meta/src/model_v2/object.rs new file mode 100644 index 0000000000000..ad14e1c7b2a13 --- /dev/null +++ b/src/meta/src/model_v2/object.rs @@ -0,0 +1,123 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "object")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub oid: i32, + pub obj_type: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::connection::Entity", + from = "Column::Oid", + to = "super::connection::Column::ConnectionId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Connection, + #[sea_orm( + belongs_to = "super::function::Entity", + from = "Column::Oid", + to = "super::function::Column::FunctionId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Function, + #[sea_orm( + belongs_to = "super::index::Entity", + from = "Column::Oid", + to = "super::index::Column::IndexId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Index, + #[sea_orm( + belongs_to = "super::sink::Entity", + from = "Column::Oid", + to = "super::sink::Column::SinkId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Sink, + #[sea_orm( + belongs_to = "super::source::Entity", + from = "Column::Oid", + to = "super::source::Column::SourceId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Source, + #[sea_orm( + belongs_to = "super::table::Entity", + from = "Column::Oid", + to = "super::table::Column::TableId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Table, + #[sea_orm(has_many = "super::user_privilege::Entity")] + UserPrivilege, + #[sea_orm( + belongs_to = "super::view::Entity", + from = "Column::Oid", + to = "super::view::Column::ViewId", + on_update = "NoAction", + on_delete = "NoAction" + )] + View, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Connection.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Function.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Index.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Sink.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Source.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Table.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::UserPrivilege.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::View.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/object_dependency.rs b/src/meta/src/model_v2/object_dependency.rs new file mode 100644 index 0000000000000..5f3dfbb8d00e7 --- /dev/null +++ b/src/meta/src/model_v2/object_dependency.rs @@ -0,0 +1,34 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "object_dependency")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: i32, + pub oid: i32, + pub used_by: i32, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::Oid", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] + Object2, + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::UsedBy", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] + Object1, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/prelude.rs b/src/meta/src/model_v2/prelude.rs new file mode 100644 index 0000000000000..47cce4ad9c047 --- /dev/null +++ b/src/meta/src/model_v2/prelude.rs @@ -0,0 +1,19 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +pub use super::actor::Entity as Actor; +pub use super::connection::Entity as Connection; +pub use super::database::Entity as Database; +pub use super::fragment::Entity as Fragment; +pub use super::function::Entity as Function; +pub use super::index::Entity as Index; +pub use super::object::Entity as Object; +pub use super::object_dependency::Entity as ObjectDependency; +pub use super::schema::Entity as Schema; +pub use super::sink::Entity as Sink; +pub use super::source::Entity as Source; +pub use super::table::Entity as Table; +pub use super::user::Entity as User; +pub use super::user_privilege::Entity as UserPrivilege; +pub use super::view::Entity as View; +pub use super::worker::Entity as Worker; +pub use super::worker_property::Entity as WorkerProperty; diff --git a/src/meta/src/model_v2/schema.rs b/src/meta/src/model_v2/schema.rs new file mode 100644 index 0000000000000..3749ff7f90621 --- /dev/null +++ b/src/meta/src/model_v2/schema.rs @@ -0,0 +1,103 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "schema")] +pub struct Model { + #[sea_orm(primary_key)] + pub schema_id: i32, + pub name: String, + pub database_id: i32, + pub owner_id: i32, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::connection::Entity")] + Connection, + #[sea_orm( + belongs_to = "super::database::Entity", + from = "Column::DatabaseId", + to = "super::database::Column::DatabaseId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Database, + #[sea_orm(has_many = "super::function::Entity")] + Function, + #[sea_orm(has_many = "super::index::Entity")] + Index, + #[sea_orm(has_many = "super::sink::Entity")] + Sink, + #[sea_orm(has_many = "super::source::Entity")] + Source, + #[sea_orm(has_many = "super::table::Entity")] + Table, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::OwnerId", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "NoAction" + )] + User, + #[sea_orm(has_many = "super::view::Entity")] + View, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Connection.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Database.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Function.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Index.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Sink.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Source.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Table.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::View.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/sink.rs b/src/meta/src/model_v2/sink.rs new file mode 100644 index 0000000000000..87a4524030f27 --- /dev/null +++ b/src/meta/src/model_v2/sink.rs @@ -0,0 +1,97 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "sink")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub sink_id: i32, + pub name: String, + pub schema_id: i32, + pub database_id: i32, + pub owner_id: i32, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub columns: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub pk_column_ids: Option, + pub distribution_key: Option>, + pub downstream_pk: Option>, + pub sink_type: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub properties: Option, + pub definition: Option, + pub connection_id: Option, + pub initialized_at_epoch: Option, + pub created_at_epoch: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::connection::Entity", + from = "Column::ConnectionId", + to = "super::connection::Column::ConnectionId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Connection, + #[sea_orm( + belongs_to = "super::database::Entity", + from = "Column::DatabaseId", + to = "super::database::Column::DatabaseId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Database, + #[sea_orm(has_many = "super::object::Entity")] + Object, + #[sea_orm( + belongs_to = "super::schema::Entity", + from = "Column::SchemaId", + to = "super::schema::Column::SchemaId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Schema, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::OwnerId", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "NoAction" + )] + User, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Connection.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Database.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Object.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Schema.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/source.rs b/src/meta/src/model_v2/source.rs new file mode 100644 index 0000000000000..0909b24f519f5 --- /dev/null +++ b/src/meta/src/model_v2/source.rs @@ -0,0 +1,114 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "source")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub source_id: i32, + pub name: String, + pub schema_id: i32, + pub database_id: i32, + pub owner_id: i32, + pub row_id_index: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub columns: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub pk_column_ids: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub properties: Option, + pub definition: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub source_info: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub watermark_descs: Option, + pub optional_associated_table_id: Option, + pub connection_id: Option, + pub initialized_at_epoch: Option, + pub created_at_epoch: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::connection::Entity", + from = "Column::ConnectionId", + to = "super::connection::Column::ConnectionId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Connection, + #[sea_orm( + belongs_to = "super::database::Entity", + from = "Column::DatabaseId", + to = "super::database::Column::DatabaseId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Database, + #[sea_orm(has_many = "super::object::Entity")] + Object, + #[sea_orm( + belongs_to = "super::schema::Entity", + from = "Column::SchemaId", + to = "super::schema::Column::SchemaId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Schema, + #[sea_orm( + belongs_to = "super::table::Entity", + from = "Column::OptionalAssociatedTableId", + to = "super::table::Column::TableId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Table, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::OwnerId", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "NoAction" + )] + User, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Connection.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Database.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Object.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Schema.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Table.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/table.rs b/src/meta/src/model_v2/table.rs new file mode 100644 index 0000000000000..1e21f75b39f19 --- /dev/null +++ b/src/meta/src/model_v2/table.rs @@ -0,0 +1,124 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "table")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub table_id: i32, + pub name: String, + pub schema_id: i32, + pub database_id: i32, + pub owner_id: i32, + pub optional_associated_source_id: Option, + pub table_type: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub columns: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub pk: Option, + pub distribution_key: Option>, + pub append_only: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub properties: Option, + pub fragment_id: Option, + pub vnode_col_index: Option, + pub value_indices: Option>, + pub definition: Option, + pub handle_pk_conflict_behavior: Option, + pub read_prefix_len_hint: Option, + pub watermark_indices: Option>, + pub dist_key_in_pk: Option>, + pub dml_fragment_id: Option, + pub cardinality: Option>, + pub initialized_at_epoch: Option, + pub created_at_epoch: Option, + pub cleaned_by_watermark: Option, + pub version: Option>, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::database::Entity", + from = "Column::DatabaseId", + to = "super::database::Column::DatabaseId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Database, + #[sea_orm( + belongs_to = "super::fragment::Entity", + from = "Column::DmlFragmentId", + to = "super::fragment::Column::FragmentId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Fragment2, + #[sea_orm( + belongs_to = "super::fragment::Entity", + from = "Column::FragmentId", + to = "super::fragment::Column::FragmentId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Fragment1, + #[sea_orm(has_many = "super::object::Entity")] + Object, + #[sea_orm( + belongs_to = "super::schema::Entity", + from = "Column::SchemaId", + to = "super::schema::Column::SchemaId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Schema, + #[sea_orm( + belongs_to = "super::source::Entity", + from = "Column::OptionalAssociatedSourceId", + to = "super::source::Column::SourceId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Source, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::OwnerId", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "NoAction" + )] + User, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Database.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Object.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Schema.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Source.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/user.rs b/src/meta/src/model_v2/user.rs new file mode 100644 index 0000000000000..1a99e25128d7d --- /dev/null +++ b/src/meta/src/model_v2/user.rs @@ -0,0 +1,94 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "user")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub user_id: i32, + pub name: String, + pub is_super: Option, + pub can_create_db: Option, + pub can_create_user: Option, + pub can_login: Option, + pub auth_type: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::connection::Entity")] + Connection, + #[sea_orm(has_many = "super::database::Entity")] + Database, + #[sea_orm(has_many = "super::function::Entity")] + Function, + #[sea_orm(has_many = "super::index::Entity")] + Index, + #[sea_orm(has_many = "super::schema::Entity")] + Schema, + #[sea_orm(has_many = "super::sink::Entity")] + Sink, + #[sea_orm(has_many = "super::source::Entity")] + Source, + #[sea_orm(has_many = "super::table::Entity")] + Table, + #[sea_orm(has_many = "super::view::Entity")] + View, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Connection.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Database.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Function.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Index.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Schema.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Sink.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Source.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Table.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::View.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/user_privilege.rs b/src/meta/src/model_v2/user_privilege.rs new file mode 100644 index 0000000000000..5f13cf9cb3bd6 --- /dev/null +++ b/src/meta/src/model_v2/user_privilege.rs @@ -0,0 +1,51 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "user_privilege")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub user_id: i32, + pub oid: i32, + pub granted_by: i32, + pub actions: Option>, + pub with_grant_option: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::Oid", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] + Object, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::GrantedBy", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "NoAction" + )] + User2, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::UserId", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "Cascade" + )] + User1, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Object.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/view.rs b/src/meta/src/model_v2/view.rs new file mode 100644 index 0000000000000..2a28e29f2ef9b --- /dev/null +++ b/src/meta/src/model_v2/view.rs @@ -0,0 +1,75 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "view")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub view_id: i32, + pub name: String, + pub schema_id: i32, + pub database_id: i32, + pub owner_id: i32, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub properties: Option, + pub sql: Option, + #[sea_orm(column_type = "JsonBinary", nullable)] + pub columns: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::database::Entity", + from = "Column::DatabaseId", + to = "super::database::Column::DatabaseId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Database, + #[sea_orm(has_many = "super::object::Entity")] + Object, + #[sea_orm( + belongs_to = "super::schema::Entity", + from = "Column::SchemaId", + to = "super::schema::Column::SchemaId", + on_update = "NoAction", + on_delete = "NoAction" + )] + Schema, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::OwnerId", + to = "super::user::Column::UserId", + on_update = "NoAction", + on_delete = "NoAction" + )] + User, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Database.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Object.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Schema.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/worker.rs b/src/meta/src/model_v2/worker.rs new file mode 100644 index 0000000000000..bc3581279e895 --- /dev/null +++ b/src/meta/src/model_v2/worker.rs @@ -0,0 +1,28 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "worker")] +pub struct Model { + #[sea_orm(primary_key)] + pub worker_id: i32, + pub worker_type: Option, + pub host: String, + pub port: i32, + pub status: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::worker_property::Entity")] + WorkerProperty, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::WorkerProperty.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/worker_property.rs b/src/meta/src/model_v2/worker_property.rs new file mode 100644 index 0000000000000..2dd5870830316 --- /dev/null +++ b/src/meta/src/model_v2/worker_property.rs @@ -0,0 +1,34 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "worker_property")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub worker_id: i32, + pub parallel_unit_ids: Option>, + pub is_streaming: Option, + pub is_serving: Option, + pub is_unschedulable: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::worker::Entity", + from = "Column::WorkerId", + to = "super::worker::Column::WorkerId", + on_update = "NoAction", + on_delete = "Cascade" + )] + Worker, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Worker.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} From d232ce8d4ae14299fd0fb9baf0e78fca0fc2977d Mon Sep 17 00:00:00 2001 From: August Date: Mon, 11 Sep 2023 18:51:20 +0800 Subject: [PATCH 03/30] add init version of migration --- src/meta/src/manager/metadata.rs | 93 +- src/meta/src/model_v2/migration/src/lib.rs | 4 +- .../migration/src/m20230908_072257_init.rs | 1130 ++++++++++++++++- src/meta/src/model_v2/mod.rs | 1 - 4 files changed, 1126 insertions(+), 102 deletions(-) diff --git a/src/meta/src/manager/metadata.rs b/src/meta/src/manager/metadata.rs index 66b3a6558265a..015859065528d 100644 --- a/src/meta/src/manager/metadata.rs +++ b/src/meta/src/manager/metadata.rs @@ -15,12 +15,7 @@ use std::iter; use anyhow::anyhow; -use prometheus::core::{Atomic, AtomicI64}; -use risingwave_common::catalog::{ - DEFAULT_DATABASE_NAME, DEFAULT_SCHEMA_NAME, DEFAULT_SUPER_USER, DEFAULT_SUPER_USER_FOR_PG, - DEFAULT_SUPER_USER_FOR_PG_ID, DEFAULT_SUPER_USER_ID, SYSTEM_SCHEMAS, -}; -use risingwave_pb::catalog::PbDatabase; +use risingwave_common::catalog::{DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS}; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use sea_orm::{ ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, Database, DatabaseConnection, @@ -58,64 +53,13 @@ impl From for MetaError { pub struct MetadataManager { env: MetaSrvEnv, - conn: DatabaseConnection, + db: DatabaseConnection, } impl MetadataManager { - pub async fn new(env: MetaSrvEnv, conn: DatabaseConnection) -> MetaResult { - //FIXME: for test only, to remove. - // let dbs: Vec = prelude::Database::find().all(&conn).await.unwrap(); - // dbs.into_iter().for_each(|db| { - // println!("db: {:?}", db); - // }); - let mgr = Self { env, conn }; - mgr.init_user().await?; - mgr.init_database().await?; - Ok(mgr) - } - - // FIXME: should be done in init.sql scripts. - async fn init_user(&self) -> MetaResult<()> { - for (id, name) in [ - (DEFAULT_SUPER_USER_ID, DEFAULT_SUPER_USER), - (DEFAULT_SUPER_USER_FOR_PG_ID, DEFAULT_SUPER_USER_FOR_PG), - ] { - if prelude::User::find_by_id(id as i32) - .one(&self.conn) - .await? - .is_some() - { - continue; - } - let user = model_v2::user::ActiveModel { - user_id: ActiveValue::Set(id as _), - name: ActiveValue::Set(name.into()), - is_super: ActiveValue::Set(Some(true)), - can_create_db: ActiveValue::Set(Some(true)), - can_create_user: ActiveValue::Set(Some(true)), - can_login: ActiveValue::Set(Some(true)), - ..Default::default() - }; - user.insert(&self.conn).await?; - } - Ok(()) - } - - // FIXME: should be done in init.sql scripts. - async fn init_database(&self) -> MetaResult<()> { - if prelude::Database::find() - .filter(model_v2::database::Column::Name.eq(DEFAULT_DATABASE_NAME)) - .one(&self.conn) - .await? - .is_some() - { - return Ok(()); - } - let mut db = model_v2::database::ActiveModel::new(); - db.name = ActiveValue::Set(DEFAULT_DATABASE_NAME.into()); - db.owner_id = ActiveValue::Set(DEFAULT_SUPER_USER_ID as _); - self.create_database(db).await?; - Ok(()) + pub async fn new(env: MetaSrvEnv, url: &str) -> MetaResult { + let db = Database::connect(url).await?; + Ok(Self { env, db }) } } @@ -139,7 +83,7 @@ impl MetadataManager { &self, db: model_v2::database::ActiveModel, ) -> MetaResult { - let txn = self.conn.begin().await?; + let txn = self.db.begin().await?; let db = db.insert(&txn).await?; let mut schemas = vec![]; for schema_name in iter::once(DEFAULT_SCHEMA_NAME).chain(SYSTEM_SCHEMAS) { @@ -165,18 +109,15 @@ impl MetadataManager { // todo: return all streaming jobs. pub async fn drop_database(&self, database_id: DatabaseId) -> MetaResult<()> { - let tables = prelude::Table::find() + let _tables = prelude::Table::find() .filter(model_v2::table::Column::DatabaseId.eq(database_id as i32)) - .all(&self.conn) + .all(&self.db) .await?; // 1. unregister source. // 2. fragments + actors, streaming manager drop streaming job. // 3. connection to drop. - // let fragments = prelude::Fragment:: - // - - let txn = self.conn.begin().await?; + let txn = self.db.begin().await?; let db: model_v2::database::ActiveModel = prelude::Database::find_by_id(database_id as i32) .one(&txn) .await? @@ -224,18 +165,18 @@ impl MetadataManager { #[cfg(test)] mod tests { - use sea_orm::{DbBackend, MockDatabase}; use super::*; #[tokio::test] + #[ignore] async fn test_create_database() { - let conn = Database::connect("postgres://postgres:@localhost:5432/postgres") - .await - .unwrap(); // let conn = MockDatabase::new(DbBackend::Postgres).into_connection(); - let mgr = MetadataManager::new(MetaSrvEnv::for_test().await, conn) - .await - .unwrap(); + let mgr = MetadataManager::new( + MetaSrvEnv::for_test().await, + "postgres://postgres:@localhost:5432/postgres", + ) + .await + .unwrap(); let db = model_v2::database::ActiveModel { name: ActiveValue::Set("test".into()), owner_id: ActiveValue::Set(1), @@ -244,7 +185,7 @@ mod tests { mgr.create_database(db).await.unwrap(); let db = prelude::Database::find() .filter(model_v2::database::Column::Name.eq("test")) - .one(&mgr.conn) + .one(&mgr.db) .await .unwrap() .unwrap(); diff --git a/src/meta/src/model_v2/migration/src/lib.rs b/src/meta/src/model_v2/migration/src/lib.rs index 3abf4a994c17d..6f5c6f3a041ae 100644 --- a/src/meta/src/model_v2/migration/src/lib.rs +++ b/src/meta/src/model_v2/migration/src/lib.rs @@ -7,8 +7,6 @@ pub struct Migrator; #[async_trait::async_trait] impl MigratorTrait for Migrator { fn migrations() -> Vec> { - vec![ - Box::new(m20230908_072257_init::Migration), - ] + vec![Box::new(m20230908_072257_init::Migration)] } } diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs index 5180e77baa95c..58f737e1e3dc5 100644 --- a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -1,51 +1,1137 @@ -use sea_orm_migration::prelude::*; +use sea_orm_migration::prelude::{Index as MigrationIndex, Table as MigrationTable, *}; #[derive(DeriveMigrationName)] pub struct Migration; -const INIT_SQL: &str = r#" - -"#; - #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - // Replace the sample below with your own migration scripts - todo!(); + // 1. check if the table exists. + assert!(!manager.has_table(Worker::Table.to_string()).await?); + assert!(!manager.has_table(WorkerProperty::Table.to_string()).await?); + assert!(!manager.has_table(User::Table.to_string()).await?); + assert!(!manager.has_table(UserPrivilege::Table.to_string()).await?); + assert!(!manager.has_table(Database::Table.to_string()).await?); + assert!(!manager.has_table(Schema::Table.to_string()).await?); + assert!(!manager.has_table(Fragment::Table.to_string()).await?); + assert!(!manager.has_table(Actor::Table.to_string()).await?); + assert!(!manager.has_table(Table::Table.to_string()).await?); + assert!(!manager.has_table(Source::Table.to_string()).await?); + assert!(!manager.has_table(Sink::Table.to_string()).await?); + assert!(!manager.has_table(Connection::Table.to_string()).await?); + assert!(!manager.has_table(View::Table.to_string()).await?); + assert!(!manager.has_table(Index::Table.to_string()).await?); + assert!(!manager.has_table(Function::Table.to_string()).await?); + assert!(!manager.has_table(Object::Table.to_string()).await?); + assert!( + !manager + .has_table(ObjectDependency::Table.to_string()) + .await? + ); + // 2. create tables. manager .create_table( - Table::create() - .table(Post::Table) - .if_not_exists() + MigrationTable::create() + .table(Worker::Table) .col( - ColumnDef::new(Post::Id) + ColumnDef::new(Worker::WorkerId) .integer() .not_null() .auto_increment() .primary_key(), ) - .col(ColumnDef::new(Post::Title).string().not_null()) - .col(ColumnDef::new(Post::Text).string().not_null()) + .col(ColumnDef::new(Worker::WorkerType).string().not_null()) + .col(ColumnDef::new(Worker::Host).string().not_null()) + .col(ColumnDef::new(Worker::Port).integer().not_null()) + .col(ColumnDef::new(Worker::Status).string().not_null()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(WorkerProperty::Table) + .col( + ColumnDef::new(WorkerProperty::WorkerId) + .integer() + .primary_key(), + ) + .col( + ColumnDef::new(WorkerProperty::ParallelUnitIds) + .array(ColumnType::Integer) + .not_null(), + ) + .col( + ColumnDef::new(WorkerProperty::IsStreaming) + .boolean() + .not_null(), + ) + .col( + ColumnDef::new(WorkerProperty::IsServing) + .boolean() + .not_null(), + ) + .col( + ColumnDef::new(WorkerProperty::IsUnschedulable) + .boolean() + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(User::Table) + .col(ColumnDef::new(User::UserId).integer().primary_key()) + .col(ColumnDef::new(User::Name).string().not_null()) + .col(ColumnDef::new(User::IsSuper).boolean().not_null()) + .col(ColumnDef::new(User::CanCreateDb).boolean().not_null()) + .col(ColumnDef::new(User::CanCreateUser).boolean().not_null()) + .col(ColumnDef::new(User::CanLogin).boolean().not_null()) + .col(ColumnDef::new(User::AuthType).string()) + .col(ColumnDef::new(User::AuthValue).string()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(UserPrivilege::Table) + .col( + ColumnDef::new(UserPrivilege::Id) + .integer() + .primary_key() + .auto_increment(), + ) + .col(ColumnDef::new(UserPrivilege::UserId).integer().not_null()) + .col(ColumnDef::new(UserPrivilege::Oid).integer().not_null()) + .col( + ColumnDef::new(UserPrivilege::GrantedBy) + .integer() + .not_null(), + ) + .col( + ColumnDef::new(UserPrivilege::Actions) + .array(ColumnType::String(None)) + .not_null(), + ) + .col( + ColumnDef::new(UserPrivilege::WithGrantOption) + .boolean() + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Database::Table) + .col( + ColumnDef::new(Database::DatabaseId) + .integer() + .auto_increment() + .primary_key(), + ) + .col( + ColumnDef::new(Database::Name) + .string() + .unique_key() + .not_null(), + ) + .col(ColumnDef::new(Database::OwnerId).integer().not_null()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Schema::Table) + .col( + ColumnDef::new(Schema::SchemaId) + .integer() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(Schema::Name).string().not_null()) + .col(ColumnDef::new(Schema::DatabaseId).integer().not_null()) + .col(ColumnDef::new(Schema::OwnerId).integer().not_null()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Fragment::Table) + .col(ColumnDef::new(Fragment::FragmentId).integer().primary_key()) + .col(ColumnDef::new(Fragment::TableId).integer().not_null()) + .col( + ColumnDef::new(Fragment::FragmentTypeMask) + .integer() + .not_null(), + ) + .col( + ColumnDef::new(Fragment::DistributionType) + .string() + .not_null(), + ) + .col(ColumnDef::new(Fragment::StreamNode).json().not_null()) + .col(ColumnDef::new(Fragment::VnodeMapping).json()) + .col(ColumnDef::new(Fragment::StateTableIds).array(ColumnType::Integer)) + .col(ColumnDef::new(Fragment::UpstreamFragmentId).array(ColumnType::Integer)) + .col(ColumnDef::new(Fragment::DispatcherType).string()) + .col(ColumnDef::new(Fragment::DistKeyIndices).array(ColumnType::Integer)) + .col(ColumnDef::new(Fragment::OutputIndices).array(ColumnType::Integer)) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Actor::Table) + .col(ColumnDef::new(Actor::ActorId).integer().primary_key()) + .col(ColumnDef::new(Actor::FragmentId).integer().not_null()) + .col(ColumnDef::new(Actor::Status).string()) + .col(ColumnDef::new(Actor::Splits).json()) + .col(ColumnDef::new(Actor::ParallelUnitId).integer().not_null()) + .col(ColumnDef::new(Actor::UpstreamActorIds).array(ColumnType::Integer)) + .col(ColumnDef::new(Actor::Dispatchers).json()) + .col(ColumnDef::new(Actor::VnodeBitmap).string()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Table::Table) + .col(ColumnDef::new(Table::TableId).integer().primary_key()) + .col(ColumnDef::new(Table::Name).string().not_null()) + .col(ColumnDef::new(Table::SchemaId).integer().not_null()) + .col(ColumnDef::new(Table::DatabaseId).integer().not_null()) + .col(ColumnDef::new(Table::OwnerId).integer().not_null()) + .col(ColumnDef::new(Table::OptionalAssociatedSourceId).integer()) + .col(ColumnDef::new(Table::TableType).string()) + .col(ColumnDef::new(Table::Columns).json()) + .col(ColumnDef::new(Table::Pk).json()) + .col(ColumnDef::new(Table::DistributionKey).array(ColumnType::Integer)) + .col(ColumnDef::new(Table::AppendOnly).boolean()) + .col(ColumnDef::new(Table::Properties).json()) + .col(ColumnDef::new(Table::FragmentId).integer()) + .col(ColumnDef::new(Table::VnodeColIndex).integer()) + .col(ColumnDef::new(Table::ValueIndices).array(ColumnType::Integer)) + .col(ColumnDef::new(Table::Definition).string()) + .col(ColumnDef::new(Table::HandlePkConflictBehavior).integer()) + .col(ColumnDef::new(Table::ReadPrefixLenHint).integer()) + .col(ColumnDef::new(Table::WatermarkIndices).array(ColumnType::Integer)) + .col(ColumnDef::new(Table::DistKeyInPk).array(ColumnType::Integer)) + .col(ColumnDef::new(Table::DmlFragmentId).integer()) + .col(ColumnDef::new(Table::Cardinality).array(ColumnType::Integer)) + .col(ColumnDef::new(Table::InitializedAtEpoch).integer()) + .col(ColumnDef::new(Table::CreatedAtEpoch).integer()) + .col(ColumnDef::new(Table::CleanedByWatermark).boolean()) + .col(ColumnDef::new(Table::Version).array(ColumnType::Integer)) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Source::Table) + .col(ColumnDef::new(Source::SourceId).integer().primary_key()) + .col(ColumnDef::new(Source::Name).string().not_null()) + .col(ColumnDef::new(Source::SchemaId).integer().not_null()) + .col(ColumnDef::new(Source::DatabaseId).integer().not_null()) + .col(ColumnDef::new(Source::OwnerId).integer().not_null()) + .col(ColumnDef::new(Source::RowIdIndex).string()) + .col(ColumnDef::new(Source::Columns).json()) + .col(ColumnDef::new(Source::PkColumnIds).json()) + .col(ColumnDef::new(Source::Properties).json()) + .col(ColumnDef::new(Source::Definition).string()) + .col(ColumnDef::new(Source::SourceInfo).json()) + .col(ColumnDef::new(Source::WatermarkDescs).json()) + .col(ColumnDef::new(Source::OptionalAssociatedTableId).integer()) + .col(ColumnDef::new(Source::ConnectionId).integer()) + .col(ColumnDef::new(Source::InitializedAtEpoch).integer()) + .col(ColumnDef::new(Source::CreatedAtEpoch).integer()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Sink::Table) + .col(ColumnDef::new(Sink::SinkId).integer().primary_key()) + .col(ColumnDef::new(Sink::Name).string().not_null()) + .col(ColumnDef::new(Sink::SchemaId).integer().not_null()) + .col(ColumnDef::new(Sink::DatabaseId).integer().not_null()) + .col(ColumnDef::new(Sink::OwnerId).integer().not_null()) + .col(ColumnDef::new(Sink::Columns).json()) + .col(ColumnDef::new(Sink::PkColumnIds).json()) + .col(ColumnDef::new(Sink::DistributionKey).array(ColumnType::Integer)) + .col(ColumnDef::new(Sink::DownstreamPk).array(ColumnType::Integer)) + .col(ColumnDef::new(Sink::SinkType).string()) + .col(ColumnDef::new(Sink::Properties).json()) + .col(ColumnDef::new(Sink::Definition).string()) + .col(ColumnDef::new(Sink::ConnectionId).integer()) + .col(ColumnDef::new(Sink::InitializedAtEpoch).integer()) + .col(ColumnDef::new(Sink::CreatedAtEpoch).integer()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Connection::Table) + .col( + ColumnDef::new(Connection::ConnectionId) + .integer() + .primary_key(), + ) + .col(ColumnDef::new(Connection::Name).string().not_null()) + .col(ColumnDef::new(Connection::SchemaId).integer().not_null()) + .col(ColumnDef::new(Connection::DatabaseId).integer().not_null()) + .col(ColumnDef::new(Connection::OwnerId).integer().not_null()) + .col(ColumnDef::new(Connection::Info).json()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(View::Table) + .col(ColumnDef::new(View::ViewId).integer().primary_key()) + .col(ColumnDef::new(View::Name).string().not_null()) + .col(ColumnDef::new(View::SchemaId).integer().not_null()) + .col(ColumnDef::new(View::DatabaseId).integer().not_null()) + .col(ColumnDef::new(View::OwnerId).integer().not_null()) + .col(ColumnDef::new(View::Properties).json()) + .col(ColumnDef::new(View::Sql).string()) + .col(ColumnDef::new(View::Columns).json()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Index::Table) + .col(ColumnDef::new(Index::IndexId).integer().primary_key()) + .col(ColumnDef::new(Index::Name).string().not_null()) + .col(ColumnDef::new(Index::SchemaId).integer().not_null()) + .col(ColumnDef::new(Index::DatabaseId).integer().not_null()) + .col(ColumnDef::new(Index::OwnerId).integer().not_null()) + .col(ColumnDef::new(Index::IndexTableId).integer().not_null()) + .col(ColumnDef::new(Index::PrimaryTableId).integer().not_null()) + .col(ColumnDef::new(Index::IndexItems).json()) + .col(ColumnDef::new(Index::OriginalColumns).array(ColumnType::Integer)) + .col(ColumnDef::new(Index::InitializedAtEpoch).integer()) + .col(ColumnDef::new(Index::CreatedAtEpoch).integer()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Function::Table) + .col(ColumnDef::new(Function::FunctionId).integer().primary_key()) + .col(ColumnDef::new(Function::Name).string().not_null()) + .col(ColumnDef::new(Function::SchemaId).integer().not_null()) + .col(ColumnDef::new(Function::DatabaseId).integer().not_null()) + .col(ColumnDef::new(Function::OwnerId).integer().not_null()) + .col(ColumnDef::new(Function::ArgTypes).json()) + .col(ColumnDef::new(Function::ReturnType).string()) + .col(ColumnDef::new(Function::Language).string()) + .col(ColumnDef::new(Function::Link).string()) + .col(ColumnDef::new(Function::Identifier).string()) + .col(ColumnDef::new(Function::Kind).json()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Object::Table) + .col(ColumnDef::new(Object::Oid).integer().primary_key()) + .col(ColumnDef::new(Object::ObjType).string().not_null()) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(ObjectDependency::Table) + .col(ColumnDef::new(ObjectDependency::Id).integer().primary_key()) + .col(ColumnDef::new(ObjectDependency::Oid).integer().not_null()) + .col( + ColumnDef::new(ObjectDependency::UsedBy) + .integer() + .not_null(), + ) + .to_owned(), + ) + .await?; + + // 3. create indexes. + manager + .create_index( + MigrationIndex::create() + .table(Worker::Table) + .unique() + .col(Worker::Host) + .col(Worker::Port) + .to_owned(), + ) + .await?; + manager + .create_index( + MigrationIndex::create() + .table(Schema::Table) + .unique() + .col(Schema::DatabaseId) + .col(Schema::Name) + .to_owned(), + ) + .await?; + + // 4. create foreign keys. + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_worker_property_worker_id") + .from(WorkerProperty::Table, WorkerProperty::WorkerId) + .to(Worker::Table, Worker::WorkerId) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_user_privilege_user_id") + .from(UserPrivilege::Table, UserPrivilege::UserId) + .to(User::Table, User::UserId) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_user_privilege_granted_by") + .from(UserPrivilege::Table, UserPrivilege::GrantedBy) + .to(User::Table, User::UserId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_user_privilege_oid") + .from(UserPrivilege::Table, UserPrivilege::Oid) + .to(Object::Table, Object::Oid) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_database_owner_id") + .from(Database::Table, Database::OwnerId) + .to(User::Table, User::UserId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_schema_database_id") + .from(Schema::Table, Schema::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_schema_owner_id") + .from(Schema::Table, Schema::OwnerId) + .to(User::Table, User::UserId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_fragment_table_id") + .from(Fragment::Table, Fragment::TableId) + .to(Table::Table, Table::TableId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_actor_fragment_id") + .from(Actor::Table, Actor::FragmentId) + .to(Fragment::Table, Fragment::FragmentId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_table_schema_id") + .from(Table::Table, Table::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_table_database_id") + .from(Table::Table, Table::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_table_owner_id") + .from(Table::Table, Table::OwnerId) + .to(User::Table, User::UserId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_table_fragment_id") + .from(Table::Table, Table::FragmentId) + .to(Fragment::Table, Fragment::FragmentId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_table_dml_fragment_id") + .from(Table::Table, Table::DmlFragmentId) + .to(Fragment::Table, Fragment::FragmentId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_table_optional_associated_source_id") + .from(Table::Table, Table::OptionalAssociatedSourceId) + .to(Source::Table, Source::SourceId) .to_owned(), ) - .await + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_source_schema_id") + .from(Source::Table, Source::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_source_database_id") + .from(Source::Table, Source::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_source_owner_id") + .from(Source::Table, Source::OwnerId) + .to(User::Table, User::UserId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_source_connection_id") + .from(Source::Table, Source::ConnectionId) + .to(Connection::Table, Connection::ConnectionId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_source_optional_associated_table_id") + .from(Source::Table, Source::OptionalAssociatedTableId) + .to(Table::Table, Table::TableId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_sink_schema_id") + .from(Sink::Table, Sink::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_sink_database_id") + .from(Sink::Table, Sink::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_sink_owner_id") + .from(Sink::Table, Sink::OwnerId) + .to(User::Table, User::UserId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_sink_connection_id") + .from(Sink::Table, Sink::ConnectionId) + .to(Connection::Table, Connection::ConnectionId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_connection_schema_id") + .from(Connection::Table, Connection::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_connection_database_id") + .from(Connection::Table, Connection::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_connection_owner_id") + .from(Connection::Table, Connection::OwnerId) + .to(User::Table, User::UserId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_view_schema_id") + .from(View::Table, View::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_view_database_id") + .from(View::Table, View::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_view_owner_id") + .from(View::Table, View::OwnerId) + .to(User::Table, User::UserId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_index_schema_id") + .from(Index::Table, Index::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_index_database_id") + .from(Index::Table, Index::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_index_owner_id") + .from(Index::Table, Index::OwnerId) + .to(User::Table, User::UserId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_index_index_table_id") + .from(Index::Table, Index::IndexTableId) + .to(Table::Table, Table::TableId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_index_primary_table_id") + .from(Index::Table, Index::PrimaryTableId) + .to(Table::Table, Table::TableId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_function_schema_id") + .from(Function::Table, Function::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_function_database_id") + .from(Function::Table, Function::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_function_owner_id") + .from(Function::Table, Function::OwnerId) + .to(User::Table, User::UserId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_object_oid_table") + .from(Object::Table, Object::Oid) + .to(Table::Table, Table::TableId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_object_oid_source") + .from(Object::Table, Object::Oid) + .to(Source::Table, Source::SourceId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_object_oid_sink") + .from(Object::Table, Object::Oid) + .to(Sink::Table, Sink::SinkId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_object_oid_index") + .from(Object::Table, Object::Oid) + .to(Index::Table, Index::IndexId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_object_oid_view") + .from(Object::Table, Object::Oid) + .to(View::Table, View::ViewId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_object_oid_connection") + .from(Object::Table, Object::Oid) + .to(Connection::Table, Connection::ConnectionId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_object_oid_function") + .from(Object::Table, Object::Oid) + .to(Function::Table, Function::FunctionId) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_object_dependency_oid") + .from(ObjectDependency::Table, ObjectDependency::Oid) + .to(Object::Table, Object::Oid) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_object_dependency_used_by") + .from(ObjectDependency::Table, ObjectDependency::UsedBy) + .to(Object::Table, Object::Oid) + .to_owned(), + ) + .await?; + + // 4. initialize data. + let insert_sys_users = Query::insert() + .into_table(User::Table) + .columns([ + User::UserId, + User::Name, + User::IsSuper, + User::CanCreateUser, + User::CanCreateDb, + User::CanLogin, + ]) + .values_panic([ + 1.into(), + "root".into(), + true.into(), + true.into(), + true.into(), + true.into(), + ]) + .values_panic([ + 2.into(), + "postgres".into(), + true.into(), + true.into(), + true.into(), + true.into(), + ]) + .to_owned(); + let insert_sys_database = Query::insert() + .into_table(Database::Table) + .columns([Database::Name, Database::OwnerId]) + .values_panic(["dev".into(), 1.into()]) + .to_owned(); + manager.exec_stmt(insert_sys_users).await?; + manager.exec_stmt(insert_sys_database).await?; + + Ok(()) } async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - // Replace the sample below with your own migration scripts - todo!(); + macro_rules! drop_tables { + ($manager:expr, $( $table:ident ),+) => { + $( + $manager + .drop_table( + MigrationTable::drop() + .table($table::Table) + .if_exists() + .cascade() + .to_owned(), + ) + .await?; + )+ + }; + } - manager - .drop_table(Table::drop().table(Post::Table).to_owned()) - .await + // drop tables cascade. + drop_tables!( + manager, + Worker, + WorkerProperty, + User, + UserPrivilege, + Database, + Schema, + Fragment, + Actor, + Table, + Source, + Sink, + Connection, + View, + Index, + Function, + Object, + ObjectDependency + ); + Ok(()) } } #[derive(DeriveIden)] -enum Post { +enum Worker { + Table, + WorkerId, + WorkerType, + Host, + Port, + Status, +} + +#[derive(DeriveIden)] +enum WorkerProperty { + Table, + WorkerId, + ParallelUnitIds, + IsStreaming, + IsServing, + IsUnschedulable, +} + +#[derive(DeriveIden)] +enum User { + Table, + UserId, + Name, + IsSuper, + CanCreateDb, + CanCreateUser, + CanLogin, + AuthType, + AuthValue, +} + +#[derive(DeriveIden)] +enum UserPrivilege { + Table, + Id, + UserId, + Oid, + GrantedBy, + Actions, + WithGrantOption, +} + +#[derive(DeriveIden)] +enum Database { + Table, + DatabaseId, + Name, + OwnerId, +} + +#[derive(DeriveIden)] +enum Schema { + Table, + SchemaId, + Name, + DatabaseId, + OwnerId, +} + +#[derive(DeriveIden)] +enum Fragment { + Table, + FragmentId, + TableId, + FragmentTypeMask, + DistributionType, + StreamNode, + VnodeMapping, + StateTableIds, + UpstreamFragmentId, + DispatcherType, + DistKeyIndices, + OutputIndices, +} + +#[derive(DeriveIden)] +enum Actor { + Table, + ActorId, + FragmentId, + Status, + Splits, + ParallelUnitId, + UpstreamActorIds, + Dispatchers, + VnodeBitmap, +} + +#[derive(DeriveIden)] +#[allow(clippy::enum_variant_names)] +enum Table { + Table, + TableId, + Name, + SchemaId, + DatabaseId, + OwnerId, + OptionalAssociatedSourceId, + TableType, + Columns, + Pk, + DistributionKey, + AppendOnly, + Properties, + FragmentId, + VnodeColIndex, + ValueIndices, + Definition, + HandlePkConflictBehavior, + ReadPrefixLenHint, + WatermarkIndices, + DistKeyInPk, + DmlFragmentId, + Cardinality, + InitializedAtEpoch, + CreatedAtEpoch, + CleanedByWatermark, + Version, +} + +#[derive(DeriveIden)] +enum Source { + Table, + SourceId, + Name, + SchemaId, + DatabaseId, + OwnerId, + RowIdIndex, + Columns, + PkColumnIds, + Properties, + Definition, + SourceInfo, + WatermarkDescs, + OptionalAssociatedTableId, + ConnectionId, + InitializedAtEpoch, + CreatedAtEpoch, +} + +#[derive(DeriveIden)] +enum Sink { + Table, + SinkId, + Name, + SchemaId, + DatabaseId, + OwnerId, + Columns, + PkColumnIds, + DistributionKey, + DownstreamPk, + SinkType, + Properties, + Definition, + ConnectionId, + InitializedAtEpoch, + CreatedAtEpoch, +} + +#[derive(DeriveIden)] +enum Connection { + Table, + ConnectionId, + Name, + SchemaId, + DatabaseId, + OwnerId, + Info, +} + +#[derive(DeriveIden)] +enum View { + Table, + ViewId, + Name, + SchemaId, + DatabaseId, + OwnerId, + Properties, + Sql, + Columns, +} + +#[derive(DeriveIden)] +enum Index { + Table, + IndexId, + Name, + SchemaId, + DatabaseId, + OwnerId, + IndexTableId, + PrimaryTableId, + IndexItems, + OriginalColumns, + InitializedAtEpoch, + CreatedAtEpoch, +} + +#[derive(DeriveIden)] +enum Function { + Table, + FunctionId, + Name, + SchemaId, + DatabaseId, + OwnerId, + ArgTypes, + ReturnType, + Language, + Link, + Identifier, + Kind, +} + +#[derive(DeriveIden)] +enum Object { + Table, + Oid, + ObjType, +} + +#[derive(DeriveIden)] +enum ObjectDependency { Table, Id, - Title, - Text, + Oid, + UsedBy, } diff --git a/src/meta/src/model_v2/mod.rs b/src/meta/src/model_v2/mod.rs index 76277e8722c16..f37234ca5c34f 100644 --- a/src/meta/src/model_v2/mod.rs +++ b/src/meta/src/model_v2/mod.rs @@ -1,7 +1,6 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 use risingwave_pb::catalog::{PbDatabase, PbSchema}; -use risingwave_pb::user::PbUserInfo; pub mod prelude; From a9bfed0af9d57b62f08ca97eba55a9893b53b22f Mon Sep 17 00:00:00 2001 From: August Date: Tue, 12 Sep 2023 13:37:05 +0800 Subject: [PATCH 04/30] init catalog controller --- src/meta/src/controller/catalog_controller.rs | 190 +++++++++++++++++ src/meta/src/controller/mod.rs | 75 +++++++ src/meta/src/lib.rs | 2 + src/meta/src/manager/metadata.rs | 194 ------------------ src/meta/src/manager/mod.rs | 2 - src/meta/src/model_v2/actor.rs | 2 - src/meta/src/model_v2/connection.rs | 1 - src/meta/src/model_v2/fragment.rs | 8 +- src/meta/src/model_v2/function.rs | 2 - src/meta/src/model_v2/index.rs | 1 - src/meta/src/model_v2/mod.rs | 38 ---- src/meta/src/model_v2/object.rs | 2 +- src/meta/src/model_v2/sink.rs | 3 - src/meta/src/model_v2/source.rs | 5 - src/meta/src/model_v2/table.rs | 3 - src/meta/src/model_v2/user.rs | 9 +- src/meta/src/model_v2/user_privilege.rs | 4 +- src/meta/src/model_v2/view.rs | 2 - src/meta/src/model_v2/worker.rs | 4 +- src/meta/src/model_v2/worker_property.rs | 8 +- 20 files changed, 284 insertions(+), 271 deletions(-) create mode 100644 src/meta/src/controller/catalog_controller.rs create mode 100644 src/meta/src/controller/mod.rs delete mode 100644 src/meta/src/manager/metadata.rs diff --git a/src/meta/src/controller/catalog_controller.rs b/src/meta/src/controller/catalog_controller.rs new file mode 100644 index 0000000000000..5c704da334d05 --- /dev/null +++ b/src/meta/src/controller/catalog_controller.rs @@ -0,0 +1,190 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::iter; + +use risingwave_common::catalog::{DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS}; +use risingwave_pb::catalog::PbDatabase; +use risingwave_pb::meta::subscribe_response::{ + Info as NotificationInfo, Operation as NotificationOperation, +}; +use sea_orm::{ + ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, Database as SeaDB, + DatabaseConnection, EntityTrait, QueryFilter, TransactionTrait, +}; +use tokio::sync::RwLock; + +use crate::manager::{DatabaseId, MetaSrvEnv, NotificationVersion}; +use crate::model_v2::prelude::*; +use crate::model_v2::{connection, database, function, index, schema, sink, source, table, view}; +use crate::{MetaError, MetaResult}; + +/// `CatalogController` is the controller for catalog related operations, including database, schema, table, view, etc. +pub struct CatalogController { + env: MetaSrvEnv, + db: DatabaseConnection, + // todo: replace it with monotonic timestamp. + revision: RwLock, +} + +impl CatalogController { + pub async fn new(env: MetaSrvEnv, url: &str) -> MetaResult { + let db = SeaDB::connect(url).await?; + Ok(Self { + env, + db, + revision: RwLock::new(0), + }) + } +} + +impl CatalogController { + pub async fn get_revision(&self) -> u64 { + *self.revision.read().await + } + + async fn notify_frontend( + &self, + operation: NotificationOperation, + info: NotificationInfo, + ) -> NotificationVersion { + self.env + .notification_manager() + .notify_frontend(operation, info) + .await + } +} + +impl CatalogController { + pub async fn snapshot(&self) -> MetaResult> { + let dbs = Database::find().all(&self.db).await?; + Ok(dbs.into_iter().map(|db| db.into()).collect()) + } + + pub async fn create_database(&self, db: PbDatabase) -> MetaResult { + let txn = self.db.begin().await?; + let db: database::ActiveModel = db.into(); + let db = db.insert(&txn).await?; + let mut schemas = vec![]; + for schema_name in iter::once(DEFAULT_SCHEMA_NAME).chain(SYSTEM_SCHEMAS) { + let mut schema = schema::ActiveModel::new(); + schema.database_id = ActiveValue::Set(db.database_id); + schema.name = ActiveValue::Set(schema_name.into()); + schema.owner_id = ActiveValue::Set(db.owner_id); + schemas.push(schema.insert(&txn).await?); + } + txn.commit().await?; + + let mut version = self + .notify_frontend( + NotificationOperation::Add, + NotificationInfo::Database(db.into()), + ) + .await; + for schema in schemas { + version = self + .notify_frontend( + NotificationOperation::Add, + NotificationInfo::Schema(schema.into()), + ) + .await; + } + + Ok(version) + } + + pub async fn drop_database(&self, database_id: DatabaseId) -> MetaResult<()> { + let _tables = Table::find() + .filter(table::Column::DatabaseId.eq(database_id as i32)) + .all(&self.db) + .await?; + // 1. unregister source. + // 2. fragments + actors, streaming manager drop streaming job. + // 3. connection to drop. + + let txn = self.db.begin().await?; + let db: database::ActiveModel = Database::find_by_id(database_id as i32) + .one(&txn) + .await? + .ok_or_else(|| MetaError::catalog_id_not_found("database", database_id))? + .into(); + + Table::delete_many() + .filter(table::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + Source::delete_many() + .filter(source::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + Sink::delete_many() + .filter(sink::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + Index::delete_many() + .filter(index::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + Function::delete_many() + .filter(function::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + Connection::delete_many() + .filter(connection::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + View::delete_many() + .filter(view::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + Schema::delete_many() + .filter(schema::Column::DatabaseId.eq(database_id as i32)) + .exec(&txn) + .await?; + Database::delete(db).exec(&txn).await?; + + txn.commit().await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + #[ignore] + async fn test_create_database() { + // let conn = MockDatabase::new(DbBackend::Postgres).into_connection(); + let mgr = CatalogController::new( + MetaSrvEnv::for_test().await, + "postgres://postgres:@localhost:5432/postgres", + ) + .await + .unwrap(); + let db = PbDatabase { + name: "test".to_string(), + owner: 1, + ..Default::default() + }; + mgr.create_database(db).await.unwrap(); + let db = Database::find() + .filter(database::Column::Name.eq("test")) + .one(&mgr.db) + .await + .unwrap() + .unwrap(); + mgr.drop_database(db.database_id as _).await.unwrap(); + } +} diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs new file mode 100644 index 0000000000000..c12eace7c9e18 --- /dev/null +++ b/src/meta/src/controller/mod.rs @@ -0,0 +1,75 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::anyhow; +use risingwave_pb::catalog::{PbDatabase, PbSchema}; +use sea_orm::ActiveValue; + +use crate::model_v2::{database, schema}; +use crate::MetaError; + +mod catalog_controller; + +// todo: refine the error transform. +impl From for MetaError { + fn from(err: sea_orm::DbErr) -> Self { + anyhow!(err).into() + } +} + +impl From for PbDatabase { + fn from(model: database::Model) -> Self { + Self { + id: model.database_id as _, + name: model.name, + owner: model.owner_id as _, + } + } +} + +impl From for database::ActiveModel { + fn from(db: PbDatabase) -> Self { + Self { + database_id: ActiveValue::Set(db.id as _), + name: ActiveValue::Set(db.name), + owner_id: ActiveValue::Set(db.owner as _), + } + } +} + +impl From for PbSchema { + fn from(model: schema::Model) -> Self { + Self { + id: model.schema_id as _, + name: model.name, + database_id: model.database_id as _, + owner: model.owner_id as _, + } + } +} + +// impl From for PbUserInfo { +// fn from(model: user::Model) -> Self { +// Self { +// id: model.user_id as _, +// name: model.name, +// is_super: model.is_super.unwrap_or_default(), +// can_create_db: model.can_create_db.unwrap_or_default(), +// can_create_user: model.can_create_user.unwrap_or_default(), +// can_login: model.can_login.unwrap_or_default(), +// auth_info: None, +// grant_privileges: vec![], +// } +// } +// } diff --git a/src/meta/src/lib.rs b/src/meta/src/lib.rs index 0f727d2f9aa51..6c3f068258892 100644 --- a/src/meta/src/lib.rs +++ b/src/meta/src/lib.rs @@ -36,6 +36,7 @@ pub mod backup_restore; mod barrier; +pub mod controller; #[cfg(not(madsim))] // no need in simulation test mod dashboard; mod error; @@ -48,6 +49,7 @@ pub(crate) mod serving; pub mod storage; mod stream; pub(crate) mod telemetry; + use std::time::Duration; use clap::Parser; diff --git a/src/meta/src/manager/metadata.rs b/src/meta/src/manager/metadata.rs deleted file mode 100644 index 015859065528d..0000000000000 --- a/src/meta/src/manager/metadata.rs +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2023 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter; - -use anyhow::anyhow; -use risingwave_common::catalog::{DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS}; -use risingwave_pb::meta::subscribe_response::{Info, Operation}; -use sea_orm::{ - ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, Database, DatabaseConnection, - EntityTrait, QueryFilter, TransactionTrait, -}; - -use crate::manager::{DatabaseId, MetaSrvEnv, NotificationVersion}; -use crate::model_v2::prelude; -use crate::{model_v2, MetaError, MetaResult}; - -// todo: refine the error transform. -impl From for MetaError { - fn from(err: sea_orm::DbErr) -> Self { - anyhow!(err).into() - } -} - -// enum Object { -// Database(database::Model), -// Schema(schema::Model), -// Table(schema::Model), -// User(user::Model), -// } -// -// impl Object { -// fn obj_type(&self) -> &str { -// match self { -// Object::Database(_) => "database", -// Object::Schema(_) => "schema", -// Object::Table(_) => "table", -// Object::User(_) => "user", -// } -// } -// } - -pub struct MetadataManager { - env: MetaSrvEnv, - db: DatabaseConnection, -} - -impl MetadataManager { - pub async fn new(env: MetaSrvEnv, url: &str) -> MetaResult { - let db = Database::connect(url).await?; - Ok(Self { env, db }) - } -} - -impl MetadataManager { - async fn notify_frontend(&self, operation: Operation, info: Info) -> NotificationVersion { - self.env - .notification_manager() - .notify_frontend(operation, info) - .await - } -} - -impl MetadataManager { - // pub async fn snapshot() -> MetaResult { - // //d,s,tdd + self.revision - // // [1,2,3] 5 - // // [1,2,3] 6 - // // 4 6 - // } - pub async fn create_database( - &self, - db: model_v2::database::ActiveModel, - ) -> MetaResult { - let txn = self.db.begin().await?; - let db = db.insert(&txn).await?; - let mut schemas = vec![]; - for schema_name in iter::once(DEFAULT_SCHEMA_NAME).chain(SYSTEM_SCHEMAS) { - let mut schema = model_v2::schema::ActiveModel::new(); - schema.database_id = ActiveValue::Set(db.database_id); - schema.name = ActiveValue::Set(schema_name.into()); - schema.owner_id = ActiveValue::Set(db.owner_id); - schemas.push(schema.insert(&txn).await?); - } - txn.commit().await?; - - let mut version = self - .notify_frontend(Operation::Add, Info::Database(db.into())) - .await; - for schema in schemas { - version = self - .notify_frontend(Operation::Add, Info::Schema(schema.into())) - .await; - } - - Ok(version) - } - - // todo: return all streaming jobs. - pub async fn drop_database(&self, database_id: DatabaseId) -> MetaResult<()> { - let _tables = prelude::Table::find() - .filter(model_v2::table::Column::DatabaseId.eq(database_id as i32)) - .all(&self.db) - .await?; - // 1. unregister source. - // 2. fragments + actors, streaming manager drop streaming job. - // 3. connection to drop. - - let txn = self.db.begin().await?; - let db: model_v2::database::ActiveModel = prelude::Database::find_by_id(database_id as i32) - .one(&txn) - .await? - .ok_or_else(|| anyhow!("database not found"))? - .into(); - - prelude::Table::delete_many() - .filter(model_v2::table::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - prelude::Source::delete_many() - .filter(model_v2::source::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - prelude::Sink::delete_many() - .filter(model_v2::sink::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - prelude::Index::delete_many() - .filter(model_v2::index::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - prelude::Function::delete_many() - .filter(model_v2::function::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - prelude::Connection::delete_many() - .filter(model_v2::connection::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - prelude::View::delete_many() - .filter(model_v2::view::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - prelude::Schema::delete_many() - .filter(model_v2::schema::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - prelude::Database::delete(db).exec(&txn).await?; - - txn.commit().await?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - #[ignore] - async fn test_create_database() { - // let conn = MockDatabase::new(DbBackend::Postgres).into_connection(); - let mgr = MetadataManager::new( - MetaSrvEnv::for_test().await, - "postgres://postgres:@localhost:5432/postgres", - ) - .await - .unwrap(); - let db = model_v2::database::ActiveModel { - name: ActiveValue::Set("test".into()), - owner_id: ActiveValue::Set(1), - ..Default::default() - }; - mgr.create_database(db).await.unwrap(); - let db = prelude::Database::find() - .filter(model_v2::database::Column::Name.eq("test")) - .one(&mgr.db) - .await - .unwrap() - .unwrap(); - mgr.drop_database(db.database_id as _).await.unwrap(); - } -} diff --git a/src/meta/src/manager/mod.rs b/src/meta/src/manager/mod.rs index 77e1c3fce6669..760c24e0df7bc 100644 --- a/src/meta/src/manager/mod.rs +++ b/src/meta/src/manager/mod.rs @@ -17,7 +17,6 @@ mod cluster; mod env; mod id; mod idle; -mod metadata; mod notification; pub(crate) mod sink_coordination; mod streaming_job; @@ -30,7 +29,6 @@ pub use env::MetaSrvEnv; pub(crate) use env::*; pub(crate) use id::*; pub(crate) use idle::*; -pub use metadata::*; pub(crate) use notification::*; pub use notification::{LocalNotification, MessageStatus, NotificationManagerRef}; pub(crate) use streaming_job::*; diff --git a/src/meta/src/model_v2/actor.rs b/src/meta/src/model_v2/actor.rs index ae3601eb1ec2b..b7107562e22f4 100644 --- a/src/meta/src/model_v2/actor.rs +++ b/src/meta/src/model_v2/actor.rs @@ -9,11 +9,9 @@ pub struct Model { pub actor_id: i32, pub fragment_id: i32, pub status: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub splits: Option, pub parallel_unit_id: i32, pub upstream_actor_ids: Option>, - #[sea_orm(column_type = "JsonBinary", nullable)] pub dispatchers: Option, pub vnode_bitmap: Option, } diff --git a/src/meta/src/model_v2/connection.rs b/src/meta/src/model_v2/connection.rs index 5a3ad066cd1b9..e9527ec24d2ec 100644 --- a/src/meta/src/model_v2/connection.rs +++ b/src/meta/src/model_v2/connection.rs @@ -11,7 +11,6 @@ pub struct Model { pub schema_id: i32, pub database_id: i32, pub owner_id: i32, - #[sea_orm(column_type = "JsonBinary", nullable)] pub info: Option, } diff --git a/src/meta/src/model_v2/fragment.rs b/src/meta/src/model_v2/fragment.rs index cbe691b8e0f27..0671e8da7439b 100644 --- a/src/meta/src/model_v2/fragment.rs +++ b/src/meta/src/model_v2/fragment.rs @@ -8,11 +8,9 @@ pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub fragment_id: i32, pub table_id: i32, - pub fragment_type_mask: Option, - pub distribution_type: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] - pub stream_node: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] + pub fragment_type_mask: i32, + pub distribution_type: String, + pub stream_node: Json, pub vnode_mapping: Option, pub state_table_ids: Option>, pub upstream_fragment_id: Option>, diff --git a/src/meta/src/model_v2/function.rs b/src/meta/src/model_v2/function.rs index abec747fa97f3..850fb402629c4 100644 --- a/src/meta/src/model_v2/function.rs +++ b/src/meta/src/model_v2/function.rs @@ -11,13 +11,11 @@ pub struct Model { pub schema_id: i32, pub database_id: i32, pub owner_id: i32, - #[sea_orm(column_type = "JsonBinary", nullable)] pub arg_types: Option, pub return_type: Option, pub language: Option, pub link: Option, pub identifier: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub kind: Option, } diff --git a/src/meta/src/model_v2/index.rs b/src/meta/src/model_v2/index.rs index 88268e47cdb07..6d93a6d2713f7 100644 --- a/src/meta/src/model_v2/index.rs +++ b/src/meta/src/model_v2/index.rs @@ -13,7 +13,6 @@ pub struct Model { pub owner_id: i32, pub index_table_id: i32, pub primary_table_id: i32, - #[sea_orm(column_type = "JsonBinary", nullable)] pub index_items: Option, pub original_columns: Option>, pub initialized_at_epoch: Option, diff --git a/src/meta/src/model_v2/mod.rs b/src/meta/src/model_v2/mod.rs index f37234ca5c34f..c798a5ac88ba1 100644 --- a/src/meta/src/model_v2/mod.rs +++ b/src/meta/src/model_v2/mod.rs @@ -1,7 +1,5 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 -use risingwave_pb::catalog::{PbDatabase, PbSchema}; - pub mod prelude; pub mod actor; @@ -21,39 +19,3 @@ pub mod user_privilege; pub mod view; pub mod worker; pub mod worker_property; - -impl From for PbDatabase { - fn from(model: database::Model) -> Self { - Self { - id: model.database_id as _, - name: model.name, - owner: model.owner_id as _, - } - } -} - -impl From for PbSchema { - fn from(model: schema::Model) -> Self { - Self { - id: model.schema_id as _, - name: model.name, - database_id: model.database_id as _, - owner: model.owner_id as _, - } - } -} - -// impl From for PbUserInfo { -// fn from(model: user::Model) -> Self { -// Self { -// id: model.user_id as _, -// name: model.name, -// is_super: model.is_super.unwrap_or_default(), -// can_create_db: model.can_create_db.unwrap_or_default(), -// can_create_user: model.can_create_user.unwrap_or_default(), -// can_login: model.can_login.unwrap_or_default(), -// auth_info: None, -// grant_privileges: vec![], -// } -// } -// } diff --git a/src/meta/src/model_v2/object.rs b/src/meta/src/model_v2/object.rs index ad14e1c7b2a13..dec1d79651c1d 100644 --- a/src/meta/src/model_v2/object.rs +++ b/src/meta/src/model_v2/object.rs @@ -7,7 +7,7 @@ use sea_orm::entity::prelude::*; pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub oid: i32, - pub obj_type: Option, + pub obj_type: String, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/sink.rs b/src/meta/src/model_v2/sink.rs index 87a4524030f27..7da386932e33e 100644 --- a/src/meta/src/model_v2/sink.rs +++ b/src/meta/src/model_v2/sink.rs @@ -11,14 +11,11 @@ pub struct Model { pub schema_id: i32, pub database_id: i32, pub owner_id: i32, - #[sea_orm(column_type = "JsonBinary", nullable)] pub columns: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub pk_column_ids: Option, pub distribution_key: Option>, pub downstream_pk: Option>, pub sink_type: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub properties: Option, pub definition: Option, pub connection_id: Option, diff --git a/src/meta/src/model_v2/source.rs b/src/meta/src/model_v2/source.rs index 0909b24f519f5..d404b906b9f9a 100644 --- a/src/meta/src/model_v2/source.rs +++ b/src/meta/src/model_v2/source.rs @@ -12,16 +12,11 @@ pub struct Model { pub database_id: i32, pub owner_id: i32, pub row_id_index: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub columns: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub pk_column_ids: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub properties: Option, pub definition: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub source_info: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub watermark_descs: Option, pub optional_associated_table_id: Option, pub connection_id: Option, diff --git a/src/meta/src/model_v2/table.rs b/src/meta/src/model_v2/table.rs index 1e21f75b39f19..25338e81a76a1 100644 --- a/src/meta/src/model_v2/table.rs +++ b/src/meta/src/model_v2/table.rs @@ -13,13 +13,10 @@ pub struct Model { pub owner_id: i32, pub optional_associated_source_id: Option, pub table_type: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub columns: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub pk: Option, pub distribution_key: Option>, pub append_only: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub properties: Option, pub fragment_id: Option, pub vnode_col_index: Option, diff --git a/src/meta/src/model_v2/user.rs b/src/meta/src/model_v2/user.rs index 1a99e25128d7d..96874e72af0b8 100644 --- a/src/meta/src/model_v2/user.rs +++ b/src/meta/src/model_v2/user.rs @@ -8,11 +8,12 @@ pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub user_id: i32, pub name: String, - pub is_super: Option, - pub can_create_db: Option, - pub can_create_user: Option, - pub can_login: Option, + pub is_super: bool, + pub can_create_db: bool, + pub can_create_user: bool, + pub can_login: bool, pub auth_type: Option, + pub auth_value: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/user_privilege.rs b/src/meta/src/model_v2/user_privilege.rs index 5f13cf9cb3bd6..e20b4dbe67a5b 100644 --- a/src/meta/src/model_v2/user_privilege.rs +++ b/src/meta/src/model_v2/user_privilege.rs @@ -10,8 +10,8 @@ pub struct Model { pub user_id: i32, pub oid: i32, pub granted_by: i32, - pub actions: Option>, - pub with_grant_option: Option, + pub actions: Vec, + pub with_grant_option: bool, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/view.rs b/src/meta/src/model_v2/view.rs index 2a28e29f2ef9b..a7bcb48d11861 100644 --- a/src/meta/src/model_v2/view.rs +++ b/src/meta/src/model_v2/view.rs @@ -11,10 +11,8 @@ pub struct Model { pub schema_id: i32, pub database_id: i32, pub owner_id: i32, - #[sea_orm(column_type = "JsonBinary", nullable)] pub properties: Option, pub sql: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] pub columns: Option, } diff --git a/src/meta/src/model_v2/worker.rs b/src/meta/src/model_v2/worker.rs index bc3581279e895..02d12d89d4457 100644 --- a/src/meta/src/model_v2/worker.rs +++ b/src/meta/src/model_v2/worker.rs @@ -7,10 +7,10 @@ use sea_orm::entity::prelude::*; pub struct Model { #[sea_orm(primary_key)] pub worker_id: i32, - pub worker_type: Option, + pub worker_type: String, pub host: String, pub port: i32, - pub status: Option, + pub status: String, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/worker_property.rs b/src/meta/src/model_v2/worker_property.rs index 2dd5870830316..3f38ddd40670e 100644 --- a/src/meta/src/model_v2/worker_property.rs +++ b/src/meta/src/model_v2/worker_property.rs @@ -7,10 +7,10 @@ use sea_orm::entity::prelude::*; pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub worker_id: i32, - pub parallel_unit_ids: Option>, - pub is_streaming: Option, - pub is_serving: Option, - pub is_unschedulable: Option, + pub parallel_unit_ids: Vec, + pub is_streaming: bool, + pub is_serving: bool, + pub is_unschedulable: bool, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] From 382dc5c1511ea05cfbfc1a2854b50729bcd16dec Mon Sep 17 00:00:00 2001 From: August Date: Tue, 12 Sep 2023 18:22:00 +0800 Subject: [PATCH 05/30] fix and some todo --- Cargo.lock | 1 + src/meta/Cargo.toml | 1 + src/meta/src/controller/catalog_controller.rs | 71 +++-- src/meta/src/controller/mod.rs | 46 ++-- src/meta/src/model_v2/actor.rs | 2 +- src/meta/src/model_v2/connection.rs | 23 +- src/meta/src/model_v2/database.rs | 31 ++- src/meta/src/model_v2/fragment.rs | 2 +- src/meta/src/model_v2/function.rs | 23 +- src/meta/src/model_v2/index.rs | 25 +- .../migration/src/m20230908_072257_init.rs | 247 ++++++++---------- src/meta/src/model_v2/object.rs | 85 +++--- src/meta/src/model_v2/schema.rs | 31 ++- src/meta/src/model_v2/sink.rs | 25 +- src/meta/src/model_v2/source.rs | 25 +- src/meta/src/model_v2/table.rs | 25 +- src/meta/src/model_v2/user.rs | 74 +----- src/meta/src/model_v2/view.rs | 23 +- 18 files changed, 318 insertions(+), 442 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e22d83c3fbb6..c5eadd449744a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7516,6 +7516,7 @@ dependencies = [ "num-integer", "num-traits", "parking_lot 0.12.1", + "parse-display", "prometheus", "prometheus-http-query", "prost", diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml index f84d14f3fc2dd..8de06f5772f6d 100644 --- a/src/meta/Cargo.toml +++ b/src/meta/Cargo.toml @@ -38,6 +38,7 @@ mime_guess = "2" num-integer = "0.1" num-traits = "0.2" parking_lot = { version = "0.12", features = ["arc_lock"] } +parse-display = "0.8" prometheus = "0.13" prometheus-http-query = "0.6" prost = "0.11" diff --git a/src/meta/src/controller/catalog_controller.rs b/src/meta/src/controller/catalog_controller.rs index 5c704da334d05..d9d90bd478384 100644 --- a/src/meta/src/controller/catalog_controller.rs +++ b/src/meta/src/controller/catalog_controller.rs @@ -15,19 +15,20 @@ use std::iter; use risingwave_common::catalog::{DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS}; -use risingwave_pb::catalog::PbDatabase; +use risingwave_pb::catalog::{PbDatabase, PbSchema}; use risingwave_pb::meta::subscribe_response::{ Info as NotificationInfo, Operation as NotificationOperation, }; -use sea_orm::{ - ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, Database as SeaDB, - DatabaseConnection, EntityTrait, QueryFilter, TransactionTrait, -}; +use sea_orm::{ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, Database as SeaDB, DatabaseConnection, DatabaseTransaction, EntityTrait, ModelTrait, QueryFilter, QueryTrait, TransactionTrait}; +use sea_orm::sea_query::Query; use tokio::sync::RwLock; -use crate::manager::{DatabaseId, MetaSrvEnv, NotificationVersion}; +use crate::controller::{ModelWithObj, ObjectType}; +use crate::manager::{DatabaseId, MetaSrvEnv, NotificationVersion, UserId}; use crate::model_v2::prelude::*; -use crate::model_v2::{connection, database, function, index, schema, sink, source, table, view}; +use crate::model_v2::{ + connection, database, function, index, object, schema, sink, source, table, view, +}; use crate::{MetaError, MetaResult}; /// `CatalogController` is the controller for catalog related operations, including database, schema, table, view, etc. @@ -68,36 +69,63 @@ impl CatalogController { impl CatalogController { pub async fn snapshot(&self) -> MetaResult> { - let dbs = Database::find().all(&self.db).await?; - Ok(dbs.into_iter().map(|db| db.into()).collect()) + let dbs = Database::find() + .find_also_related(Object) + .all(&self.db) + .await?; + let _tables = Table::find() + .find_also_related(Object) + .all(&self.db) + .await?; + + Ok(dbs + .into_iter() + .map(|(db, obj)| ModelWithObj(db, obj.unwrap()).into()) + .collect()) + } + + async fn create_object( + txn: &DatabaseTransaction, + obj_type: ObjectType, + owner_id: UserId, + ) -> MetaResult { + let mut active_db = object::ActiveModel::new(); + active_db.obj_type = ActiveValue::Set(obj_type.to_string()); + active_db.owner_id = ActiveValue::Set(owner_id as _); + Ok(active_db.insert(txn).await?) } pub async fn create_database(&self, db: PbDatabase) -> MetaResult { let txn = self.db.begin().await?; - let db: database::ActiveModel = db.into(); + let owner_id = db.owner; + + let db_obj = Self::create_object(&txn, ObjectType::Database, owner_id).await?; + let mut db: database::ActiveModel = db.into(); + db.database_id = ActiveValue::Set(db_obj.oid); let db = db.insert(&txn).await?; + let mut schemas = vec![]; for schema_name in iter::once(DEFAULT_SCHEMA_NAME).chain(SYSTEM_SCHEMAS) { let mut schema = schema::ActiveModel::new(); + let schema_obj = Self::create_object(&txn, ObjectType::Schema, owner_id).await?; + schema.schema_id = ActiveValue::Set(schema_obj.oid); schema.database_id = ActiveValue::Set(db.database_id); schema.name = ActiveValue::Set(schema_name.into()); - schema.owner_id = ActiveValue::Set(db.owner_id); - schemas.push(schema.insert(&txn).await?); + let schema = schema.insert(&txn).await?; + + schemas.push(ModelWithObj(schema, schema_obj).into()); } txn.commit().await?; let mut version = self .notify_frontend( NotificationOperation::Add, - NotificationInfo::Database(db.into()), + NotificationInfo::Database(ModelWithObj(db, db_obj).into()), ) .await; for schema in schemas { version = self - .notify_frontend( - NotificationOperation::Add, - NotificationInfo::Schema(schema.into()), - ) + .notify_frontend(NotificationOperation::Add, NotificationInfo::Schema(schema)) .await; } @@ -120,6 +148,12 @@ impl CatalogController { .ok_or_else(|| MetaError::catalog_id_not_found("database", database_id))? .into(); + // todo: drop objects. + // Object::delete_many().filter(object::Column::Oid.in_subquery( + // + // )).exec(&txn).await?; + // + Table::delete_many() .filter(table::Column::DatabaseId.eq(database_id as i32)) .exec(&txn) @@ -161,6 +195,7 @@ impl CatalogController { #[cfg(test)] mod tests { + use risingwave_common::catalog::DEFAULT_SUPER_USER_ID; use super::*; #[tokio::test] @@ -175,7 +210,7 @@ mod tests { .unwrap(); let db = PbDatabase { name: "test".to_string(), - owner: 1, + owner: DEFAULT_SUPER_USER_ID, ..Default::default() }; mgr.create_database(db).await.unwrap(); diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index c12eace7c9e18..245f11dd23298 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -13,10 +13,11 @@ // limitations under the License. use anyhow::anyhow; -use risingwave_pb::catalog::{PbDatabase, PbSchema}; -use sea_orm::ActiveValue; +use parse_display::Display; +use risingwave_pb::catalog::{PbDatabase, PbSchema, PbTable}; +use sea_orm::{ActiveValue, EntityTrait, ModelTrait}; -use crate::model_v2::{database, schema}; +use crate::model_v2::{database, object, schema}; use crate::MetaError; mod catalog_controller; @@ -28,12 +29,28 @@ impl From for MetaError { } } -impl From for PbDatabase { - fn from(model: database::Model) -> Self { +#[derive(Clone, Display, Debug)] +#[display(style = "UPPERCASE")] +enum ObjectType { + Database, + Schema, + Table, + Source, + Sink, + View, + Index, + Function, + Connection, +} + +pub struct ModelWithObj(M, object::Model); + +impl From> for PbDatabase { + fn from(value: ModelWithObj) -> Self { Self { - id: model.database_id as _, - name: model.name, - owner: model.owner_id as _, + id: value.0.database_id as _, + name: value.0.name, + owner: value.1.owner_id as _, } } } @@ -43,18 +60,17 @@ impl From for database::ActiveModel { Self { database_id: ActiveValue::Set(db.id as _), name: ActiveValue::Set(db.name), - owner_id: ActiveValue::Set(db.owner as _), } } } -impl From for PbSchema { - fn from(model: schema::Model) -> Self { +impl From> for PbSchema { + fn from(value: ModelWithObj) -> Self { Self { - id: model.schema_id as _, - name: model.name, - database_id: model.database_id as _, - owner: model.owner_id as _, + id: value.0.schema_id as _, + name: value.0.name, + database_id: value.0.database_id as _, + owner: value.1.owner_id as _, } } } diff --git a/src/meta/src/model_v2/actor.rs b/src/meta/src/model_v2/actor.rs index b7107562e22f4..7ce243aa981ce 100644 --- a/src/meta/src/model_v2/actor.rs +++ b/src/meta/src/model_v2/actor.rs @@ -5,7 +5,7 @@ use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "actor")] pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] + #[sea_orm(primary_key)] pub actor_id: i32, pub fragment_id: i32, pub status: Option, diff --git a/src/meta/src/model_v2/connection.rs b/src/meta/src/model_v2/connection.rs index e9527ec24d2ec..69e5235b4b0e6 100644 --- a/src/meta/src/model_v2/connection.rs +++ b/src/meta/src/model_v2/connection.rs @@ -10,7 +10,6 @@ pub struct Model { pub name: String, pub schema_id: i32, pub database_id: i32, - pub owner_id: i32, pub info: Option, } @@ -24,7 +23,13 @@ pub enum Relation { on_delete = "NoAction" )] Database, - #[sea_orm(has_many = "super::object::Entity")] + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::ConnectionId", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] Object, #[sea_orm( belongs_to = "super::schema::Entity", @@ -38,14 +43,6 @@ pub enum Relation { Sink, #[sea_orm(has_many = "super::source::Entity")] Source, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::OwnerId", - to = "super::user::Column::UserId", - on_update = "NoAction", - on_delete = "NoAction" - )] - User, } impl Related for Entity { @@ -78,10 +75,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::User.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/database.rs b/src/meta/src/model_v2/database.rs index 2cb31cd5f4002..7c8ef63c89eb0 100644 --- a/src/meta/src/model_v2/database.rs +++ b/src/meta/src/model_v2/database.rs @@ -5,11 +5,10 @@ use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "database")] pub struct Model { - #[sea_orm(primary_key)] + #[sea_orm(primary_key, auto_increment = false)] pub database_id: i32, #[sea_orm(unique)] pub name: String, - pub owner_id: i32, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] @@ -20,6 +19,14 @@ pub enum Relation { Function, #[sea_orm(has_many = "super::index::Entity")] Index, + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::DatabaseId", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] + Object, #[sea_orm(has_many = "super::schema::Entity")] Schema, #[sea_orm(has_many = "super::sink::Entity")] @@ -28,14 +35,6 @@ pub enum Relation { Source, #[sea_orm(has_many = "super::table::Entity")] Table, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::OwnerId", - to = "super::user::Column::UserId", - on_update = "NoAction", - on_delete = "NoAction" - )] - User, #[sea_orm(has_many = "super::view::Entity")] View, } @@ -58,6 +57,12 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::Object.def() + } +} + impl Related for Entity { fn to() -> RelationDef { Relation::Schema.def() @@ -82,12 +87,6 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::User.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::View.def() diff --git a/src/meta/src/model_v2/fragment.rs b/src/meta/src/model_v2/fragment.rs index 0671e8da7439b..8457f390012c2 100644 --- a/src/meta/src/model_v2/fragment.rs +++ b/src/meta/src/model_v2/fragment.rs @@ -5,7 +5,7 @@ use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "fragment")] pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] + #[sea_orm(primary_key)] pub fragment_id: i32, pub table_id: i32, pub fragment_type_mask: i32, diff --git a/src/meta/src/model_v2/function.rs b/src/meta/src/model_v2/function.rs index 850fb402629c4..04313f4d8ac0d 100644 --- a/src/meta/src/model_v2/function.rs +++ b/src/meta/src/model_v2/function.rs @@ -10,7 +10,6 @@ pub struct Model { pub name: String, pub schema_id: i32, pub database_id: i32, - pub owner_id: i32, pub arg_types: Option, pub return_type: Option, pub language: Option, @@ -29,7 +28,13 @@ pub enum Relation { on_delete = "NoAction" )] Database, - #[sea_orm(has_many = "super::object::Entity")] + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::FunctionId", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] Object, #[sea_orm( belongs_to = "super::schema::Entity", @@ -39,14 +44,6 @@ pub enum Relation { on_delete = "NoAction" )] Schema, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::OwnerId", - to = "super::user::Column::UserId", - on_update = "NoAction", - on_delete = "NoAction" - )] - User, } impl Related for Entity { @@ -67,10 +64,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::User.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/index.rs b/src/meta/src/model_v2/index.rs index 6d93a6d2713f7..4edd8457c4434 100644 --- a/src/meta/src/model_v2/index.rs +++ b/src/meta/src/model_v2/index.rs @@ -10,13 +10,10 @@ pub struct Model { pub name: String, pub schema_id: i32, pub database_id: i32, - pub owner_id: i32, pub index_table_id: i32, pub primary_table_id: i32, pub index_items: Option, pub original_columns: Option>, - pub initialized_at_epoch: Option, - pub created_at_epoch: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] @@ -29,7 +26,13 @@ pub enum Relation { on_delete = "NoAction" )] Database, - #[sea_orm(has_many = "super::object::Entity")] + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::IndexId", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] Object, #[sea_orm( belongs_to = "super::schema::Entity", @@ -55,14 +58,6 @@ pub enum Relation { on_delete = "NoAction" )] Table1, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::OwnerId", - to = "super::user::Column::UserId", - on_update = "NoAction", - on_delete = "NoAction" - )] - User, } impl Related for Entity { @@ -83,10 +78,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::User.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs index 58f737e1e3dc5..efa805de32def 100644 --- a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -84,7 +84,12 @@ impl MigrationTrait for Migration { .create_table( MigrationTable::create() .table(User::Table) - .col(ColumnDef::new(User::UserId).integer().primary_key()) + .col( + ColumnDef::new(User::UserId) + .integer() + .primary_key() + .auto_increment(), + ) .col(ColumnDef::new(User::Name).string().not_null()) .col(ColumnDef::new(User::IsSuper).boolean().not_null()) .col(ColumnDef::new(User::CanCreateDb).boolean().not_null()) @@ -129,19 +134,13 @@ impl MigrationTrait for Migration { .create_table( MigrationTable::create() .table(Database::Table) - .col( - ColumnDef::new(Database::DatabaseId) - .integer() - .auto_increment() - .primary_key(), - ) + .col(ColumnDef::new(Database::DatabaseId).integer().primary_key()) .col( ColumnDef::new(Database::Name) .string() .unique_key() .not_null(), ) - .col(ColumnDef::new(Database::OwnerId).integer().not_null()) .to_owned(), ) .await?; @@ -149,15 +148,9 @@ impl MigrationTrait for Migration { .create_table( MigrationTable::create() .table(Schema::Table) - .col( - ColumnDef::new(Schema::SchemaId) - .integer() - .auto_increment() - .primary_key(), - ) + .col(ColumnDef::new(Schema::SchemaId).integer().primary_key()) .col(ColumnDef::new(Schema::Name).string().not_null()) .col(ColumnDef::new(Schema::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Schema::OwnerId).integer().not_null()) .to_owned(), ) .await?; @@ -165,7 +158,12 @@ impl MigrationTrait for Migration { .create_table( MigrationTable::create() .table(Fragment::Table) - .col(ColumnDef::new(Fragment::FragmentId).integer().primary_key()) + .col( + ColumnDef::new(Fragment::FragmentId) + .integer() + .primary_key() + .auto_increment(), + ) .col(ColumnDef::new(Fragment::TableId).integer().not_null()) .col( ColumnDef::new(Fragment::FragmentTypeMask) @@ -191,7 +189,12 @@ impl MigrationTrait for Migration { .create_table( MigrationTable::create() .table(Actor::Table) - .col(ColumnDef::new(Actor::ActorId).integer().primary_key()) + .col( + ColumnDef::new(Actor::ActorId) + .integer() + .primary_key() + .auto_increment(), + ) .col(ColumnDef::new(Actor::FragmentId).integer().not_null()) .col(ColumnDef::new(Actor::Status).string()) .col(ColumnDef::new(Actor::Splits).json()) @@ -210,7 +213,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Table::Name).string().not_null()) .col(ColumnDef::new(Table::SchemaId).integer().not_null()) .col(ColumnDef::new(Table::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Table::OwnerId).integer().not_null()) .col(ColumnDef::new(Table::OptionalAssociatedSourceId).integer()) .col(ColumnDef::new(Table::TableType).string()) .col(ColumnDef::new(Table::Columns).json()) @@ -228,8 +230,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Table::DistKeyInPk).array(ColumnType::Integer)) .col(ColumnDef::new(Table::DmlFragmentId).integer()) .col(ColumnDef::new(Table::Cardinality).array(ColumnType::Integer)) - .col(ColumnDef::new(Table::InitializedAtEpoch).integer()) - .col(ColumnDef::new(Table::CreatedAtEpoch).integer()) .col(ColumnDef::new(Table::CleanedByWatermark).boolean()) .col(ColumnDef::new(Table::Version).array(ColumnType::Integer)) .to_owned(), @@ -243,7 +243,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Source::Name).string().not_null()) .col(ColumnDef::new(Source::SchemaId).integer().not_null()) .col(ColumnDef::new(Source::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Source::OwnerId).integer().not_null()) .col(ColumnDef::new(Source::RowIdIndex).string()) .col(ColumnDef::new(Source::Columns).json()) .col(ColumnDef::new(Source::PkColumnIds).json()) @@ -253,8 +252,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Source::WatermarkDescs).json()) .col(ColumnDef::new(Source::OptionalAssociatedTableId).integer()) .col(ColumnDef::new(Source::ConnectionId).integer()) - .col(ColumnDef::new(Source::InitializedAtEpoch).integer()) - .col(ColumnDef::new(Source::CreatedAtEpoch).integer()) .to_owned(), ) .await?; @@ -266,7 +263,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Sink::Name).string().not_null()) .col(ColumnDef::new(Sink::SchemaId).integer().not_null()) .col(ColumnDef::new(Sink::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Sink::OwnerId).integer().not_null()) .col(ColumnDef::new(Sink::Columns).json()) .col(ColumnDef::new(Sink::PkColumnIds).json()) .col(ColumnDef::new(Sink::DistributionKey).array(ColumnType::Integer)) @@ -275,8 +271,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Sink::Properties).json()) .col(ColumnDef::new(Sink::Definition).string()) .col(ColumnDef::new(Sink::ConnectionId).integer()) - .col(ColumnDef::new(Sink::InitializedAtEpoch).integer()) - .col(ColumnDef::new(Sink::CreatedAtEpoch).integer()) .to_owned(), ) .await?; @@ -292,7 +286,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Connection::Name).string().not_null()) .col(ColumnDef::new(Connection::SchemaId).integer().not_null()) .col(ColumnDef::new(Connection::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Connection::OwnerId).integer().not_null()) .col(ColumnDef::new(Connection::Info).json()) .to_owned(), ) @@ -305,7 +298,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(View::Name).string().not_null()) .col(ColumnDef::new(View::SchemaId).integer().not_null()) .col(ColumnDef::new(View::DatabaseId).integer().not_null()) - .col(ColumnDef::new(View::OwnerId).integer().not_null()) .col(ColumnDef::new(View::Properties).json()) .col(ColumnDef::new(View::Sql).string()) .col(ColumnDef::new(View::Columns).json()) @@ -320,13 +312,10 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Index::Name).string().not_null()) .col(ColumnDef::new(Index::SchemaId).integer().not_null()) .col(ColumnDef::new(Index::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Index::OwnerId).integer().not_null()) .col(ColumnDef::new(Index::IndexTableId).integer().not_null()) .col(ColumnDef::new(Index::PrimaryTableId).integer().not_null()) .col(ColumnDef::new(Index::IndexItems).json()) .col(ColumnDef::new(Index::OriginalColumns).array(ColumnType::Integer)) - .col(ColumnDef::new(Index::InitializedAtEpoch).integer()) - .col(ColumnDef::new(Index::CreatedAtEpoch).integer()) .to_owned(), ) .await?; @@ -338,7 +327,6 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Function::Name).string().not_null()) .col(ColumnDef::new(Function::SchemaId).integer().not_null()) .col(ColumnDef::new(Function::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Function::OwnerId).integer().not_null()) .col(ColumnDef::new(Function::ArgTypes).json()) .col(ColumnDef::new(Function::ReturnType).string()) .col(ColumnDef::new(Function::Language).string()) @@ -352,8 +340,26 @@ impl MigrationTrait for Migration { .create_table( MigrationTable::create() .table(Object::Table) - .col(ColumnDef::new(Object::Oid).integer().primary_key()) + .col( + ColumnDef::new(Object::Oid) + .integer() + .auto_increment() + .primary_key(), + ) .col(ColumnDef::new(Object::ObjType).string().not_null()) + .col(ColumnDef::new(Object::OwnerId).integer().not_null()) + .col( + ColumnDef::new(Object::InitializedAt) + .timestamp() + .default(Expr::current_timestamp()) + .not_null(), + ) + .col( + ColumnDef::new(Object::CreatedAt) + .timestamp() + .default(Expr::current_timestamp()) + .not_null(), + ) .to_owned(), ) .await?; @@ -436,9 +442,10 @@ impl MigrationTrait for Migration { manager .create_foreign_key( ForeignKey::create() - .name("FK_database_owner_id") - .from(Database::Table, Database::OwnerId) - .to(User::Table, User::UserId) + .name("FK_database_object_id") + .from(Database::Table, Database::DatabaseId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -454,9 +461,10 @@ impl MigrationTrait for Migration { manager .create_foreign_key( ForeignKey::create() - .name("FK_schema_owner_id") - .from(Schema::Table, Schema::OwnerId) - .to(User::Table, User::UserId) + .name("FK_schema_object_id") + .from(Schema::Table, Schema::SchemaId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -499,9 +507,10 @@ impl MigrationTrait for Migration { manager .create_foreign_key( ForeignKey::create() - .name("FK_table_owner_id") - .from(Table::Table, Table::OwnerId) - .to(User::Table, User::UserId) + .name("FK_table_object_id") + .from(Table::Table, Table::TableId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -553,9 +562,10 @@ impl MigrationTrait for Migration { manager .create_foreign_key( ForeignKey::create() - .name("FK_source_owner_id") - .from(Source::Table, Source::OwnerId) - .to(User::Table, User::UserId) + .name("FK_source_object_id") + .from(Source::Table, Source::SourceId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -598,9 +608,10 @@ impl MigrationTrait for Migration { manager .create_foreign_key( ForeignKey::create() - .name("FK_sink_owner_id") - .from(Sink::Table, Sink::OwnerId) - .to(User::Table, User::UserId) + .name("FK_sink_object_id") + .from(Sink::Table, Sink::SinkId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -634,9 +645,10 @@ impl MigrationTrait for Migration { manager .create_foreign_key( ForeignKey::create() - .name("FK_connection_owner_id") - .from(Connection::Table, Connection::OwnerId) - .to(User::Table, User::UserId) + .name("FK_connection_object_id") + .from(Connection::Table, Connection::ConnectionId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -661,9 +673,10 @@ impl MigrationTrait for Migration { manager .create_foreign_key( ForeignKey::create() - .name("FK_view_owner_id") - .from(View::Table, View::OwnerId) - .to(User::Table, User::UserId) + .name("FK_view_object_id") + .from(View::Table, View::ViewId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -688,9 +701,10 @@ impl MigrationTrait for Migration { manager .create_foreign_key( ForeignKey::create() - .name("FK_index_owner_id") - .from(Index::Table, Index::OwnerId) - .to(User::Table, User::UserId) + .name("FK_index_object_id") + .from(Index::Table, Index::IndexId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -733,72 +747,19 @@ impl MigrationTrait for Migration { manager .create_foreign_key( ForeignKey::create() - .name("FK_function_owner_id") - .from(Function::Table, Function::OwnerId) - .to(User::Table, User::UserId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_object_oid_table") - .from(Object::Table, Object::Oid) - .to(Table::Table, Table::TableId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_object_oid_source") - .from(Object::Table, Object::Oid) - .to(Source::Table, Source::SourceId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_object_oid_sink") - .from(Object::Table, Object::Oid) - .to(Sink::Table, Sink::SinkId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_object_oid_index") - .from(Object::Table, Object::Oid) - .to(Index::Table, Index::IndexId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_object_oid_view") - .from(Object::Table, Object::Oid) - .to(View::Table, View::ViewId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_object_oid_connection") - .from(Object::Table, Object::Oid) - .to(Connection::Table, Connection::ConnectionId) + .name("FK_function_object_id") + .from(Function::Table, Function::FunctionId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; manager .create_foreign_key( ForeignKey::create() - .name("FK_object_oid_function") - .from(Object::Table, Object::Oid) - .to(Function::Table, Function::FunctionId) + .name("FK_object_owner_id") + .from(Object::Table, Object::OwnerId) + .to(User::Table, User::UserId) .to_owned(), ) .await?; @@ -817,6 +778,7 @@ impl MigrationTrait for Migration { .name("FK_object_dependency_used_by") .from(ObjectDependency::Table, ObjectDependency::UsedBy) .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -825,7 +787,6 @@ impl MigrationTrait for Migration { let insert_sys_users = Query::insert() .into_table(User::Table) .columns([ - User::UserId, User::Name, User::IsSuper, User::CanCreateUser, @@ -833,7 +794,6 @@ impl MigrationTrait for Migration { User::CanLogin, ]) .values_panic([ - 1.into(), "root".into(), true.into(), true.into(), @@ -841,7 +801,6 @@ impl MigrationTrait for Migration { true.into(), ]) .values_panic([ - 2.into(), "postgres".into(), true.into(), true.into(), @@ -849,13 +808,37 @@ impl MigrationTrait for Migration { true.into(), ]) .to_owned(); + + // Since User table is newly created, we assume that the initial user id of `root` is 1 and `postgres` is 2. + let insert_objects = Query::insert() + .into_table(Object::Table) + .columns([Object::ObjType, Object::OwnerId]) + .values_panic(["DATABASE".into(), 1.into()]) + .values_panic(["SCHEMA".into(), 1.into()]) // public + .values_panic(["SCHEMA".into(), 1.into()]) // pg_catalog + .values_panic(["SCHEMA".into(), 1.into()]) // information_schema + .values_panic(["SCHEMA".into(), 1.into()]) // rw_catalog + .to_owned(); + + // Since all tables are newly created, we assume that the initial object id of `dev` is 1 and the schemas' ids are 2, 3, 4, 5. let insert_sys_database = Query::insert() .into_table(Database::Table) - .columns([Database::Name, Database::OwnerId]) - .values_panic(["dev".into(), 1.into()]) + .columns([Database::DatabaseId, Database::Name]) + .values_panic([1.into(), "dev".into()]) .to_owned(); + let insert_sys_schemas = Query::insert() + .into_table(Schema::Table) + .columns([Schema::SchemaId, Schema::Name, Schema::DatabaseId]) + .values_panic([2.into(), "public".into(), 1.into()]) + .values_panic([3.into(), "pg_catalog".into(), 1.into()]) + .values_panic([4.into(), "information_schema".into(), 1.into()]) + .values_panic([5.into(), "rw_catalog".into(), 1.into()]) + .to_owned(); + manager.exec_stmt(insert_sys_users).await?; + manager.exec_stmt(insert_objects).await?; manager.exec_stmt(insert_sys_database).await?; + manager.exec_stmt(insert_sys_schemas).await?; Ok(()) } @@ -951,7 +934,6 @@ enum Database { Table, DatabaseId, Name, - OwnerId, } #[derive(DeriveIden)] @@ -960,7 +942,6 @@ enum Schema { SchemaId, Name, DatabaseId, - OwnerId, } #[derive(DeriveIden)] @@ -1000,7 +981,6 @@ enum Table { Name, SchemaId, DatabaseId, - OwnerId, OptionalAssociatedSourceId, TableType, Columns, @@ -1018,8 +998,6 @@ enum Table { DistKeyInPk, DmlFragmentId, Cardinality, - InitializedAtEpoch, - CreatedAtEpoch, CleanedByWatermark, Version, } @@ -1031,7 +1009,6 @@ enum Source { Name, SchemaId, DatabaseId, - OwnerId, RowIdIndex, Columns, PkColumnIds, @@ -1041,8 +1018,6 @@ enum Source { WatermarkDescs, OptionalAssociatedTableId, ConnectionId, - InitializedAtEpoch, - CreatedAtEpoch, } #[derive(DeriveIden)] @@ -1052,7 +1027,6 @@ enum Sink { Name, SchemaId, DatabaseId, - OwnerId, Columns, PkColumnIds, DistributionKey, @@ -1061,8 +1035,6 @@ enum Sink { Properties, Definition, ConnectionId, - InitializedAtEpoch, - CreatedAtEpoch, } #[derive(DeriveIden)] @@ -1072,7 +1044,6 @@ enum Connection { Name, SchemaId, DatabaseId, - OwnerId, Info, } @@ -1083,7 +1054,6 @@ enum View { Name, SchemaId, DatabaseId, - OwnerId, Properties, Sql, Columns, @@ -1096,13 +1066,10 @@ enum Index { Name, SchemaId, DatabaseId, - OwnerId, IndexTableId, PrimaryTableId, IndexItems, OriginalColumns, - InitializedAtEpoch, - CreatedAtEpoch, } #[derive(DeriveIden)] @@ -1112,7 +1079,6 @@ enum Function { Name, SchemaId, DatabaseId, - OwnerId, ArgTypes, ReturnType, Language, @@ -1126,6 +1092,9 @@ enum Object { Table, Oid, ObjType, + OwnerId, + InitializedAt, + CreatedAt, } #[derive(DeriveIden)] diff --git a/src/meta/src/model_v2/object.rs b/src/meta/src/model_v2/object.rs index dec1d79651c1d..be72e45094fd6 100644 --- a/src/meta/src/model_v2/object.rs +++ b/src/meta/src/model_v2/object.rs @@ -5,70 +5,43 @@ use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "object")] pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] + #[sea_orm(primary_key)] pub oid: i32, pub obj_type: String, + pub owner_id: i32, + pub initialized_at: DateTime, + pub created_at: DateTime, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm( - belongs_to = "super::connection::Entity", - from = "Column::Oid", - to = "super::connection::Column::ConnectionId", - on_update = "NoAction", - on_delete = "NoAction" - )] + #[sea_orm(has_many = "super::connection::Entity")] Connection, - #[sea_orm( - belongs_to = "super::function::Entity", - from = "Column::Oid", - to = "super::function::Column::FunctionId", - on_update = "NoAction", - on_delete = "NoAction" - )] + #[sea_orm(has_many = "super::database::Entity")] + Database, + #[sea_orm(has_many = "super::function::Entity")] Function, - #[sea_orm( - belongs_to = "super::index::Entity", - from = "Column::Oid", - to = "super::index::Column::IndexId", - on_update = "NoAction", - on_delete = "NoAction" - )] + #[sea_orm(has_many = "super::index::Entity")] Index, - #[sea_orm( - belongs_to = "super::sink::Entity", - from = "Column::Oid", - to = "super::sink::Column::SinkId", - on_update = "NoAction", - on_delete = "NoAction" - )] + #[sea_orm(has_many = "super::schema::Entity")] + Schema, + #[sea_orm(has_many = "super::sink::Entity")] Sink, - #[sea_orm( - belongs_to = "super::source::Entity", - from = "Column::Oid", - to = "super::source::Column::SourceId", - on_update = "NoAction", - on_delete = "NoAction" - )] + #[sea_orm(has_many = "super::source::Entity")] Source, + #[sea_orm(has_many = "super::table::Entity")] + Table, #[sea_orm( - belongs_to = "super::table::Entity", - from = "Column::Oid", - to = "super::table::Column::TableId", + belongs_to = "super::user::Entity", + from = "Column::OwnerId", + to = "super::user::Column::UserId", on_update = "NoAction", on_delete = "NoAction" )] - Table, + User, #[sea_orm(has_many = "super::user_privilege::Entity")] UserPrivilege, - #[sea_orm( - belongs_to = "super::view::Entity", - from = "Column::Oid", - to = "super::view::Column::ViewId", - on_update = "NoAction", - on_delete = "NoAction" - )] + #[sea_orm(has_many = "super::view::Entity")] View, } @@ -78,6 +51,12 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::Database.def() + } +} + impl Related for Entity { fn to() -> RelationDef { Relation::Function.def() @@ -90,6 +69,12 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::Schema.def() + } +} + impl Related for Entity { fn to() -> RelationDef { Relation::Sink.def() @@ -108,6 +93,12 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + impl Related for Entity { fn to() -> RelationDef { Relation::UserPrivilege.def() diff --git a/src/meta/src/model_v2/schema.rs b/src/meta/src/model_v2/schema.rs index 3749ff7f90621..8216bf948a1bc 100644 --- a/src/meta/src/model_v2/schema.rs +++ b/src/meta/src/model_v2/schema.rs @@ -5,11 +5,10 @@ use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "schema")] pub struct Model { - #[sea_orm(primary_key)] + #[sea_orm(primary_key, auto_increment = false)] pub schema_id: i32, pub name: String, pub database_id: i32, - pub owner_id: i32, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] @@ -28,20 +27,20 @@ pub enum Relation { Function, #[sea_orm(has_many = "super::index::Entity")] Index, + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::SchemaId", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] + Object, #[sea_orm(has_many = "super::sink::Entity")] Sink, #[sea_orm(has_many = "super::source::Entity")] Source, #[sea_orm(has_many = "super::table::Entity")] Table, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::OwnerId", - to = "super::user::Column::UserId", - on_update = "NoAction", - on_delete = "NoAction" - )] - User, #[sea_orm(has_many = "super::view::Entity")] View, } @@ -70,6 +69,12 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::Object.def() + } +} + impl Related for Entity { fn to() -> RelationDef { Relation::Sink.def() @@ -88,12 +93,6 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::User.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::View.def() diff --git a/src/meta/src/model_v2/sink.rs b/src/meta/src/model_v2/sink.rs index 7da386932e33e..8ce1be2c55093 100644 --- a/src/meta/src/model_v2/sink.rs +++ b/src/meta/src/model_v2/sink.rs @@ -10,7 +10,6 @@ pub struct Model { pub name: String, pub schema_id: i32, pub database_id: i32, - pub owner_id: i32, pub columns: Option, pub pk_column_ids: Option, pub distribution_key: Option>, @@ -19,8 +18,6 @@ pub struct Model { pub properties: Option, pub definition: Option, pub connection_id: Option, - pub initialized_at_epoch: Option, - pub created_at_epoch: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] @@ -41,7 +38,13 @@ pub enum Relation { on_delete = "NoAction" )] Database, - #[sea_orm(has_many = "super::object::Entity")] + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::SinkId", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] Object, #[sea_orm( belongs_to = "super::schema::Entity", @@ -51,14 +54,6 @@ pub enum Relation { on_delete = "NoAction" )] Schema, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::OwnerId", - to = "super::user::Column::UserId", - on_update = "NoAction", - on_delete = "NoAction" - )] - User, } impl Related for Entity { @@ -85,10 +80,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::User.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/source.rs b/src/meta/src/model_v2/source.rs index d404b906b9f9a..edc4f5cefca09 100644 --- a/src/meta/src/model_v2/source.rs +++ b/src/meta/src/model_v2/source.rs @@ -10,7 +10,6 @@ pub struct Model { pub name: String, pub schema_id: i32, pub database_id: i32, - pub owner_id: i32, pub row_id_index: Option, pub columns: Option, pub pk_column_ids: Option, @@ -20,8 +19,6 @@ pub struct Model { pub watermark_descs: Option, pub optional_associated_table_id: Option, pub connection_id: Option, - pub initialized_at_epoch: Option, - pub created_at_epoch: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] @@ -42,7 +39,13 @@ pub enum Relation { on_delete = "NoAction" )] Database, - #[sea_orm(has_many = "super::object::Entity")] + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::SourceId", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] Object, #[sea_orm( belongs_to = "super::schema::Entity", @@ -60,14 +63,6 @@ pub enum Relation { on_delete = "NoAction" )] Table, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::OwnerId", - to = "super::user::Column::UserId", - on_update = "NoAction", - on_delete = "NoAction" - )] - User, } impl Related for Entity { @@ -100,10 +95,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::User.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/table.rs b/src/meta/src/model_v2/table.rs index 25338e81a76a1..ed0773bfba015 100644 --- a/src/meta/src/model_v2/table.rs +++ b/src/meta/src/model_v2/table.rs @@ -10,7 +10,6 @@ pub struct Model { pub name: String, pub schema_id: i32, pub database_id: i32, - pub owner_id: i32, pub optional_associated_source_id: Option, pub table_type: Option, pub columns: Option, @@ -28,8 +27,6 @@ pub struct Model { pub dist_key_in_pk: Option>, pub dml_fragment_id: Option, pub cardinality: Option>, - pub initialized_at_epoch: Option, - pub created_at_epoch: Option, pub cleaned_by_watermark: Option, pub version: Option>, } @@ -60,7 +57,13 @@ pub enum Relation { on_delete = "NoAction" )] Fragment1, - #[sea_orm(has_many = "super::object::Entity")] + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::TableId", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] Object, #[sea_orm( belongs_to = "super::schema::Entity", @@ -78,14 +81,6 @@ pub enum Relation { on_delete = "NoAction" )] Source, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::OwnerId", - to = "super::user::Column::UserId", - on_update = "NoAction", - on_delete = "NoAction" - )] - User, } impl Related for Entity { @@ -112,10 +107,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::User.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/user.rs b/src/meta/src/model_v2/user.rs index 96874e72af0b8..d3b87f99a2c0b 100644 --- a/src/meta/src/model_v2/user.rs +++ b/src/meta/src/model_v2/user.rs @@ -5,7 +5,7 @@ use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "user")] pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] + #[sea_orm(primary_key)] pub user_id: i32, pub name: String, pub is_super: bool, @@ -18,77 +18,13 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm(has_many = "super::connection::Entity")] - Connection, - #[sea_orm(has_many = "super::database::Entity")] - Database, - #[sea_orm(has_many = "super::function::Entity")] - Function, - #[sea_orm(has_many = "super::index::Entity")] - Index, - #[sea_orm(has_many = "super::schema::Entity")] - Schema, - #[sea_orm(has_many = "super::sink::Entity")] - Sink, - #[sea_orm(has_many = "super::source::Entity")] - Source, - #[sea_orm(has_many = "super::table::Entity")] - Table, - #[sea_orm(has_many = "super::view::Entity")] - View, + #[sea_orm(has_many = "super::object::Entity")] + Object, } -impl Related for Entity { +impl Related for Entity { fn to() -> RelationDef { - Relation::Connection.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Database.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Function.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Index.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Schema.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Sink.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Source.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Table.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::View.def() + Relation::Object.def() } } diff --git a/src/meta/src/model_v2/view.rs b/src/meta/src/model_v2/view.rs index a7bcb48d11861..807a2390ac3a2 100644 --- a/src/meta/src/model_v2/view.rs +++ b/src/meta/src/model_v2/view.rs @@ -10,7 +10,6 @@ pub struct Model { pub name: String, pub schema_id: i32, pub database_id: i32, - pub owner_id: i32, pub properties: Option, pub sql: Option, pub columns: Option, @@ -26,7 +25,13 @@ pub enum Relation { on_delete = "NoAction" )] Database, - #[sea_orm(has_many = "super::object::Entity")] + #[sea_orm( + belongs_to = "super::object::Entity", + from = "Column::ViewId", + to = "super::object::Column::Oid", + on_update = "NoAction", + on_delete = "NoAction" + )] Object, #[sea_orm( belongs_to = "super::schema::Entity", @@ -36,14 +41,6 @@ pub enum Relation { on_delete = "NoAction" )] Schema, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::OwnerId", - to = "super::user::Column::UserId", - on_update = "NoAction", - on_delete = "NoAction" - )] - User, } impl Related for Entity { @@ -64,10 +61,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::User.def() - } -} - impl ActiveModelBehavior for ActiveModel {} From 2abc6b2c928be0460c7f2b704c3b931cf6cae3a1 Mon Sep 17 00:00:00 2001 From: August Date: Wed, 13 Sep 2023 19:47:51 +0800 Subject: [PATCH 06/30] add system parameter and cluster info/ctl, refine catalog ctl --- Cargo.lock | 9 +- src/common/src/system_param/mod.rs | 8 +- src/meta/src/controller/catalog_controller.rs | 198 +++++++---- src/meta/src/controller/mod.rs | 8 +- .../src/controller/system_param_controller.rs | 331 ++++++++++++++++++ src/meta/src/model_v2/connection.rs | 2 +- src/meta/src/model_v2/database.rs | 2 +- src/meta/src/model_v2/function.rs | 2 +- src/meta/src/model_v2/index.rs | 2 +- src/meta/src/model_v2/migration/Cargo.toml | 3 +- .../migration/src/m20230908_072257_init.rs | 73 +++- src/meta/src/model_v2/mod.rs | 2 + src/meta/src/model_v2/object_dependency.rs | 2 +- src/meta/src/model_v2/prelude.rs | 2 + src/meta/src/model_v2/schema.rs | 2 +- src/meta/src/model_v2/sink.rs | 2 +- src/meta/src/model_v2/source.rs | 2 +- src/meta/src/model_v2/table.rs | 2 +- src/meta/src/model_v2/view.rs | 2 +- 19 files changed, 568 insertions(+), 86 deletions(-) create mode 100644 src/meta/src/controller/system_param_controller.rs diff --git a/Cargo.lock b/Cargo.lock index 8f5868cd45fe2..9b2b04f9e4d40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3958,7 +3958,7 @@ checksum = "ce243b1bfa62ffc028f1cc3b6034ec63d649f3031bc8a4fbbb004e1ac17d1f68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -4677,6 +4677,7 @@ version = "0.1.0" dependencies = [ "async-std", "sea-orm-migration", + "uuid", ] [[package]] @@ -5477,7 +5478,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -8300,7 +8301,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.32", ] [[package]] @@ -8358,7 +8359,7 @@ dependencies = [ "proc-macro2", "quote", "sea-bae", - "syn 2.0.31", + "syn 2.0.32", "unicode-ident", ] diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs index abd2e110c8492..344501d8442ec 100644 --- a/src/common/src/system_param/mod.rs +++ b/src/common/src/system_param/mod.rs @@ -129,7 +129,7 @@ macro_rules! impl_system_params_to_kv { macro_rules! impl_derive_missing_fields { ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { - fn derive_missing_fields(params: &mut PbSystemParams) { + pub fn derive_missing_fields(params: &mut PbSystemParams) { $( if params.$field.is_none() && let Some(v) = OverrideFromParams::$field(params) { params.$field = Some(v); @@ -246,7 +246,7 @@ macro_rules! impl_default_from_other_params { macro_rules! impl_set_system_param { ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { - pub fn set_system_param(params: &mut PbSystemParams, key: &str, value: Option) -> Result<()> { + pub fn set_system_param(params: &mut PbSystemParams, key: &str, value: Option) -> Result { match key { $( key_of!($field) => { @@ -256,7 +256,8 @@ macro_rules! impl_set_system_param { $default.ok_or_else(|| format!("{} does not have a default value", key))? }; OverrideValidateOnSet::$field(&v)?; - params.$field = Some(v); + params.$field = Some(v.clone()); + return Ok(v.to_string()) }, )* _ => { @@ -266,7 +267,6 @@ macro_rules! impl_set_system_param { )); } }; - Ok(()) } }; } diff --git a/src/meta/src/controller/catalog_controller.rs b/src/meta/src/controller/catalog_controller.rs index d9d90bd478384..b2a410b966f6f 100644 --- a/src/meta/src/controller/catalog_controller.rs +++ b/src/meta/src/controller/catalog_controller.rs @@ -13,14 +13,20 @@ // limitations under the License. use std::iter; +use std::time::Duration; +use itertools::Itertools; use risingwave_common::catalog::{DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS}; -use risingwave_pb::catalog::{PbDatabase, PbSchema}; +use risingwave_pb::catalog::PbDatabase; use risingwave_pb::meta::subscribe_response::{ Info as NotificationInfo, Operation as NotificationOperation, }; -use sea_orm::{ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, Database as SeaDB, DatabaseConnection, DatabaseTransaction, EntityTrait, ModelTrait, QueryFilter, QueryTrait, TransactionTrait}; -use sea_orm::sea_query::Query; +use sea_orm::sea_query::{Query, UnionType}; +use sea_orm::{ + ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectOptions, + ConnectionTrait, Database as SeaDB, DatabaseConnection, DatabaseTransaction, EntityTrait, + ModelTrait, QueryFilter, QuerySelect, TransactionTrait, +}; use tokio::sync::RwLock; use crate::controller::{ModelWithObj, ObjectType}; @@ -34,27 +40,34 @@ use crate::{MetaError, MetaResult}; /// `CatalogController` is the controller for catalog related operations, including database, schema, table, view, etc. pub struct CatalogController { env: MetaSrvEnv, - db: DatabaseConnection, - // todo: replace it with monotonic timestamp. - revision: RwLock, + inner: RwLock, +} + +pub struct ReleaseContext { + streaming_jobs: Vec, + source_ids: Vec, + connections: Vec, } impl CatalogController { pub async fn new(env: MetaSrvEnv, url: &str) -> MetaResult { - let db = SeaDB::connect(url).await?; + let mut opts = ConnectOptions::new(url); + opts.max_connections(20) + .connect_timeout(Duration::from_secs(10)); + + let db = SeaDB::connect(opts).await?; Ok(Self { env, - db, - revision: RwLock::new(0), + inner: RwLock::new(CatalogControllerInner { db }), }) } } -impl CatalogController { - pub async fn get_revision(&self) -> u64 { - *self.revision.read().await - } +struct CatalogControllerInner { + db: DatabaseConnection, +} +impl CatalogController { async fn notify_frontend( &self, operation: NotificationOperation, @@ -69,13 +82,14 @@ impl CatalogController { impl CatalogController { pub async fn snapshot(&self) -> MetaResult> { + let inner = self.inner.read().await; let dbs = Database::find() .find_also_related(Object) - .all(&self.db) + .all(&inner.db) .await?; let _tables = Table::find() .find_also_related(Object) - .all(&self.db) + .all(&inner.db) .await?; Ok(dbs @@ -96,7 +110,8 @@ impl CatalogController { } pub async fn create_database(&self, db: PbDatabase) -> MetaResult { - let txn = self.db.begin().await?; + let inner = self.inner.write().await; + let txn = inner.db.begin().await?; let owner_id = db.owner; let db_obj = Self::create_object(&txn, ObjectType::Database, owner_id).await?; @@ -132,70 +147,125 @@ impl CatalogController { Ok(version) } - pub async fn drop_database(&self, database_id: DatabaseId) -> MetaResult<()> { - let _tables = Table::find() - .filter(table::Column::DatabaseId.eq(database_id as i32)) - .all(&self.db) - .await?; - // 1. unregister source. - // 2. fragments + actors, streaming manager drop streaming job. - // 3. connection to drop. + pub async fn drop_database( + &self, + database_id: DatabaseId, + ) -> MetaResult<(ReleaseContext, NotificationVersion)> { + let inner = self.inner.write().await; + let txn = inner.db.begin().await?; - let txn = self.db.begin().await?; - let db: database::ActiveModel = Database::find_by_id(database_id as i32) + let db = Database::find_by_id(database_id as i32) .one(&txn) .await? - .ok_or_else(|| MetaError::catalog_id_not_found("database", database_id))? - .into(); + .ok_or_else(|| MetaError::catalog_id_not_found("database", database_id))?; + let db_obj = db.find_related(Object).one(&txn).await?; + let pb_db = ModelWithObj(db.clone(), db_obj.unwrap()).into(); - // todo: drop objects. - // Object::delete_many().filter(object::Column::Oid.in_subquery( - // - // )).exec(&txn).await?; - // + let tables = db.find_related(Table).all(&txn).await?; + let sinks = db.find_related(Sink).all(&txn).await?; + let indexes = db.find_related(Index).all(&txn).await?; + let streaming_jobs = tables + .into_iter() + .map(|t| t.table_id) + .chain(sinks.into_iter().map(|s| s.sink_id)) + .chain(indexes.into_iter().map(|i| i.index_id)) + .collect::>(); + let source_ids = db + .find_related(Source) + .all(&txn) + .await? + .into_iter() + .map(|s| s.source_id) + .collect_vec(); + let connections = db + .find_related(Connection) + .all(&txn) + .await? + .into_iter() + .flat_map(|c| c.info) + .collect_vec(); - Table::delete_many() - .filter(table::Column::DatabaseId.eq(database_id as i32)) + let mut query_schema = Query::select() + .column(schema::Column::SchemaId) + .from(schema::Entity) + .and_where(schema::Column::DatabaseId.eq(database_id as i32)) + .to_owned(); + let query_table = Query::select() + .column(table::Column::TableId) + .from(table::Entity) + .and_where(table::Column::DatabaseId.eq(database_id as i32)) + .to_owned(); + let query_source = Query::select() + .column(source::Column::SourceId) + .from(source::Entity) + .and_where(source::Column::DatabaseId.eq(database_id as i32)) + .to_owned(); + let query_sink = Query::select() + .column(sink::Column::SinkId) + .from(sink::Entity) + .and_where(sink::Column::DatabaseId.eq(database_id as i32)) + .to_owned(); + let query_index = Query::select() + .column(index::Column::IndexId) + .from(index::Entity) + .and_where(index::Column::DatabaseId.eq(database_id as i32)) + .to_owned(); + let query_view = Query::select() + .column(view::Column::ViewId) + .from(view::Entity) + .and_where(view::Column::DatabaseId.eq(database_id as i32)) + .to_owned(); + let query_function = Query::select() + .column(function::Column::FunctionId) + .from(function::Entity) + .and_where(function::Column::DatabaseId.eq(database_id as i32)) + .to_owned(); + let query_connection = Query::select() + .column(connection::Column::ConnectionId) + .from(connection::Entity) + .and_where(connection::Column::DatabaseId.eq(database_id as i32)) + .to_owned(); + let query_all = query_schema + .union(UnionType::All, query_table) + .union(UnionType::All, query_source) + .union(UnionType::All, query_sink) + .union(UnionType::All, query_index) + .union(UnionType::All, query_view) + .union(UnionType::All, query_function) + .union(UnionType::All, query_connection) + .to_owned(); + // drop related objects, the relations will be dropped cascade. + Object::delete_many() + .filter(object::Column::Oid.in_subquery(query_all)) .exec(&txn) .await?; - Source::delete_many() - .filter(source::Column::DatabaseId.eq(database_id as i32)) + Database::delete(database::ActiveModel::from(db)) .exec(&txn) .await?; - Sink::delete_many() - .filter(sink::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - Index::delete_many() - .filter(index::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - Function::delete_many() - .filter(function::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - Connection::delete_many() - .filter(connection::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - View::delete_many() - .filter(view::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - Schema::delete_many() - .filter(schema::Column::DatabaseId.eq(database_id as i32)) - .exec(&txn) - .await?; - Database::delete(db).exec(&txn).await?; txn.commit().await?; - Ok(()) + + let version = self + .notify_frontend( + NotificationOperation::Delete, + NotificationInfo::Database(pb_db), + ) + .await; + Ok(( + ReleaseContext { + streaming_jobs, + source_ids, + connections, + }, + version, + )) } } #[cfg(test)] mod tests { use risingwave_common::catalog::DEFAULT_SUPER_USER_ID; + use super::*; #[tokio::test] @@ -216,7 +286,7 @@ mod tests { mgr.create_database(db).await.unwrap(); let db = Database::find() .filter(database::Column::Name.eq("test")) - .one(&mgr.db) + .one(&mgr.inner.read().await.db) .await .unwrap() .unwrap(); diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index 245f11dd23298..d27d14ebc7fee 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -14,13 +14,16 @@ use anyhow::anyhow; use parse_display::Display; -use risingwave_pb::catalog::{PbDatabase, PbSchema, PbTable}; -use sea_orm::{ActiveValue, EntityTrait, ModelTrait}; +use risingwave_pb::catalog::{PbDatabase, PbSchema}; +use sea_orm::{ActiveValue, ModelTrait}; use crate::model_v2::{database, object, schema}; use crate::MetaError; +#[allow(dead_code)] mod catalog_controller; +#[allow(dead_code)] +mod system_param_controller; // todo: refine the error transform. impl From for MetaError { @@ -31,6 +34,7 @@ impl From for MetaError { #[derive(Clone, Display, Debug)] #[display(style = "UPPERCASE")] +#[allow(dead_code)] enum ObjectType { Database, Schema, diff --git a/src/meta/src/controller/system_param_controller.rs b/src/meta/src/controller/system_param_controller.rs new file mode 100644 index 0000000000000..a3f1dc5b751dd --- /dev/null +++ b/src/meta/src/controller/system_param_controller.rs @@ -0,0 +1,331 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::anyhow; +use risingwave_common::system_param::reader::SystemParamsReader; +use risingwave_common::system_param::{ + check_missing_params, derive_missing_fields, set_system_param, +}; +use risingwave_common::{for_all_params, key_of}; +use risingwave_pb::meta::subscribe_response::{Info, Operation}; +use risingwave_pb::meta::PbSystemParams; +use sea_orm::{ActiveModelTrait, ActiveValue, DatabaseConnection, EntityTrait, TransactionTrait}; +use tokio::sync::oneshot::Sender; +use tokio::sync::RwLock; +use tokio::task::JoinHandle; +use tracing::info; + +use crate::manager::{LocalNotification, NotificationManagerRef}; +use crate::model_v2::prelude::SystemParameter; +use crate::model_v2::system_parameter; +use crate::{MetaError, MetaResult}; + +pub type SystemParamsControllerRef = Arc; + +pub struct SystemParamsController { + db: DatabaseConnection, + // Notify workers and local subscribers of parameter change. + notification_manager: NotificationManagerRef, + // Cached parameters. + params: RwLock, +} + +/// Derive system params from db models. +macro_rules! impl_system_params_from_db { + ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + /// Try to deserialize deprecated fields as well. + /// Warn if there are unrecognized fields. + pub fn system_params_from_db(mut models: Vec) -> MetaResult { + let mut params = PbSystemParams::default(); + models.retain(|model| { + match model.name.as_str() { + $( + key_of!($field) => { + params.$field = Some(model.value.parse::<$type>().unwrap()); + false + } + )* + _ => true, + } + }); + derive_missing_fields(&mut params); + if !models.is_empty() { + let unrecognized_params = models.into_iter().map(|model| model.name).collect::>(); + tracing::warn!("unrecognized system params {:?}", unrecognized_params); + } + Ok(params) + } + }; +} + +/// Derive serialization to db models. +macro_rules! impl_system_params_to_models { + ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + #[allow(clippy::vec_init_then_push)] + pub fn system_params_to_model(params: &PbSystemParams) -> MetaResult> { + check_missing_params(params).map_err(|e| anyhow!(e))?; + let mut models = Vec::new(); + $( + let value = params.$field.as_ref().unwrap().to_string(); + models.push(system_parameter::ActiveModel { + name: ActiveValue::Set(key_of!($field).to_string()), + value: ActiveValue::Set(value), + is_mutable: ActiveValue::Set($is_mutable), + description: ActiveValue::Set(None), + }); + )* + Ok(models) + } + }; +} + +// For each field in `persisted` and `init` +// 1. Some, None: The persisted field is deprecated, so just ignore it. +// 2. Some, Some: Check equality and warn if they differ. +// 3. None, Some: A new version of RW cluster is launched for the first time and newly introduced +// params are not set. Use init value. +// 4. None, None: A new version of RW cluster is launched for the first time and newly introduced +// params are not set. The new field is not initialized either, just leave it as `None`. +macro_rules! impl_merge_params { + ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + fn merge_params(mut persisted: PbSystemParams, init: PbSystemParams) -> PbSystemParams { + $( + match (persisted.$field.as_ref(), init.$field) { + (Some(persisted), Some(init)) => { + if persisted != &init { + tracing::warn!( + "The initializing value of \"{:?}\" ({}) differ from persisted ({}), using persisted value", + key_of!($field), + init, + persisted + ); + } + }, + (None, Some(init)) => persisted.$field = Some(init), + _ => {}, + } + )* + persisted + } + }; +} + +for_all_params!(impl_system_params_from_db); +for_all_params!(impl_merge_params); +for_all_params!(impl_system_params_to_models); + +impl SystemParamsController { + pub async fn new( + db: DatabaseConnection, + notification_manager: NotificationManagerRef, + init_params: PbSystemParams, + cluster_first_launch: bool, + ) -> MetaResult { + let params = if cluster_first_launch { + init_params + } else { + let params = SystemParameter::find().all(&db).await?; + if params.is_empty() { + return Err(MetaError::system_param( + "cluster is not newly created but no system parameters can be found", + )); + } + merge_params(system_params_from_db(params)?, init_params) + }; + + info!("system parameters: {:?}", params); + check_missing_params(¶ms).map_err(|e| anyhow!(e))?; + + let ctl = Self { + db, + notification_manager, + params: RwLock::new(params), + }; + // flush to db. + ctl.flush_params().await?; + + Ok(ctl) + } + + pub async fn get_pb_params(&self) -> PbSystemParams { + self.params.read().await.clone() + } + + pub async fn get_params(&self) -> SystemParamsReader { + self.params.read().await.clone().into() + } + + async fn flush_params(&self) -> MetaResult<()> { + let params = self.params.read().await; + let models = system_params_to_model(¶ms)?; + let txn = self.db.begin().await?; + // delete all params first and then insert all params. It follows the same logic + // as the old code, we'd better change it to another way later to keep consistency. + SystemParameter::delete_many().exec(&txn).await?; + + for model in models { + model.insert(&txn).await?; + } + txn.commit().await?; + Ok(()) + } + + pub async fn set_param(&self, name: &str, value: Option) -> MetaResult { + let mut params_guard = self.params.write().await; + + let Some(param) = SystemParameter::find_by_id(name.to_string()) + .one(&self.db) + .await? + else { + return Err(MetaError::system_param(format!( + "unrecognized system parameter {}", + name + ))); + }; + let mut params = params_guard.clone(); + let mut param: system_parameter::ActiveModel = param.into(); + param.value = ActiveValue::Set( + set_system_param(&mut params, name, value).map_err(MetaError::system_param)?, + ); + param.update(&self.db).await?; + *params_guard = params.clone(); + + // Sync params to other managers on the meta node only once, since it's infallible. + self.notification_manager + .notify_local_subscribers(LocalNotification::SystemParamsChange(params.clone().into())) + .await; + + // Sync params to worker nodes. + self.notify_workers(¶ms).await; + + Ok(params) + } + + // Periodically sync params to worker nodes. + pub fn start_params_notifier( + system_params_controller: Arc, + ) -> (JoinHandle<()>, Sender<()>) { + const NOTIFY_INTERVAL: Duration = Duration::from_millis(5000); + + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); + let join_handle = tokio::spawn(async move { + let mut interval = tokio::time::interval(NOTIFY_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + _ = interval.tick() => {}, + _ = &mut shutdown_rx => { + tracing::info!("System params notifier is stopped"); + return; + } + } + system_params_controller + .notify_workers(&*system_params_controller.params.read().await) + .await; + } + }); + + (join_handle, shutdown_tx) + } + + // Notify workers of parameter change. + async fn notify_workers(&self, params: &PbSystemParams) { + self.notification_manager + .notify_frontend(Operation::Update, Info::SystemParams(params.clone())) + .await; + self.notification_manager + .notify_compute(Operation::Update, Info::SystemParams(params.clone())) + .await; + self.notification_manager + .notify_compactor(Operation::Update, Info::SystemParams(params.clone())) + .await; + } +} + +#[cfg(test)] +mod tests { + use risingwave_common::system_param::system_params_for_test; + use sea_orm::{ConnectionTrait, Schema}; + + use super::*; + use crate::manager::MetaSrvEnv; + + #[tokio::test] + async fn test_system_params() { + let db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let builder = db.get_database_backend(); + let schema = Schema::new(builder); + let stmt = schema.create_table_from_entity(SystemParameter); + db.execute(builder.build(&stmt)).await.unwrap(); + + let env = MetaSrvEnv::for_test().await; + let init_params = system_params_for_test(); + + // init system parameter controller as first launch. + let system_param_ctl = SystemParamsController::new( + db, + env.notification_manager_ref(), + init_params.clone(), + true, + ) + .await + .unwrap(); + let params = system_param_ctl.get_pb_params().await; + assert_eq!(params, system_params_for_test()); + + // set parameter. + let new_params = system_param_ctl + .set_param("pause_on_next_bootstrap", Some("true".into())) + .await + .unwrap(); + + // insert deprecated params. + let deprecated_param = system_parameter::ActiveModel { + name: ActiveValue::Set("deprecated_param".into()), + value: ActiveValue::Set("foo".into()), + is_mutable: ActiveValue::Set(true), + description: ActiveValue::Set(None), + }; + deprecated_param.insert(&system_param_ctl.db).await.unwrap(); + + // init system parameter controller as not first launch. + let system_param_ctl = SystemParamsController::new( + system_param_ctl.db, + env.notification_manager_ref(), + init_params.clone(), + false, + ) + .await + .unwrap(); + // check deprecated params are cleaned up. + assert!(SystemParameter::find_by_id("deprecated_param".to_string()) + .one(&system_param_ctl.db) + .await + .unwrap() + .is_none()); + // check new params are set. + let params = system_param_ctl.get_pb_params().await; + assert_eq!(params, new_params); + // check db consistency. + let models = SystemParameter::find() + .all(&system_param_ctl.db) + .await + .unwrap(); + let db_params = system_params_from_db(models).unwrap(); + assert_eq!(db_params, new_params); + } +} diff --git a/src/meta/src/model_v2/connection.rs b/src/meta/src/model_v2/connection.rs index 69e5235b4b0e6..e105ee1e4047a 100644 --- a/src/meta/src/model_v2/connection.rs +++ b/src/meta/src/model_v2/connection.rs @@ -28,7 +28,7 @@ pub enum Relation { from = "Column::ConnectionId", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object, #[sea_orm( diff --git a/src/meta/src/model_v2/database.rs b/src/meta/src/model_v2/database.rs index 7c8ef63c89eb0..c0fdeca4bb959 100644 --- a/src/meta/src/model_v2/database.rs +++ b/src/meta/src/model_v2/database.rs @@ -24,7 +24,7 @@ pub enum Relation { from = "Column::DatabaseId", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object, #[sea_orm(has_many = "super::schema::Entity")] diff --git a/src/meta/src/model_v2/function.rs b/src/meta/src/model_v2/function.rs index 04313f4d8ac0d..e0855613b71da 100644 --- a/src/meta/src/model_v2/function.rs +++ b/src/meta/src/model_v2/function.rs @@ -33,7 +33,7 @@ pub enum Relation { from = "Column::FunctionId", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object, #[sea_orm( diff --git a/src/meta/src/model_v2/index.rs b/src/meta/src/model_v2/index.rs index 4edd8457c4434..69810e1ea4214 100644 --- a/src/meta/src/model_v2/index.rs +++ b/src/meta/src/model_v2/index.rs @@ -31,7 +31,7 @@ pub enum Relation { from = "Column::IndexId", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object, #[sea_orm( diff --git a/src/meta/src/model_v2/migration/Cargo.toml b/src/meta/src/model_v2/migration/Cargo.toml index 722f8ae0ba225..8a8bc23dd9759 100644 --- a/src/meta/src/model_v2/migration/Cargo.toml +++ b/src/meta/src/model_v2/migration/Cargo.toml @@ -10,7 +10,8 @@ path = "src/lib.rs" [dependencies] async-std = { version = "1", features = ["attributes", "tokio1"] } +uuid = { version = "1", features = ["v4"] } [dependencies.sea-orm-migration] version = "0.12.0" -features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls" ] +features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls", "with-uuid" ] diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs index efa805de32def..8b9384882f21f 100644 --- a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -7,6 +7,7 @@ pub struct Migration; impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { // 1. check if the table exists. + assert!(!manager.has_table(Cluster::Table.to_string()).await?); assert!(!manager.has_table(Worker::Table.to_string()).await?); assert!(!manager.has_table(WorkerProperty::Table.to_string()).await?); assert!(!manager.has_table(User::Table.to_string()).await?); @@ -28,8 +29,32 @@ impl MigrationTrait for Migration { .has_table(ObjectDependency::Table.to_string()) .await? ); + assert!( + !manager + .has_table(SystemParameter::Table.to_string()) + .await? + ); // 2. create tables. + manager + .create_table( + MigrationTable::create() + .table(Cluster::Table) + .col( + ColumnDef::new(Cluster::ClusterId) + .uuid() + .not_null() + .primary_key(), + ) + .col( + ColumnDef::new(Cluster::CreatedAt) + .timestamp() + .default(Expr::current_timestamp()) + .not_null(), + ) + .to_owned(), + ) + .await?; manager .create_table( MigrationTable::create() @@ -377,6 +402,26 @@ impl MigrationTrait for Migration { .to_owned(), ) .await?; + manager + .create_table( + MigrationTable::create() + .table(SystemParameter::Table) + .col( + ColumnDef::new(SystemParameter::Name) + .string() + .primary_key() + .not_null(), + ) + .col(ColumnDef::new(SystemParameter::Value).string().not_null()) + .col( + ColumnDef::new(SystemParameter::IsMutable) + .boolean() + .not_null(), + ) + .col(ColumnDef::new(SystemParameter::Description).string()) + .to_owned(), + ) + .await?; // 3. create indexes. manager @@ -474,6 +519,7 @@ impl MigrationTrait for Migration { .name("FK_fragment_table_id") .from(Fragment::Table, Fragment::TableId) .to(Table::Table, Table::TableId) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -483,6 +529,7 @@ impl MigrationTrait for Migration { .name("FK_actor_fragment_id") .from(Actor::Table, Actor::FragmentId) .to(Fragment::Table, Fragment::FragmentId) + .on_delete(ForeignKeyAction::Cascade) .to_owned(), ) .await?; @@ -784,6 +831,11 @@ impl MigrationTrait for Migration { .await?; // 4. initialize data. + let insert_cluster_id = Query::insert() + .into_table(Cluster::Table) + .columns([Cluster::ClusterId]) + .values_panic([uuid::Uuid::new_v4().into()]) + .to_owned(); let insert_sys_users = Query::insert() .into_table(User::Table) .columns([ @@ -835,6 +887,7 @@ impl MigrationTrait for Migration { .values_panic([5.into(), "rw_catalog".into(), 1.into()]) .to_owned(); + manager.exec_stmt(insert_cluster_id).await?; manager.exec_stmt(insert_sys_users).await?; manager.exec_stmt(insert_objects).await?; manager.exec_stmt(insert_sys_database).await?; @@ -863,6 +916,7 @@ impl MigrationTrait for Migration { // drop tables cascade. drop_tables!( manager, + Cluster, Worker, WorkerProperty, User, @@ -879,12 +933,20 @@ impl MigrationTrait for Migration { Index, Function, Object, - ObjectDependency + ObjectDependency, + SystemParameter ); Ok(()) } } +#[derive(DeriveIden)] +enum Cluster { + Table, + ClusterId, + CreatedAt, +} + #[derive(DeriveIden)] enum Worker { Table, @@ -1104,3 +1166,12 @@ enum ObjectDependency { Oid, UsedBy, } + +#[derive(DeriveIden)] +enum SystemParameter { + Table, + Name, + Value, + IsMutable, + Description, +} diff --git a/src/meta/src/model_v2/mod.rs b/src/meta/src/model_v2/mod.rs index c798a5ac88ba1..59c2876f15bd7 100644 --- a/src/meta/src/model_v2/mod.rs +++ b/src/meta/src/model_v2/mod.rs @@ -3,6 +3,7 @@ pub mod prelude; pub mod actor; +pub mod cluster; pub mod connection; pub mod database; pub mod fragment; @@ -13,6 +14,7 @@ pub mod object_dependency; pub mod schema; pub mod sink; pub mod source; +pub mod system_parameter; pub mod table; pub mod user; pub mod user_privilege; diff --git a/src/meta/src/model_v2/object_dependency.rs b/src/meta/src/model_v2/object_dependency.rs index 5f3dfbb8d00e7..b81c19361e733 100644 --- a/src/meta/src/model_v2/object_dependency.rs +++ b/src/meta/src/model_v2/object_dependency.rs @@ -26,7 +26,7 @@ pub enum Relation { from = "Column::UsedBy", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object1, } diff --git a/src/meta/src/model_v2/prelude.rs b/src/meta/src/model_v2/prelude.rs index 47cce4ad9c047..acdf40decd72f 100644 --- a/src/meta/src/model_v2/prelude.rs +++ b/src/meta/src/model_v2/prelude.rs @@ -1,6 +1,7 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 pub use super::actor::Entity as Actor; +pub use super::cluster::Entity as Cluster; pub use super::connection::Entity as Connection; pub use super::database::Entity as Database; pub use super::fragment::Entity as Fragment; @@ -11,6 +12,7 @@ pub use super::object_dependency::Entity as ObjectDependency; pub use super::schema::Entity as Schema; pub use super::sink::Entity as Sink; pub use super::source::Entity as Source; +pub use super::system_parameter::Entity as SystemParameter; pub use super::table::Entity as Table; pub use super::user::Entity as User; pub use super::user_privilege::Entity as UserPrivilege; diff --git a/src/meta/src/model_v2/schema.rs b/src/meta/src/model_v2/schema.rs index 8216bf948a1bc..759ab4c0fc9fc 100644 --- a/src/meta/src/model_v2/schema.rs +++ b/src/meta/src/model_v2/schema.rs @@ -32,7 +32,7 @@ pub enum Relation { from = "Column::SchemaId", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object, #[sea_orm(has_many = "super::sink::Entity")] diff --git a/src/meta/src/model_v2/sink.rs b/src/meta/src/model_v2/sink.rs index 8ce1be2c55093..d67cb632d6c0f 100644 --- a/src/meta/src/model_v2/sink.rs +++ b/src/meta/src/model_v2/sink.rs @@ -43,7 +43,7 @@ pub enum Relation { from = "Column::SinkId", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object, #[sea_orm( diff --git a/src/meta/src/model_v2/source.rs b/src/meta/src/model_v2/source.rs index edc4f5cefca09..da96e42b9c50c 100644 --- a/src/meta/src/model_v2/source.rs +++ b/src/meta/src/model_v2/source.rs @@ -44,7 +44,7 @@ pub enum Relation { from = "Column::SourceId", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object, #[sea_orm( diff --git a/src/meta/src/model_v2/table.rs b/src/meta/src/model_v2/table.rs index ed0773bfba015..d36df4b47a813 100644 --- a/src/meta/src/model_v2/table.rs +++ b/src/meta/src/model_v2/table.rs @@ -62,7 +62,7 @@ pub enum Relation { from = "Column::TableId", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object, #[sea_orm( diff --git a/src/meta/src/model_v2/view.rs b/src/meta/src/model_v2/view.rs index 807a2390ac3a2..9f520e73d9fa0 100644 --- a/src/meta/src/model_v2/view.rs +++ b/src/meta/src/model_v2/view.rs @@ -30,7 +30,7 @@ pub enum Relation { from = "Column::ViewId", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object, #[sea_orm( From 67b3a3bd40ab7d9546445e6fb28203271f5467a8 Mon Sep 17 00:00:00 2001 From: August Date: Wed, 13 Sep 2023 20:00:10 +0800 Subject: [PATCH 07/30] update models --- src/meta/src/model_v2/actor.rs | 2 +- src/meta/src/model_v2/fragment.rs | 2 +- src/meta/src/model_v2/migration/src/m20230908_072257_init.rs | 2 +- src/meta/src/model_v2/object_dependency.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/meta/src/model_v2/actor.rs b/src/meta/src/model_v2/actor.rs index 7ce243aa981ce..2ed597197a5c7 100644 --- a/src/meta/src/model_v2/actor.rs +++ b/src/meta/src/model_v2/actor.rs @@ -23,7 +23,7 @@ pub enum Relation { from = "Column::FragmentId", to = "super::fragment::Column::FragmentId", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Fragment, } diff --git a/src/meta/src/model_v2/fragment.rs b/src/meta/src/model_v2/fragment.rs index 8457f390012c2..8054c59bf8063 100644 --- a/src/meta/src/model_v2/fragment.rs +++ b/src/meta/src/model_v2/fragment.rs @@ -28,7 +28,7 @@ pub enum Relation { from = "Column::TableId", to = "super::table::Column::TableId", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Table, } diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs index 8b9384882f21f..a99c83089dafa 100644 --- a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -392,7 +392,7 @@ impl MigrationTrait for Migration { .create_table( MigrationTable::create() .table(ObjectDependency::Table) - .col(ColumnDef::new(ObjectDependency::Id).integer().primary_key()) + .col(ColumnDef::new(ObjectDependency::Id).integer().auto_increment().primary_key()) .col(ColumnDef::new(ObjectDependency::Oid).integer().not_null()) .col( ColumnDef::new(ObjectDependency::UsedBy) diff --git a/src/meta/src/model_v2/object_dependency.rs b/src/meta/src/model_v2/object_dependency.rs index b81c19361e733..36abc298fec57 100644 --- a/src/meta/src/model_v2/object_dependency.rs +++ b/src/meta/src/model_v2/object_dependency.rs @@ -5,7 +5,7 @@ use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "object_dependency")] pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] + #[sea_orm(primary_key)] pub id: i32, pub oid: i32, pub used_by: i32, From d43c58e1f2a1795086910dc7f3d2b56e5d393ac2 Mon Sep 17 00:00:00 2001 From: August Date: Thu, 14 Sep 2023 12:20:55 +0800 Subject: [PATCH 08/30] add models --- src/meta/src/model_v2/cluster.rs | 16 ++++++++++++++++ src/meta/src/model_v2/system_parameter.rs | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 src/meta/src/model_v2/cluster.rs create mode 100644 src/meta/src/model_v2/system_parameter.rs diff --git a/src/meta/src/model_v2/cluster.rs b/src/meta/src/model_v2/cluster.rs new file mode 100644 index 0000000000000..be1e662f07913 --- /dev/null +++ b/src/meta/src/model_v2/cluster.rs @@ -0,0 +1,16 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "cluster")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub cluster_id: Uuid, + pub created_at: DateTime, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/system_parameter.rs b/src/meta/src/model_v2/system_parameter.rs new file mode 100644 index 0000000000000..422e10f0722b2 --- /dev/null +++ b/src/meta/src/model_v2/system_parameter.rs @@ -0,0 +1,18 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "system_parameter")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub name: String, + pub value: String, + pub is_mutable: bool, + pub description: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} From 98cdaaf7c5039c54d17e9b7a7cb52fda49d31a69 Mon Sep 17 00:00:00 2001 From: August Date: Thu, 14 Sep 2023 16:01:03 +0800 Subject: [PATCH 09/30] add find all downstream objects --- src/meta/src/controller/catalog_controller.rs | 93 ++++++++++++++++++- .../migration/src/m20230908_072257_init.rs | 7 +- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/src/meta/src/controller/catalog_controller.rs b/src/meta/src/controller/catalog_controller.rs index b2a410b966f6f..fad11a0547dc0 100644 --- a/src/meta/src/controller/catalog_controller.rs +++ b/src/meta/src/controller/catalog_controller.rs @@ -21,11 +21,14 @@ use risingwave_pb::catalog::PbDatabase; use risingwave_pb::meta::subscribe_response::{ Info as NotificationInfo, Operation as NotificationOperation, }; -use sea_orm::sea_query::{Query, UnionType}; +use sea_orm::sea_query::{ + Alias, CommonTableExpression, Expr, Query, QueryStatementBuilder, SelectStatement, UnionType, + WithClause, +}; use sea_orm::{ ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectOptions, - ConnectionTrait, Database as SeaDB, DatabaseConnection, DatabaseTransaction, EntityTrait, - ModelTrait, QueryFilter, QuerySelect, TransactionTrait, + ConnectionTrait, Database as SeaDB, DatabaseConnection, DatabaseTransaction, EntityTrait, Iden, + JoinType, ModelTrait, Order, QueryFilter, Statement, TransactionTrait, }; use tokio::sync::RwLock; @@ -33,7 +36,8 @@ use crate::controller::{ModelWithObj, ObjectType}; use crate::manager::{DatabaseId, MetaSrvEnv, NotificationVersion, UserId}; use crate::model_v2::prelude::*; use crate::model_v2::{ - connection, database, function, index, object, schema, sink, source, table, view, + connection, database, function, index, object, object_dependency, schema, sink, source, table, + view, }; use crate::{MetaError, MetaResult}; @@ -147,6 +151,87 @@ impl CatalogController { Ok(version) } + /// This function will list all the objects that are used the given one. It runs a recursive CTE to find all the dependencies. + /// The cte and the query is as follows: + /// ```sql + /// WITH RECURSIVE used_by_object_ids (used_by) AS + /// ( + /// SELECT used_by FROM object_dependency WHERE object_dependency.oid = $1 + /// UNION ALL + /// ( + /// SELECT object_dependency.used_by + /// FROM object_dependency + /// INNER JOIN used_by_object_ids + /// ON used_by_object_ids.used_by = oid + /// ) + /// ) + /// SELECT DISTINCT used_by FROM used_by_object_ids ORDER BY used_by DESC; + /// ``` + async fn list_used_by(&self, obj_id: i32) -> MetaResult> { + let inner = self.inner.read().await; + + let cte_alias = Alias::new("used_by_object_ids"); + let cte_return_alias = Alias::new("used_by"); + + let base_query = SelectStatement::new() + .column(object_dependency::Column::UsedBy) + .from(object_dependency::Entity) + .and_where(object_dependency::Column::Oid.eq(obj_id)) + .to_owned(); + + let cte_referencing = Query::select() + .column((object_dependency::Entity, object_dependency::Column::UsedBy)) + .from(object_dependency::Entity) + .join( + JoinType::InnerJoin, + cte_alias.clone(), + Expr::col((cte_alias.clone(), cte_return_alias.clone())) + .equals(object_dependency::Column::Oid), + ) + .to_owned(); + + let common_table_expr = CommonTableExpression::new() + .query( + base_query + .clone() + .union(UnionType::All, cte_referencing) + .to_owned(), + ) + .column(cte_return_alias.clone()) + .table_name(cte_alias.clone()) + .to_owned(); + + let query = SelectStatement::new() + .distinct() + .column(cte_return_alias.clone()) + .from(cte_alias) + .order_by(cte_return_alias.clone(), Order::Desc) + .to_owned() + .with( + WithClause::new() + .recursive(true) + .cte(common_table_expr) + .to_owned(), + ) + .to_owned(); + + let (sql, values) = query.build_any(&*inner.db.get_database_backend().get_query_builder()); + let res = inner + .db + .query_all(Statement::from_sql_and_values( + inner.db.get_database_backend(), + sql, + values, + )) + .await?; + + let ids: Vec = res + .into_iter() + .map(|row| row.try_get("", &cte_return_alias.to_string()).unwrap()) + .collect_vec(); + Ok(ids) + } + pub async fn drop_database( &self, database_id: DatabaseId, diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs index a99c83089dafa..0160ca51604cb 100644 --- a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -392,7 +392,12 @@ impl MigrationTrait for Migration { .create_table( MigrationTable::create() .table(ObjectDependency::Table) - .col(ColumnDef::new(ObjectDependency::Id).integer().auto_increment().primary_key()) + .col( + ColumnDef::new(ObjectDependency::Id) + .integer() + .auto_increment() + .primary_key(), + ) .col(ColumnDef::new(ObjectDependency::Oid).integer().not_null()) .col( ColumnDef::new(ObjectDependency::UsedBy) From 847ac769dbce2556c5ef7d13ccb9c84e1fc3abfc Mon Sep 17 00:00:00 2001 From: August Date: Thu, 14 Sep 2023 16:09:08 +0800 Subject: [PATCH 10/30] fix drop database --- src/meta/src/controller/catalog_controller.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/meta/src/controller/catalog_controller.rs b/src/meta/src/controller/catalog_controller.rs index fad11a0547dc0..a1100ce625a89 100644 --- a/src/meta/src/controller/catalog_controller.rs +++ b/src/meta/src/controller/catalog_controller.rs @@ -318,15 +318,18 @@ impl CatalogController { .union(UnionType::All, query_view) .union(UnionType::All, query_function) .union(UnionType::All, query_connection) + .union( + UnionType::All, + Query::select() + .expr(Expr::value(database_id as i32)) + .to_owned(), + ) .to_owned(); // drop related objects, the relations will be dropped cascade. Object::delete_many() .filter(object::Column::Oid.in_subquery(query_all)) .exec(&txn) .await?; - Database::delete(database::ActiveModel::from(db)) - .exec(&txn) - .await?; txn.commit().await?; From 20dc75732da90ea5949336304791f02c40285330 Mon Sep 17 00:00:00 2001 From: August Date: Fri, 15 Sep 2023 17:31:11 +0800 Subject: [PATCH 11/30] rename --- .../{catalog_controller.rs => catalog.rs} | 2 +- src/meta/src/controller/mod.rs | 19 ++----------------- ...em_param_controller.rs => system_param.rs} | 0 3 files changed, 3 insertions(+), 18 deletions(-) rename src/meta/src/controller/{catalog_controller.rs => catalog.rs} (99%) rename src/meta/src/controller/{system_param_controller.rs => system_param.rs} (100%) diff --git a/src/meta/src/controller/catalog_controller.rs b/src/meta/src/controller/catalog.rs similarity index 99% rename from src/meta/src/controller/catalog_controller.rs rename to src/meta/src/controller/catalog.rs index a1100ce625a89..9ee6fcf31e6bb 100644 --- a/src/meta/src/controller/catalog_controller.rs +++ b/src/meta/src/controller/catalog.rs @@ -151,7 +151,7 @@ impl CatalogController { Ok(version) } - /// This function will list all the objects that are used the given one. It runs a recursive CTE to find all the dependencies. + /// This function will list all the objects that are using the given one. It runs a recursive CTE to find all the dependencies. /// The cte and the query is as follows: /// ```sql /// WITH RECURSIVE used_by_object_ids (used_by) AS diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index d27d14ebc7fee..4ad26b6171303 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -21,9 +21,9 @@ use crate::model_v2::{database, object, schema}; use crate::MetaError; #[allow(dead_code)] -mod catalog_controller; +mod catalog; #[allow(dead_code)] -mod system_param_controller; +mod system_param; // todo: refine the error transform. impl From for MetaError { @@ -78,18 +78,3 @@ impl From> for PbSchema { } } } - -// impl From for PbUserInfo { -// fn from(model: user::Model) -> Self { -// Self { -// id: model.user_id as _, -// name: model.name, -// is_super: model.is_super.unwrap_or_default(), -// can_create_db: model.can_create_db.unwrap_or_default(), -// can_create_user: model.can_create_user.unwrap_or_default(), -// can_login: model.can_login.unwrap_or_default(), -// auth_info: None, -// grant_privileges: vec![], -// } -// } -// } diff --git a/src/meta/src/controller/system_param_controller.rs b/src/meta/src/controller/system_param.rs similarity index 100% rename from src/meta/src/controller/system_param_controller.rs rename to src/meta/src/controller/system_param.rs From 14496b2f6e329f69277b80463cb82e70cc99f16a Mon Sep 17 00:00:00 2001 From: August Date: Mon, 18 Sep 2023 17:42:38 +0800 Subject: [PATCH 12/30] apply sql meta store for system params and telemetry --- src/meta/Cargo.toml | 2 +- src/meta/src/controller/catalog.rs | 41 +++++------- src/meta/src/controller/mod.rs | 31 ++++++--- src/meta/src/controller/system_param.rs | 30 +++------ src/meta/src/lib.rs | 10 ++- src/meta/src/manager/env.rs | 59 ++++++++++++++++- src/meta/src/rpc/server.rs | 66 ++++++++++++++++--- .../src/rpc/service/system_params_service.rs | 31 +++++++-- src/meta/src/rpc/service/telemetry_service.rs | 25 +++++-- 9 files changed, 218 insertions(+), 77 deletions(-) diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml index 8de06f5772f6d..b1e3d1b7a2248 100644 --- a/src/meta/Cargo.toml +++ b/src/meta/Cargo.toml @@ -55,7 +55,7 @@ risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } risingwave_sqlparser = { workspace = true } scopeguard = "1.2.0" -sea-orm = { version = "0.12.0", features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "mock", "runtime-tokio-native-tls", "macros" ] } +sea-orm = { version = "0.12.0", features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls", "macros" ] } serde = { version = "1", features = ["derive"] } serde_json = "1" sync-point = { path = "../utils/sync-point" } diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index 9ee6fcf31e6bb..6fe661cb017f3 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::iter; -use std::time::Duration; use itertools::Itertools; use risingwave_common::catalog::{DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS}; @@ -26,13 +25,13 @@ use sea_orm::sea_query::{ WithClause, }; use sea_orm::{ - ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectOptions, - ConnectionTrait, Database as SeaDB, DatabaseConnection, DatabaseTransaction, EntityTrait, Iden, - JoinType, ModelTrait, Order, QueryFilter, Statement, TransactionTrait, + ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, + DatabaseConnection, DatabaseTransaction, EntityTrait, Iden, JoinType, ModelTrait, Order, + QueryFilter, Statement, TransactionTrait, }; use tokio::sync::RwLock; -use crate::controller::{ModelWithObj, ObjectType}; +use crate::controller::{ObjectModel, ObjectType}; use crate::manager::{DatabaseId, MetaSrvEnv, NotificationVersion, UserId}; use crate::model_v2::prelude::*; use crate::model_v2::{ @@ -54,15 +53,15 @@ pub struct ReleaseContext { } impl CatalogController { - pub async fn new(env: MetaSrvEnv, url: &str) -> MetaResult { - let mut opts = ConnectOptions::new(url); - opts.max_connections(20) - .connect_timeout(Duration::from_secs(10)); - - let db = SeaDB::connect(opts).await?; + pub async fn new(env: MetaSrvEnv) -> MetaResult { + let meta_store = env + .sql_meta_store() + .expect("sql meta store is not initialized"); Ok(Self { env, - inner: RwLock::new(CatalogControllerInner { db }), + inner: RwLock::new(CatalogControllerInner { + db: meta_store.conn, + }), }) } } @@ -98,7 +97,7 @@ impl CatalogController { Ok(dbs .into_iter() - .map(|(db, obj)| ModelWithObj(db, obj.unwrap()).into()) + .map(|(db, obj)| ObjectModel(db, obj.unwrap()).into()) .collect()) } @@ -132,14 +131,14 @@ impl CatalogController { schema.name = ActiveValue::Set(schema_name.into()); let schema = schema.insert(&txn).await?; - schemas.push(ModelWithObj(schema, schema_obj).into()); + schemas.push(ObjectModel(schema, schema_obj).into()); } txn.commit().await?; let mut version = self .notify_frontend( NotificationOperation::Add, - NotificationInfo::Database(ModelWithObj(db, db_obj).into()), + NotificationInfo::Database(ObjectModel(db, db_obj).into()), ) .await; for schema in schemas { @@ -244,7 +243,7 @@ impl CatalogController { .await? .ok_or_else(|| MetaError::catalog_id_not_found("database", database_id))?; let db_obj = db.find_related(Object).one(&txn).await?; - let pb_db = ModelWithObj(db.clone(), db_obj.unwrap()).into(); + let pb_db = ObjectModel(db.clone(), db_obj.unwrap()).into(); let tables = db.find_related(Table).all(&txn).await?; let sinks = db.find_related(Sink).all(&txn).await?; @@ -359,13 +358,9 @@ mod tests { #[tokio::test] #[ignore] async fn test_create_database() { - // let conn = MockDatabase::new(DbBackend::Postgres).into_connection(); - let mgr = CatalogController::new( - MetaSrvEnv::for_test().await, - "postgres://postgres:@localhost:5432/postgres", - ) - .await - .unwrap(); + let mgr = CatalogController::new(MetaSrvEnv::for_test().await) + .await + .unwrap(); let db = PbDatabase { name: "test".to_string(), owner: DEFAULT_SUPER_USER_ID, diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index 4ad26b6171303..b6f8438994bfa 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -15,15 +15,14 @@ use anyhow::anyhow; use parse_display::Display; use risingwave_pb::catalog::{PbDatabase, PbSchema}; -use sea_orm::{ActiveValue, ModelTrait}; +use sea_orm::{ActiveValue, DatabaseConnection, ModelTrait}; use crate::model_v2::{database, object, schema}; use crate::MetaError; #[allow(dead_code)] mod catalog; -#[allow(dead_code)] -mod system_param; +pub mod system_param; // todo: refine the error transform. impl From for MetaError { @@ -32,6 +31,22 @@ impl From for MetaError { } } +#[derive(Clone)] +pub struct SqlMetaStore { + pub(crate) conn: DatabaseConnection, +} + +impl SqlMetaStore { + pub fn new(conn: DatabaseConnection) -> Self { + Self { conn } + } + + pub async fn for_test() -> Self { + let conn = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + Self { conn } + } +} + #[derive(Clone, Display, Debug)] #[display(style = "UPPERCASE")] #[allow(dead_code)] @@ -47,10 +62,10 @@ enum ObjectType { Connection, } -pub struct ModelWithObj(M, object::Model); +pub struct ObjectModel(M, object::Model); -impl From> for PbDatabase { - fn from(value: ModelWithObj) -> Self { +impl From> for PbDatabase { + fn from(value: ObjectModel) -> Self { Self { id: value.0.database_id as _, name: value.0.name, @@ -68,8 +83,8 @@ impl From for database::ActiveModel { } } -impl From> for PbSchema { - fn from(value: ModelWithObj) -> Self { +impl From> for PbSchema { + fn from(value: ObjectModel) -> Self { Self { id: value.0.schema_id as _, name: value.0.name, diff --git a/src/meta/src/controller/system_param.rs b/src/meta/src/controller/system_param.rs index a3f1dc5b751dd..0a326a72f086c 100644 --- a/src/meta/src/controller/system_param.rs +++ b/src/meta/src/controller/system_param.rs @@ -29,6 +29,7 @@ use tokio::sync::RwLock; use tokio::task::JoinHandle; use tracing::info; +use crate::controller::SqlMetaStore; use crate::manager::{LocalNotification, NotificationManagerRef}; use crate::model_v2::prelude::SystemParameter; use crate::model_v2::system_parameter; @@ -130,22 +131,13 @@ for_all_params!(impl_system_params_to_models); impl SystemParamsController { pub async fn new( - db: DatabaseConnection, + sql_meta_store: SqlMetaStore, notification_manager: NotificationManagerRef, init_params: PbSystemParams, - cluster_first_launch: bool, ) -> MetaResult { - let params = if cluster_first_launch { - init_params - } else { - let params = SystemParameter::find().all(&db).await?; - if params.is_empty() { - return Err(MetaError::system_param( - "cluster is not newly created but no system parameters can be found", - )); - } - merge_params(system_params_from_db(params)?, init_params) - }; + let db = sql_meta_store.conn; + let params = SystemParameter::find().all(&db).await?; + let params = merge_params(system_params_from_db(params)?, init_params); info!("system parameters: {:?}", params); check_missing_params(¶ms).map_err(|e| anyhow!(e))?; @@ -266,21 +258,20 @@ mod tests { #[tokio::test] async fn test_system_params() { - let db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); - let builder = db.get_database_backend(); + let meta_store = SqlMetaStore::for_test().await; + let builder = meta_store.conn.get_database_backend(); let schema = Schema::new(builder); let stmt = schema.create_table_from_entity(SystemParameter); - db.execute(builder.build(&stmt)).await.unwrap(); + meta_store.conn.execute(builder.build(&stmt)).await.unwrap(); let env = MetaSrvEnv::for_test().await; let init_params = system_params_for_test(); // init system parameter controller as first launch. let system_param_ctl = SystemParamsController::new( - db, + meta_store.clone(), env.notification_manager_ref(), init_params.clone(), - true, ) .await .unwrap(); @@ -304,10 +295,9 @@ mod tests { // init system parameter controller as not first launch. let system_param_ctl = SystemParamsController::new( - system_param_ctl.db, + meta_store, env.notification_manager_ref(), init_params.clone(), - false, ) .await .unwrap(); diff --git a/src/meta/src/lib.rs b/src/meta/src/lib.rs index 6c3f068258892..f214f359cbcbb 100644 --- a/src/meta/src/lib.rs +++ b/src/meta/src/lib.rs @@ -59,7 +59,7 @@ use risingwave_common::{GIT_SHA, RW_VERSION}; pub use rpc::{ElectionClient, ElectionMember, EtcdElectionClient}; use crate::manager::MetaOpts; -use crate::rpc::server::{rpc_serve, AddressInfo, MetaStoreBackend}; +use crate::rpc::server::{rpc_serve, AddressInfo, MetaStoreBackend, MetaStoreSqlBackend}; #[derive(Debug, Clone, Parser, OverrideConfig)] #[command(version, about = "The central metadata management service")] @@ -102,6 +102,10 @@ pub struct MetaNodeOpts { #[clap(long, env = "RW_ETCD_PASSWORD", default_value = "")] etcd_password: String, + /// Endpoint of the SQL service, make it non-option when SQL service is required. + #[clap(long, env = "RW_SQL_ENDPOINT")] + sql_endpoint: Option, + #[clap(long, env = "RW_DASHBOARD_UI_PATH")] dashboard_ui_path: Option, @@ -222,6 +226,9 @@ pub fn start(opts: MetaNodeOpts) -> Pin + Send>> { }, MetaBackend::Mem => MetaStoreBackend::Mem, }; + let sql_backend = opts + .sql_endpoint + .map(|endpoint| MetaStoreSqlBackend { endpoint }); validate_config(&config); @@ -251,6 +258,7 @@ pub fn start(opts: MetaNodeOpts) -> Pin + Send>> { let (mut join_handle, leader_lost_handle, shutdown_send) = rpc_serve( add_info, backend, + sql_backend, max_heartbeat_interval, config.meta.meta_leader_lease_secs, MetaOpts { diff --git a/src/meta/src/manager/env.rs b/src/meta/src/manager/env.rs index 40f81dbfd7a64..b876fb71a9639 100644 --- a/src/meta/src/manager/env.rs +++ b/src/meta/src/manager/env.rs @@ -18,13 +18,17 @@ use std::sync::Arc; use risingwave_common::config::{CompactionConfig, DefaultParallelism}; use risingwave_pb::meta::SystemParams; use risingwave_rpc_client::{ConnectorClient, StreamClientPool, StreamClientPoolRef}; +use sea_orm::EntityTrait; use super::{SystemParamsManager, SystemParamsManagerRef}; +use crate::controller::system_param::{SystemParamsController, SystemParamsControllerRef}; +use crate::controller::SqlMetaStore; use crate::manager::{ IdGeneratorManager, IdGeneratorManagerRef, IdleManager, IdleManagerRef, NotificationManager, NotificationManagerRef, }; use crate::model::ClusterId; +use crate::model_v2::prelude::Cluster; use crate::storage::MetaStoreRef; #[cfg(any(test, feature = "test"))] use crate::storage::{MemStore, MetaStoreBoxExt}; @@ -40,6 +44,9 @@ pub struct MetaSrvEnv { /// meta store. meta_store: MetaStoreRef, + /// sql meta store. + meta_store_sql: Option, + /// notification manager. notification_manager: NotificationManagerRef, @@ -52,6 +59,9 @@ pub struct MetaSrvEnv { /// system param manager. system_params_manager: SystemParamsManagerRef, + /// system param controller. + system_params_controller: Option, + /// Unique identifier of the cluster. cluster_id: ClusterId, @@ -205,13 +215,14 @@ impl MetaSrvEnv { opts: MetaOpts, init_system_params: SystemParams, meta_store: MetaStoreRef, + meta_store_sql: Option, ) -> MetaResult { // change to sync after refactor `IdGeneratorManager::new` sync. let id_gen_manager = Arc::new(IdGeneratorManager::new(meta_store.clone()).await); let stream_client_pool = Arc::new(StreamClientPool::default()); let notification_manager = Arc::new(NotificationManager::new(meta_store.clone()).await); let idle_manager = Arc::new(IdleManager::new(opts.max_idle_ms)); - let (cluster_id, cluster_first_launch) = + let (mut cluster_id, cluster_first_launch) = if let Some(id) = ClusterId::from_meta_store(&meta_store).await? { (id, false) } else { @@ -221,21 +232,41 @@ impl MetaSrvEnv { SystemParamsManager::new( meta_store.clone(), notification_manager.clone(), - init_system_params, + init_system_params.clone(), cluster_first_launch, ) .await?, ); + let system_params_controller = match &meta_store_sql { + Some(store) => { + cluster_id = Cluster::find() + .one(&store.conn) + .await? + .map(|c| c.cluster_id.to_string().into()) + .unwrap(); + Some(Arc::new( + SystemParamsController::new( + store.clone(), + notification_manager.clone(), + init_system_params, + ) + .await?, + )) + } + None => None, + }; let connector_client = ConnectorClient::try_new(opts.connector_rpc_endpoint.as_ref()).await; Ok(Self { id_gen_manager, meta_store, + meta_store_sql, notification_manager, stream_client_pool, idle_manager, system_params_manager, + system_params_controller, cluster_id, cluster_first_launch, connector_client, @@ -251,6 +282,10 @@ impl MetaSrvEnv { &self.meta_store } + pub fn sql_meta_store(&self) -> Option { + self.meta_store_sql.clone() + } + pub fn id_gen_manager_ref(&self) -> IdGeneratorManagerRef { self.id_gen_manager.clone() } @@ -283,6 +318,14 @@ impl MetaSrvEnv { self.system_params_manager.deref() } + pub fn system_params_controller_ref(&self) -> Option { + self.system_params_controller.clone() + } + + pub fn system_params_controller(&self) -> Option<&SystemParamsControllerRef> { + self.system_params_controller.as_ref() + } + pub fn stream_client_pool_ref(&self) -> StreamClientPoolRef { self.stream_client_pool.clone() } @@ -314,6 +357,7 @@ impl MetaSrvEnv { pub async fn for_test_opts(opts: Arc) -> Self { // change to sync after refactor `IdGeneratorManager::new` sync. let meta_store = MemStore::default().into_ref(); + let sql_meta_store = SqlMetaStore::for_test().await; let id_gen_manager = Arc::new(IdGeneratorManager::new(meta_store.clone()).await); let notification_manager = Arc::new(NotificationManager::new(meta_store.clone()).await); let stream_client_pool = Arc::new(StreamClientPool::default()); @@ -329,14 +373,25 @@ impl MetaSrvEnv { .await .unwrap(), ); + let system_params_controller = Some(Arc::new( + SystemParamsController::new( + sql_meta_store.clone(), + notification_manager.clone(), + risingwave_common::system_param::system_params_for_test(), + ) + .await + .unwrap(), + )); Self { id_gen_manager, meta_store, + meta_store_sql: Some(sql_meta_store), notification_manager, stream_client_pool, idle_manager, system_params_manager, + system_params_controller, cluster_id, cluster_first_launch, connector_client: None, diff --git a/src/meta/src/rpc/server.rs b/src/meta/src/rpc/server.rs index cb64adc13dae9..20ea4519c678b 100644 --- a/src/meta/src/rpc/server.rs +++ b/src/meta/src/rpc/server.rs @@ -57,6 +57,8 @@ use super::service::serving_service::ServingServiceImpl; use super::DdlServiceImpl; use crate::backup_restore::BackupManager; use crate::barrier::{BarrierScheduler, GlobalBarrierManager}; +use crate::controller::system_param::SystemParamsController; +use crate::controller::SqlMetaStore; use crate::hummock::HummockManager; use crate::manager::sink_coordination::SinkCoordinatorManager; use crate::manager::{ @@ -97,6 +99,11 @@ pub enum MetaStoreBackend { Mem, } +#[derive(Debug)] +pub struct MetaStoreSqlBackend { + pub(crate) endpoint: String, +} + #[derive(Clone)] pub struct AddressInfo { pub advertise_addr: String, @@ -123,11 +130,24 @@ pub type ElectionClientRef = Arc; pub async fn rpc_serve( address_info: AddressInfo, meta_store_backend: MetaStoreBackend, + meta_store_sql_backend: Option, max_cluster_heartbeat_interval: Duration, lease_interval_secs: u64, opts: MetaOpts, init_system_params: SystemParams, ) -> MetaResult<(JoinHandle<()>, Option>, WatchSender<()>)> { + let meta_store_sql = match meta_store_sql_backend { + Some(backend) => { + let mut options = sea_orm::ConnectOptions::new(backend.endpoint); + options + .max_connections(20) + .connect_timeout(Duration::from_secs(10)) + .idle_timeout(Duration::from_secs(30)); + let conn = sea_orm::Database::connect(options).await?; + Some(SqlMetaStore::new(conn)) + } + None => None, + }; match meta_store_backend { MetaStoreBackend::Etcd { endpoints, @@ -164,6 +184,7 @@ pub async fn rpc_serve( rpc_serve_with_store( meta_store, Some(election_client), + meta_store_sql, address_info, max_cluster_heartbeat_interval, lease_interval_secs, @@ -176,6 +197,7 @@ pub async fn rpc_serve( rpc_serve_with_store( meta_store, None, + meta_store_sql, address_info, max_cluster_heartbeat_interval, lease_interval_secs, @@ -190,6 +212,7 @@ pub async fn rpc_serve( pub fn rpc_serve_with_store( meta_store: MetaStoreRef, election_client: Option, + meta_store_sql: Option, address_info: AddressInfo, max_cluster_heartbeat_interval: Duration, lease_interval_secs: u64, @@ -272,6 +295,7 @@ pub fn rpc_serve_with_store( start_service_as_election_leader( meta_store, + meta_store_sql, address_info, max_cluster_heartbeat_interval, opts, @@ -343,6 +367,7 @@ pub async fn start_service_as_election_follower( /// Returns an error if the service initialization failed pub async fn start_service_as_election_leader( meta_store: MetaStoreRef, + meta_store_sql: Option, address_info: AddressInfo, max_cluster_heartbeat_interval: Duration, opts: MetaOpts, @@ -352,11 +377,23 @@ pub async fn start_service_as_election_leader( ) -> MetaResult<()> { tracing::info!("Defining leader services"); let prometheus_endpoint = opts.prometheus_endpoint.clone(); - let env = MetaSrvEnv::new(opts, init_system_params, meta_store.clone()).await?; + let env = MetaSrvEnv::new( + opts, + init_system_params, + meta_store.clone(), + meta_store_sql.clone(), + ) + .await?; let fragment_manager = Arc::new(FragmentManager::new(env.clone()).await.unwrap()); let system_params_manager = env.system_params_manager_ref(); - let system_params_reader = system_params_manager.get_params().await; + let mut system_params_reader = system_params_manager.get_params().await; + + // Using new reader instead if the controller is set. + let system_params_controller = env.system_params_controller_ref(); + if let Some(ctl) = &system_params_controller { + system_params_reader = ctl.get_params().await; + } let data_directory = system_params_reader.data_directory(); if !is_correct_data_directory(data_directory) { @@ -570,8 +607,11 @@ pub async fn start_service_as_election_leader( ); let health_srv = HealthServiceImpl::new(); let backup_srv = BackupServiceImpl::new(backup_manager); - let telemetry_srv = TelemetryInfoServiceImpl::new(meta_store.clone()); - let system_params_srv = SystemParamsServiceImpl::new(system_params_manager.clone()); + let telemetry_srv = TelemetryInfoServiceImpl::new(meta_store.clone(), env.sql_meta_store()); + let system_params_srv = SystemParamsServiceImpl::new( + system_params_manager.clone(), + system_params_controller.clone(), + ); let serving_srv = ServingServiceImpl::new(serving_vnode_mapping.clone(), fragment_manager.clone()); let cloud_srv = CloudServiceImpl::new(catalog_manager.clone(), aws_cli); @@ -606,9 +646,15 @@ pub async fn start_service_as_election_leader( ) .await, ); - sub_tasks.push(SystemParamsManager::start_params_notifier( - system_params_manager.clone(), - )); + if let Some(system_params_ctl) = system_params_controller { + sub_tasks.push(SystemParamsController::start_params_notifier( + system_params_ctl, + )); + } else { + sub_tasks.push(SystemParamsManager::start_params_notifier( + system_params_manager.clone(), + )); + } sub_tasks.push(HummockManager::hummock_timer_task(hummock_manager.clone())); sub_tasks.push(HummockManager::compaction_event_loop( hummock_manager, @@ -691,8 +737,10 @@ pub async fn start_service_as_election_leader( // Persist params before starting services so that invalid params that cause meta node // to crash will not be persisted. - system_params_manager.flush_params().await?; - env.cluster_id().put_at_meta_store(&meta_store).await?; + if meta_store_sql.is_none() { + system_params_manager.flush_params().await?; + env.cluster_id().put_at_meta_store(&meta_store).await?; + } tracing::info!("Assigned cluster id {:?}", *env.cluster_id()); tracing::info!("Starting meta services"); diff --git a/src/meta/src/rpc/service/system_params_service.rs b/src/meta/src/rpc/service/system_params_service.rs index 114c9aa917a68..8d557b401a2ed 100644 --- a/src/meta/src/rpc/service/system_params_service.rs +++ b/src/meta/src/rpc/service/system_params_service.rs @@ -19,16 +19,22 @@ use risingwave_pb::meta::{ }; use tonic::{Request, Response, Status}; +use crate::controller::system_param::SystemParamsControllerRef; use crate::manager::SystemParamsManagerRef; pub struct SystemParamsServiceImpl { system_params_manager: SystemParamsManagerRef, + system_params_controller: Option, } impl SystemParamsServiceImpl { - pub fn new(system_params_manager: SystemParamsManagerRef) -> Self { + pub fn new( + system_params_manager: SystemParamsManagerRef, + system_params_controller: Option, + ) -> Self { Self { system_params_manager, + system_params_controller, } } } @@ -39,8 +45,15 @@ impl SystemParamsService for SystemParamsServiceImpl { &self, _request: Request, ) -> Result, Status> { - let params = Some(self.system_params_manager.get_pb_params().await); - Ok(Response::new(GetSystemParamsResponse { params })) + let params = if let Some(ctl) = &self.system_params_controller { + ctl.get_pb_params().await + } else { + self.system_params_manager.get_pb_params().await + }; + + Ok(Response::new(GetSystemParamsResponse { + params: Some(params), + })) } async fn set_system_param( @@ -48,10 +61,14 @@ impl SystemParamsService for SystemParamsServiceImpl { request: Request, ) -> Result, Status> { let req = request.into_inner(); - let params = self - .system_params_manager - .set_param(&req.param, req.value) - .await?; + let params = if let Some(ctl) = &self.system_params_controller { + ctl.set_param(&req.param, req.value).await? + } else { + self.system_params_manager + .set_param(&req.param, req.value) + .await? + }; + Ok(Response::new(SetSystemParamResponse { params: Some(params), })) diff --git a/src/meta/src/rpc/service/telemetry_service.rs b/src/meta/src/rpc/service/telemetry_service.rs index b1a9cdec3ef34..7c413406f13e5 100644 --- a/src/meta/src/rpc/service/telemetry_service.rs +++ b/src/meta/src/rpc/service/telemetry_service.rs @@ -14,25 +14,38 @@ use risingwave_pb::meta::telemetry_info_service_server::TelemetryInfoService; use risingwave_pb::meta::{GetTelemetryInfoRequest, TelemetryInfoResponse}; +use sea_orm::EntityTrait; use tonic::{Request, Response, Status}; +use crate::controller::SqlMetaStore; use crate::model::ClusterId; +use crate::model_v2::prelude::Cluster; use crate::storage::MetaStoreRef; +use crate::MetaResult; pub struct TelemetryInfoServiceImpl { meta_store: MetaStoreRef, + sql_meta_store: Option, } impl TelemetryInfoServiceImpl { - pub fn new(meta_store: MetaStoreRef) -> Self { - Self { meta_store } + pub fn new(meta_store: MetaStoreRef, sql_meta_store: Option) -> Self { + Self { + meta_store, + sql_meta_store, + } } - async fn get_tracking_id(&self) -> Option { - ClusterId::from_meta_store(&self.meta_store) + async fn get_tracking_id(&self) -> MetaResult> { + if let Some(store) = &self.sql_meta_store { + let cluster = Cluster::find().one(&store.conn).await?; + return Ok(cluster.map(|c| c.cluster_id.to_string().into())); + } + + Ok(ClusterId::from_meta_store(&self.meta_store) .await .ok() - .flatten() + .flatten()) } } @@ -42,7 +55,7 @@ impl TelemetryInfoService for TelemetryInfoServiceImpl { &self, _request: Request, ) -> Result, Status> { - match self.get_tracking_id().await { + match self.get_tracking_id().await? { Some(tracking_id) => Ok(Response::new(TelemetryInfoResponse { tracking_id: Some(tracking_id.into()), })), From 49e5e882685eb671b830d0651045d47d08614263 Mon Sep 17 00:00:00 2001 From: August Date: Tue, 19 Sep 2023 17:34:49 +0800 Subject: [PATCH 13/30] support sqlite --- Cargo.lock | 19 +- src/meta/Cargo.toml | 1 + src/meta/src/controller/catalog.rs | 1 - src/meta/src/controller/mod.rs | 3 + src/meta/src/controller/system_param.rs | 8 +- src/meta/src/model_v2/actor.rs | 2 +- src/meta/src/model_v2/fragment.rs | 18 +- src/meta/src/model_v2/index.rs | 2 +- src/meta/src/model_v2/migration/Cargo.toml | 4 +- .../migration/src/m20230908_072257_init.rs | 868 ++++++++---------- src/meta/src/model_v2/migration/src/main.rs | 2 +- src/meta/src/model_v2/object.rs | 8 + src/meta/src/model_v2/object_dependency.rs | 2 +- src/meta/src/model_v2/sink.rs | 4 +- src/meta/src/model_v2/source.rs | 8 +- src/meta/src/model_v2/table.rs | 12 +- src/meta/src/model_v2/user_privilege.rs | 2 +- src/meta/src/model_v2/worker_property.rs | 2 +- src/meta/src/rpc/server.rs | 6 + 19 files changed, 443 insertions(+), 529 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e611a897e05d..e915e7252f159 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4645,15 +4645,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "migration" -version = "0.1.0" -dependencies = [ - "async-std", - "sea-orm-migration", - "uuid", -] - [[package]] name = "mime" version = "0.3.17" @@ -4724,6 +4715,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "model_migration" +version = "0.1.0" +dependencies = [ + "async-std", + "sea-orm-migration", + "uuid", +] + [[package]] name = "moka" version = "0.11.3" @@ -7494,6 +7494,7 @@ dependencies = [ "maplit", "memcomparable", "mime_guess", + "model_migration", "num-integer", "num-traits", "parking_lot 0.12.1", diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml index 0adf87f63c433..c7e39bdd76127 100644 --- a/src/meta/Cargo.toml +++ b/src/meta/Cargo.toml @@ -36,6 +36,7 @@ hyper = "0.14" itertools = "0.11" memcomparable = { version = "0.2" } mime_guess = "2" +model_migration = { path = "src/model_v2/migration" } num-integer = "0.1" num-traits = "0.2" parking_lot = { version = "0.12", features = ["arc_lock"] } diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index 6fe661cb017f3..64b59b773d859 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -356,7 +356,6 @@ mod tests { use super::*; #[tokio::test] - #[ignore] async fn test_create_database() { let mgr = CatalogController::new(MetaSrvEnv::for_test().await) .await diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index b6f8438994bfa..be543838f540d 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -41,8 +41,11 @@ impl SqlMetaStore { Self { conn } } + #[cfg(any(test, feature = "test"))] pub async fn for_test() -> Self { + use model_migration::{Migrator, MigratorTrait}; let conn = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + Migrator::up(&conn, None).await.unwrap(); Self { conn } } } diff --git a/src/meta/src/controller/system_param.rs b/src/meta/src/controller/system_param.rs index 0a326a72f086c..c91b0f2484366 100644 --- a/src/meta/src/controller/system_param.rs +++ b/src/meta/src/controller/system_param.rs @@ -251,20 +251,14 @@ impl SystemParamsController { #[cfg(test)] mod tests { use risingwave_common::system_param::system_params_for_test; - use sea_orm::{ConnectionTrait, Schema}; use super::*; use crate::manager::MetaSrvEnv; #[tokio::test] async fn test_system_params() { - let meta_store = SqlMetaStore::for_test().await; - let builder = meta_store.conn.get_database_backend(); - let schema = Schema::new(builder); - let stmt = schema.create_table_from_entity(SystemParameter); - meta_store.conn.execute(builder.build(&stmt)).await.unwrap(); - let env = MetaSrvEnv::for_test().await; + let meta_store = env.sql_meta_store().unwrap(); let init_params = system_params_for_test(); // init system parameter controller as first launch. diff --git a/src/meta/src/model_v2/actor.rs b/src/meta/src/model_v2/actor.rs index 2ed597197a5c7..92c18111f6a77 100644 --- a/src/meta/src/model_v2/actor.rs +++ b/src/meta/src/model_v2/actor.rs @@ -11,7 +11,7 @@ pub struct Model { pub status: Option, pub splits: Option, pub parallel_unit_id: i32, - pub upstream_actor_ids: Option>, + pub upstream_actor_ids: Option, pub dispatchers: Option, pub vnode_bitmap: Option, } diff --git a/src/meta/src/model_v2/fragment.rs b/src/meta/src/model_v2/fragment.rs index 8054c59bf8063..0c116cf8929c5 100644 --- a/src/meta/src/model_v2/fragment.rs +++ b/src/meta/src/model_v2/fragment.rs @@ -12,11 +12,11 @@ pub struct Model { pub distribution_type: String, pub stream_node: Json, pub vnode_mapping: Option, - pub state_table_ids: Option>, - pub upstream_fragment_id: Option>, + pub state_table_ids: Option, + pub upstream_fragment_id: Option, pub dispatcher_type: Option, - pub dist_key_indices: Option>, - pub output_indices: Option>, + pub dist_key_indices: Option, + pub output_indices: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] @@ -24,13 +24,13 @@ pub enum Relation { #[sea_orm(has_many = "super::actor::Entity")] Actor, #[sea_orm( - belongs_to = "super::table::Entity", + belongs_to = "super::object::Entity", from = "Column::TableId", - to = "super::table::Column::TableId", + to = "super::object::Column::Oid", on_update = "NoAction", on_delete = "Cascade" )] - Table, + Object, } impl Related for Entity { @@ -39,9 +39,9 @@ impl Related for Entity { } } -impl Related for Entity { +impl Related for Entity { fn to() -> RelationDef { - Relation::Table.def() + Relation::Object.def() } } diff --git a/src/meta/src/model_v2/index.rs b/src/meta/src/model_v2/index.rs index 69810e1ea4214..6211f4bf3c16b 100644 --- a/src/meta/src/model_v2/index.rs +++ b/src/meta/src/model_v2/index.rs @@ -13,7 +13,7 @@ pub struct Model { pub index_table_id: i32, pub primary_table_id: i32, pub index_items: Option, - pub original_columns: Option>, + pub original_columns: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/migration/Cargo.toml b/src/meta/src/model_v2/migration/Cargo.toml index 8a8bc23dd9759..30b61d5504926 100644 --- a/src/meta/src/model_v2/migration/Cargo.toml +++ b/src/meta/src/model_v2/migration/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "migration" +name = "model_migration" version = "0.1.0" edition = "2021" publish = false [lib] -name = "migration" +name = "model_migration" path = "src/lib.rs" [dependencies] diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs index 0160ca51604cb..b675f335392cb 100644 --- a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -84,7 +84,7 @@ impl MigrationTrait for Migration { ) .col( ColumnDef::new(WorkerProperty::ParallelUnitIds) - .array(ColumnType::Integer) + .json() .not_null(), ) .col( @@ -102,6 +102,14 @@ impl MigrationTrait for Migration { .boolean() .not_null(), ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_worker_property_worker_id") + .from(WorkerProperty::Table, WorkerProperty::WorkerId) + .to(Worker::Table, Worker::WorkerId) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) .to_owned(), ) .await?; @@ -125,6 +133,40 @@ impl MigrationTrait for Migration { .to_owned(), ) .await?; + manager + .create_table( + MigrationTable::create() + .table(Object::Table) + .col( + ColumnDef::new(Object::Oid) + .integer() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(Object::ObjType).string().not_null()) + .col(ColumnDef::new(Object::OwnerId).integer().not_null()) + .col( + ColumnDef::new(Object::InitializedAt) + .timestamp() + .default(Expr::current_timestamp()) + .not_null(), + ) + .col( + ColumnDef::new(Object::CreatedAt) + .timestamp() + .default(Expr::current_timestamp()) + .not_null(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_object_owner_id") + .from(Object::Table, Object::OwnerId) + .to(User::Table, User::UserId) + .to_owned(), + ) + .to_owned(), + ) + .await?; manager .create_table( MigrationTable::create() @@ -142,16 +184,69 @@ impl MigrationTrait for Migration { .integer() .not_null(), ) + .col(ColumnDef::new(UserPrivilege::Actions).string().not_null()) .col( - ColumnDef::new(UserPrivilege::Actions) - .array(ColumnType::String(None)) + ColumnDef::new(UserPrivilege::WithGrantOption) + .boolean() .not_null(), ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_user_privilege_user_id") + .from(UserPrivilege::Table, UserPrivilege::UserId) + .to(User::Table, User::UserId) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_user_privilege_granted_by") + .from(UserPrivilege::Table, UserPrivilege::GrantedBy) + .to(User::Table, User::UserId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_user_privilege_oid") + .from(UserPrivilege::Table, UserPrivilege::Oid) + .to(Object::Table, Object::Oid) + .to_owned(), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(ObjectDependency::Table) .col( - ColumnDef::new(UserPrivilege::WithGrantOption) - .boolean() + ColumnDef::new(ObjectDependency::Id) + .integer() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(ObjectDependency::Oid).integer().not_null()) + .col( + ColumnDef::new(ObjectDependency::UsedBy) + .integer() .not_null(), ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_object_dependency_oid") + .from(ObjectDependency::Table, ObjectDependency::Oid) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_object_dependency_used_by") + .from(ObjectDependency::Table, ObjectDependency::UsedBy) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) .to_owned(), ) .await?; @@ -166,6 +261,14 @@ impl MigrationTrait for Migration { .unique_key() .not_null(), ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_database_object_id") + .from(Database::Table, Database::DatabaseId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) .to_owned(), ) .await?; @@ -176,6 +279,21 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Schema::SchemaId).integer().primary_key()) .col(ColumnDef::new(Schema::Name).string().not_null()) .col(ColumnDef::new(Schema::DatabaseId).integer().not_null()) + .foreign_key( + &mut ForeignKey::create() + .name("FK_schema_database_id") + .from(Schema::Table, Schema::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_schema_object_id") + .from(Schema::Table, Schema::SchemaId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) .to_owned(), ) .await?; @@ -202,11 +320,19 @@ impl MigrationTrait for Migration { ) .col(ColumnDef::new(Fragment::StreamNode).json().not_null()) .col(ColumnDef::new(Fragment::VnodeMapping).json()) - .col(ColumnDef::new(Fragment::StateTableIds).array(ColumnType::Integer)) - .col(ColumnDef::new(Fragment::UpstreamFragmentId).array(ColumnType::Integer)) + .col(ColumnDef::new(Fragment::StateTableIds).json()) + .col(ColumnDef::new(Fragment::UpstreamFragmentId).json()) .col(ColumnDef::new(Fragment::DispatcherType).string()) - .col(ColumnDef::new(Fragment::DistKeyIndices).array(ColumnType::Integer)) - .col(ColumnDef::new(Fragment::OutputIndices).array(ColumnType::Integer)) + .col(ColumnDef::new(Fragment::DistKeyIndices).json()) + .col(ColumnDef::new(Fragment::OutputIndices).json()) + .foreign_key( + &mut ForeignKey::create() + .name("FK_fragment_table_id") + .from(Fragment::Table, Fragment::TableId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) .to_owned(), ) .await?; @@ -224,39 +350,55 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Actor::Status).string()) .col(ColumnDef::new(Actor::Splits).json()) .col(ColumnDef::new(Actor::ParallelUnitId).integer().not_null()) - .col(ColumnDef::new(Actor::UpstreamActorIds).array(ColumnType::Integer)) + .col(ColumnDef::new(Actor::UpstreamActorIds).json()) .col(ColumnDef::new(Actor::Dispatchers).json()) .col(ColumnDef::new(Actor::VnodeBitmap).string()) + .foreign_key( + &mut ForeignKey::create() + .name("FK_actor_fragment_id") + .from(Actor::Table, Actor::FragmentId) + .to(Fragment::Table, Fragment::FragmentId) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) .to_owned(), ) .await?; manager .create_table( MigrationTable::create() - .table(Table::Table) - .col(ColumnDef::new(Table::TableId).integer().primary_key()) - .col(ColumnDef::new(Table::Name).string().not_null()) - .col(ColumnDef::new(Table::SchemaId).integer().not_null()) - .col(ColumnDef::new(Table::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Table::OptionalAssociatedSourceId).integer()) - .col(ColumnDef::new(Table::TableType).string()) - .col(ColumnDef::new(Table::Columns).json()) - .col(ColumnDef::new(Table::Pk).json()) - .col(ColumnDef::new(Table::DistributionKey).array(ColumnType::Integer)) - .col(ColumnDef::new(Table::AppendOnly).boolean()) - .col(ColumnDef::new(Table::Properties).json()) - .col(ColumnDef::new(Table::FragmentId).integer()) - .col(ColumnDef::new(Table::VnodeColIndex).integer()) - .col(ColumnDef::new(Table::ValueIndices).array(ColumnType::Integer)) - .col(ColumnDef::new(Table::Definition).string()) - .col(ColumnDef::new(Table::HandlePkConflictBehavior).integer()) - .col(ColumnDef::new(Table::ReadPrefixLenHint).integer()) - .col(ColumnDef::new(Table::WatermarkIndices).array(ColumnType::Integer)) - .col(ColumnDef::new(Table::DistKeyInPk).array(ColumnType::Integer)) - .col(ColumnDef::new(Table::DmlFragmentId).integer()) - .col(ColumnDef::new(Table::Cardinality).array(ColumnType::Integer)) - .col(ColumnDef::new(Table::CleanedByWatermark).boolean()) - .col(ColumnDef::new(Table::Version).array(ColumnType::Integer)) + .table(Connection::Table) + .col( + ColumnDef::new(Connection::ConnectionId) + .integer() + .primary_key(), + ) + .col(ColumnDef::new(Connection::Name).string().not_null()) + .col(ColumnDef::new(Connection::SchemaId).integer().not_null()) + .col(ColumnDef::new(Connection::DatabaseId).integer().not_null()) + .col(ColumnDef::new(Connection::Info).json()) + .foreign_key( + &mut ForeignKey::create() + .name("FK_connection_database_id") + .from(Connection::Table, Connection::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_connection_schema_id") + .from(Connection::Table, Connection::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_connection_object_id") + .from(Connection::Table, Connection::ConnectionId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) .to_owned(), ) .await?; @@ -277,6 +419,108 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Source::WatermarkDescs).json()) .col(ColumnDef::new(Source::OptionalAssociatedTableId).integer()) .col(ColumnDef::new(Source::ConnectionId).integer()) + .foreign_key( + &mut ForeignKey::create() + .name("FK_source_database_id") + .from(Source::Table, Source::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_source_schema_id") + .from(Source::Table, Source::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_source_object_id") + .from(Source::Table, Source::SourceId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_source_connection_id") + .from(Source::Table, Source::ConnectionId) + .to(Connection::Table, Connection::ConnectionId) + .to_owned(), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(Table::Table) + .col(ColumnDef::new(Table::TableId).integer().primary_key()) + .col(ColumnDef::new(Table::Name).string().not_null()) + .col(ColumnDef::new(Table::SchemaId).integer().not_null()) + .col(ColumnDef::new(Table::DatabaseId).integer().not_null()) + .col(ColumnDef::new(Table::OptionalAssociatedSourceId).integer()) + .col(ColumnDef::new(Table::TableType).string()) + .col(ColumnDef::new(Table::Columns).json()) + .col(ColumnDef::new(Table::Pk).json()) + .col(ColumnDef::new(Table::DistributionKey).json()) + .col(ColumnDef::new(Table::AppendOnly).boolean()) + .col(ColumnDef::new(Table::Properties).json()) + .col(ColumnDef::new(Table::FragmentId).integer()) + .col(ColumnDef::new(Table::VnodeColIndex).integer()) + .col(ColumnDef::new(Table::ValueIndices).json()) + .col(ColumnDef::new(Table::Definition).string()) + .col(ColumnDef::new(Table::HandlePkConflictBehavior).integer()) + .col(ColumnDef::new(Table::ReadPrefixLenHint).integer()) + .col(ColumnDef::new(Table::WatermarkIndices).json()) + .col(ColumnDef::new(Table::DistKeyInPk).json()) + .col(ColumnDef::new(Table::DmlFragmentId).integer()) + .col(ColumnDef::new(Table::Cardinality).json()) + .col(ColumnDef::new(Table::CleanedByWatermark).boolean()) + .col(ColumnDef::new(Table::Version).json()) + .foreign_key( + &mut ForeignKey::create() + .name("FK_table_database_id") + .from(Table::Table, Table::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_table_schema_id") + .from(Table::Table, Table::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_view_object_id") + .from(Table::Table, Table::TableId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_table_fragment_id") + .from(Table::Table, Table::FragmentId) + .to(Fragment::Table, Fragment::FragmentId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_table_dml_fragment_id") + .from(Table::Table, Table::DmlFragmentId) + .to(Fragment::Table, Fragment::FragmentId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_table_optional_associated_source_id") + .from(Table::Table, Table::OptionalAssociatedSourceId) + .to(Source::Table, Source::SourceId) + .to_owned(), + ) .to_owned(), ) .await?; @@ -290,28 +534,41 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Sink::DatabaseId).integer().not_null()) .col(ColumnDef::new(Sink::Columns).json()) .col(ColumnDef::new(Sink::PkColumnIds).json()) - .col(ColumnDef::new(Sink::DistributionKey).array(ColumnType::Integer)) - .col(ColumnDef::new(Sink::DownstreamPk).array(ColumnType::Integer)) + .col(ColumnDef::new(Sink::DistributionKey).json()) + .col(ColumnDef::new(Sink::DownstreamPk).json()) .col(ColumnDef::new(Sink::SinkType).string()) .col(ColumnDef::new(Sink::Properties).json()) .col(ColumnDef::new(Sink::Definition).string()) .col(ColumnDef::new(Sink::ConnectionId).integer()) - .to_owned(), - ) - .await?; - manager - .create_table( - MigrationTable::create() - .table(Connection::Table) - .col( - ColumnDef::new(Connection::ConnectionId) - .integer() - .primary_key(), + .foreign_key( + &mut ForeignKey::create() + .name("FK_sink_database_id") + .from(Sink::Table, Sink::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_sink_schema_id") + .from(Sink::Table, Sink::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_sink_object_id") + .from(Sink::Table, Sink::SinkId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_sink_connection_id") + .from(Sink::Table, Sink::ConnectionId) + .to(Connection::Table, Connection::ConnectionId) + .to_owned(), ) - .col(ColumnDef::new(Connection::Name).string().not_null()) - .col(ColumnDef::new(Connection::SchemaId).integer().not_null()) - .col(ColumnDef::new(Connection::DatabaseId).integer().not_null()) - .col(ColumnDef::new(Connection::Info).json()) .to_owned(), ) .await?; @@ -326,6 +583,28 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(View::Properties).json()) .col(ColumnDef::new(View::Sql).string()) .col(ColumnDef::new(View::Columns).json()) + .foreign_key( + &mut ForeignKey::create() + .name("FK_view_database_id") + .from(View::Table, View::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_view_schema_id") + .from(View::Table, View::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_view_object_id") + .from(View::Table, View::ViewId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) .to_owned(), ) .await?; @@ -340,7 +619,43 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Index::IndexTableId).integer().not_null()) .col(ColumnDef::new(Index::PrimaryTableId).integer().not_null()) .col(ColumnDef::new(Index::IndexItems).json()) - .col(ColumnDef::new(Index::OriginalColumns).array(ColumnType::Integer)) + .col(ColumnDef::new(Index::OriginalColumns).json()) + .foreign_key( + &mut ForeignKey::create() + .name("FK_index_database_id") + .from(Index::Table, Index::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_index_schema_id") + .from(Index::Table, Index::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_index_object_id") + .from(Index::Table, Index::IndexId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_index_index_table_id") + .from(Index::Table, Index::IndexTableId) + .to(Table::Table, Table::TableId) + .to_owned(), + ) + .foreign_key( + &mut ForeignKey::create() + .name("FK_index_primary_table_id") + .from(Index::Table, Index::PrimaryTableId) + .to(Table::Table, Table::TableId) + .to_owned(), + ) .to_owned(), ) .await?; @@ -358,51 +673,27 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Function::Link).string()) .col(ColumnDef::new(Function::Identifier).string()) .col(ColumnDef::new(Function::Kind).json()) - .to_owned(), - ) - .await?; - manager - .create_table( - MigrationTable::create() - .table(Object::Table) - .col( - ColumnDef::new(Object::Oid) - .integer() - .auto_increment() - .primary_key(), - ) - .col(ColumnDef::new(Object::ObjType).string().not_null()) - .col(ColumnDef::new(Object::OwnerId).integer().not_null()) - .col( - ColumnDef::new(Object::InitializedAt) - .timestamp() - .default(Expr::current_timestamp()) - .not_null(), + .foreign_key( + &mut ForeignKey::create() + .name("FK_function_database_id") + .from(Function::Table, Function::DatabaseId) + .to(Database::Table, Database::DatabaseId) + .to_owned(), ) - .col( - ColumnDef::new(Object::CreatedAt) - .timestamp() - .default(Expr::current_timestamp()) - .not_null(), - ) - .to_owned(), - ) - .await?; - manager - .create_table( - MigrationTable::create() - .table(ObjectDependency::Table) - .col( - ColumnDef::new(ObjectDependency::Id) - .integer() - .auto_increment() - .primary_key(), + .foreign_key( + &mut ForeignKey::create() + .name("FK_function_schema_id") + .from(Function::Table, Function::SchemaId) + .to(Schema::Table, Schema::SchemaId) + .to_owned(), ) - .col(ColumnDef::new(ObjectDependency::Oid).integer().not_null()) - .col( - ColumnDef::new(ObjectDependency::UsedBy) - .integer() - .not_null(), + .foreign_key( + &mut ForeignKey::create() + .name("FK_function_object_id") + .from(Function::Table, Function::FunctionId) + .to(Object::Table, Object::Oid) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), ) .to_owned(), ) @@ -433,6 +724,7 @@ impl MigrationTrait for Migration { .create_index( MigrationIndex::create() .table(Worker::Table) + .name("idx_worker_host_port") .unique() .col(Worker::Host) .col(Worker::Port) @@ -443,6 +735,7 @@ impl MigrationTrait for Migration { .create_index( MigrationIndex::create() .table(Schema::Table) + .name("idx_schema_database_id_name") .unique() .col(Schema::DatabaseId) .col(Schema::Name) @@ -450,391 +743,6 @@ impl MigrationTrait for Migration { ) .await?; - // 4. create foreign keys. - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_worker_property_worker_id") - .from(WorkerProperty::Table, WorkerProperty::WorkerId) - .to(Worker::Table, Worker::WorkerId) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_user_privilege_user_id") - .from(UserPrivilege::Table, UserPrivilege::UserId) - .to(User::Table, User::UserId) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_user_privilege_granted_by") - .from(UserPrivilege::Table, UserPrivilege::GrantedBy) - .to(User::Table, User::UserId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_user_privilege_oid") - .from(UserPrivilege::Table, UserPrivilege::Oid) - .to(Object::Table, Object::Oid) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_database_object_id") - .from(Database::Table, Database::DatabaseId) - .to(Object::Table, Object::Oid) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_schema_database_id") - .from(Schema::Table, Schema::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_schema_object_id") - .from(Schema::Table, Schema::SchemaId) - .to(Object::Table, Object::Oid) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_fragment_table_id") - .from(Fragment::Table, Fragment::TableId) - .to(Table::Table, Table::TableId) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_actor_fragment_id") - .from(Actor::Table, Actor::FragmentId) - .to(Fragment::Table, Fragment::FragmentId) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_table_schema_id") - .from(Table::Table, Table::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_table_database_id") - .from(Table::Table, Table::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_table_object_id") - .from(Table::Table, Table::TableId) - .to(Object::Table, Object::Oid) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_table_fragment_id") - .from(Table::Table, Table::FragmentId) - .to(Fragment::Table, Fragment::FragmentId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_table_dml_fragment_id") - .from(Table::Table, Table::DmlFragmentId) - .to(Fragment::Table, Fragment::FragmentId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_table_optional_associated_source_id") - .from(Table::Table, Table::OptionalAssociatedSourceId) - .to(Source::Table, Source::SourceId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_source_schema_id") - .from(Source::Table, Source::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_source_database_id") - .from(Source::Table, Source::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_source_object_id") - .from(Source::Table, Source::SourceId) - .to(Object::Table, Object::Oid) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_source_connection_id") - .from(Source::Table, Source::ConnectionId) - .to(Connection::Table, Connection::ConnectionId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_source_optional_associated_table_id") - .from(Source::Table, Source::OptionalAssociatedTableId) - .to(Table::Table, Table::TableId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_sink_schema_id") - .from(Sink::Table, Sink::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_sink_database_id") - .from(Sink::Table, Sink::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_sink_object_id") - .from(Sink::Table, Sink::SinkId) - .to(Object::Table, Object::Oid) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_sink_connection_id") - .from(Sink::Table, Sink::ConnectionId) - .to(Connection::Table, Connection::ConnectionId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_connection_schema_id") - .from(Connection::Table, Connection::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_connection_database_id") - .from(Connection::Table, Connection::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_connection_object_id") - .from(Connection::Table, Connection::ConnectionId) - .to(Object::Table, Object::Oid) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_view_schema_id") - .from(View::Table, View::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_view_database_id") - .from(View::Table, View::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_view_object_id") - .from(View::Table, View::ViewId) - .to(Object::Table, Object::Oid) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_index_schema_id") - .from(Index::Table, Index::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_index_database_id") - .from(Index::Table, Index::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_index_object_id") - .from(Index::Table, Index::IndexId) - .to(Object::Table, Object::Oid) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_index_index_table_id") - .from(Index::Table, Index::IndexTableId) - .to(Table::Table, Table::TableId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_index_primary_table_id") - .from(Index::Table, Index::PrimaryTableId) - .to(Table::Table, Table::TableId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_function_schema_id") - .from(Function::Table, Function::SchemaId) - .to(Schema::Table, Schema::SchemaId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_function_database_id") - .from(Function::Table, Function::DatabaseId) - .to(Database::Table, Database::DatabaseId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_function_object_id") - .from(Function::Table, Function::FunctionId) - .to(Object::Table, Object::Oid) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_object_owner_id") - .from(Object::Table, Object::OwnerId) - .to(User::Table, User::UserId) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_object_dependency_oid") - .from(ObjectDependency::Table, ObjectDependency::Oid) - .to(Object::Table, Object::Oid) - .to_owned(), - ) - .await?; - manager - .create_foreign_key( - ForeignKey::create() - .name("FK_object_dependency_used_by") - .from(ObjectDependency::Table, ObjectDependency::UsedBy) - .to(Object::Table, Object::Oid) - .on_delete(ForeignKeyAction::Cascade) - .to_owned(), - ) - .await?; - // 4. initialize data. let insert_cluster_id = Query::insert() .into_table(Cluster::Table) diff --git a/src/meta/src/model_v2/migration/src/main.rs b/src/meta/src/model_v2/migration/src/main.rs index c6b6e48dbc06b..9354e45ecd198 100644 --- a/src/meta/src/model_v2/migration/src/main.rs +++ b/src/meta/src/model_v2/migration/src/main.rs @@ -2,5 +2,5 @@ use sea_orm_migration::prelude::*; #[async_std::main] async fn main() { - cli::run_cli(migration::Migrator).await; + cli::run_cli(model_migration::Migrator).await; } diff --git a/src/meta/src/model_v2/object.rs b/src/meta/src/model_v2/object.rs index be72e45094fd6..b6a22dcc54086 100644 --- a/src/meta/src/model_v2/object.rs +++ b/src/meta/src/model_v2/object.rs @@ -19,6 +19,8 @@ pub enum Relation { Connection, #[sea_orm(has_many = "super::database::Entity")] Database, + #[sea_orm(has_many = "super::fragment::Entity")] + Fragment, #[sea_orm(has_many = "super::function::Entity")] Function, #[sea_orm(has_many = "super::index::Entity")] @@ -57,6 +59,12 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::Fragment.def() + } +} + impl Related for Entity { fn to() -> RelationDef { Relation::Function.def() diff --git a/src/meta/src/model_v2/object_dependency.rs b/src/meta/src/model_v2/object_dependency.rs index 36abc298fec57..c24e389bae106 100644 --- a/src/meta/src/model_v2/object_dependency.rs +++ b/src/meta/src/model_v2/object_dependency.rs @@ -18,7 +18,7 @@ pub enum Relation { from = "Column::Oid", to = "super::object::Column::Oid", on_update = "NoAction", - on_delete = "NoAction" + on_delete = "Cascade" )] Object2, #[sea_orm( diff --git a/src/meta/src/model_v2/sink.rs b/src/meta/src/model_v2/sink.rs index d67cb632d6c0f..95229595b0c04 100644 --- a/src/meta/src/model_v2/sink.rs +++ b/src/meta/src/model_v2/sink.rs @@ -12,8 +12,8 @@ pub struct Model { pub database_id: i32, pub columns: Option, pub pk_column_ids: Option, - pub distribution_key: Option>, - pub downstream_pk: Option>, + pub distribution_key: Option, + pub downstream_pk: Option, pub sink_type: Option, pub properties: Option, pub definition: Option, diff --git a/src/meta/src/model_v2/source.rs b/src/meta/src/model_v2/source.rs index da96e42b9c50c..b65e9051a206f 100644 --- a/src/meta/src/model_v2/source.rs +++ b/src/meta/src/model_v2/source.rs @@ -55,13 +55,7 @@ pub enum Relation { on_delete = "NoAction" )] Schema, - #[sea_orm( - belongs_to = "super::table::Entity", - from = "Column::OptionalAssociatedTableId", - to = "super::table::Column::TableId", - on_update = "NoAction", - on_delete = "NoAction" - )] + #[sea_orm(has_many = "super::table::Entity")] Table, } diff --git a/src/meta/src/model_v2/table.rs b/src/meta/src/model_v2/table.rs index d36df4b47a813..8a9ad68bd6786 100644 --- a/src/meta/src/model_v2/table.rs +++ b/src/meta/src/model_v2/table.rs @@ -14,21 +14,21 @@ pub struct Model { pub table_type: Option, pub columns: Option, pub pk: Option, - pub distribution_key: Option>, + pub distribution_key: Option, pub append_only: Option, pub properties: Option, pub fragment_id: Option, pub vnode_col_index: Option, - pub value_indices: Option>, + pub value_indices: Option, pub definition: Option, pub handle_pk_conflict_behavior: Option, pub read_prefix_len_hint: Option, - pub watermark_indices: Option>, - pub dist_key_in_pk: Option>, + pub watermark_indices: Option, + pub dist_key_in_pk: Option, pub dml_fragment_id: Option, - pub cardinality: Option>, + pub cardinality: Option, pub cleaned_by_watermark: Option, - pub version: Option>, + pub version: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/user_privilege.rs b/src/meta/src/model_v2/user_privilege.rs index e20b4dbe67a5b..35e8644bc5383 100644 --- a/src/meta/src/model_v2/user_privilege.rs +++ b/src/meta/src/model_v2/user_privilege.rs @@ -10,7 +10,7 @@ pub struct Model { pub user_id: i32, pub oid: i32, pub granted_by: i32, - pub actions: Vec, + pub actions: String, pub with_grant_option: bool, } diff --git a/src/meta/src/model_v2/worker_property.rs b/src/meta/src/model_v2/worker_property.rs index 3f38ddd40670e..8d15af22eeee8 100644 --- a/src/meta/src/model_v2/worker_property.rs +++ b/src/meta/src/model_v2/worker_property.rs @@ -7,7 +7,7 @@ use sea_orm::entity::prelude::*; pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub worker_id: i32, - pub parallel_unit_ids: Vec, + pub parallel_unit_ids: Json, pub is_streaming: bool, pub is_serving: bool, pub is_unschedulable: bool, diff --git a/src/meta/src/rpc/server.rs b/src/meta/src/rpc/server.rs index 2206599e73eb7..9c511dbb3bef3 100644 --- a/src/meta/src/rpc/server.rs +++ b/src/meta/src/rpc/server.rs @@ -20,6 +20,7 @@ use either::Either; use etcd_client::ConnectOptions; use futures::future::join_all; use itertools::Itertools; +use model_migration::{Migrator, MigratorTrait}; use regex::Regex; use risingwave_common::monitor::connection::{RouterExt, TcpConfig}; use risingwave_common::telemetry::manager::TelemetryManager; @@ -144,6 +145,11 @@ pub async fn rpc_serve( .connect_timeout(Duration::from_secs(10)) .idle_timeout(Duration::from_secs(30)); let conn = sea_orm::Database::connect(options).await?; + // Try to upgrade if any new model changes are added. + Migrator::up(&conn, None) + .await + .expect("Failed to upgrade models in meta store"); + Some(SqlMetaStore::new(conn)) } None => None, From 9f0491b6e368d5c713b21cd38217d27f2a82c92f Mon Sep 17 00:00:00 2001 From: August Date: Tue, 19 Sep 2023 17:47:00 +0800 Subject: [PATCH 14/30] add election tables --- .../migration/src/m20230908_072257_init.rs | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs index b675f335392cb..ae0106c91bfef 100644 --- a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -34,6 +34,8 @@ impl MigrationTrait for Migration { .has_table(SystemParameter::Table.to_string()) .await? ); + assert!(!manager.has_table(ElectionLeader::Table.to_string()).await?); + assert!(!manager.has_table(ElectionMember::Table.to_string()).await?); // 2. create tables. manager @@ -718,6 +720,39 @@ impl MigrationTrait for Migration { .to_owned(), ) .await?; + manager + .create_table( + MigrationTable::create() + .table(ElectionLeader::Table) + .col( + ColumnDef::new(ElectionLeader::Service) + .string() + .primary_key() + .not_null(), + ) + .col(ColumnDef::new(ElectionLeader::Id).string().not_null()) + .col( + ColumnDef::new(ElectionLeader::LastHeartbeat) + .timestamp() + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + MigrationTable::create() + .table(ElectionMember::Table) + .col(ColumnDef::new(ElectionMember::Service).string().not_null()) + .col(ColumnDef::new(ElectionMember::Id).string().not_null()) + .col( + ColumnDef::new(ElectionMember::LastHeartbeat) + .timestamp() + .not_null(), + ) + .to_owned(), + ) + .await?; // 3. create indexes. manager @@ -742,6 +777,17 @@ impl MigrationTrait for Migration { .to_owned(), ) .await?; + manager + .create_index( + MigrationIndex::create() + .table(ElectionMember::Table) + .name("election_member_pk") + .unique() + .col(ElectionMember::Service) + .col(ElectionMember::Id) + .to_owned(), + ) + .await?; // 4. initialize data. let insert_cluster_id = Query::insert() @@ -847,7 +893,9 @@ impl MigrationTrait for Migration { Function, Object, ObjectDependency, - SystemParameter + SystemParameter, + ElectionLeader, + ElectionMember ); Ok(()) } @@ -1088,3 +1136,19 @@ enum SystemParameter { IsMutable, Description, } + +#[derive(DeriveIden)] +enum ElectionLeader { + Table, + Service, + Id, + LastHeartbeat, +} + +#[derive(DeriveIden)] +enum ElectionMember { + Table, + Service, + Id, + LastHeartbeat, +} From d4baff9eee4fe88d5932e1a47412a79fc8d632fe Mon Sep 17 00:00:00 2001 From: August Date: Tue, 19 Sep 2023 19:00:43 +0800 Subject: [PATCH 15/30] add create schema --- src/meta/src/controller/catalog.rs | 40 ++++++++++++++++++------------ src/meta/src/controller/mod.rs | 10 ++++++++ 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index 64b59b773d859..f28b687ccb07a 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -16,7 +16,7 @@ use std::iter; use itertools::Itertools; use risingwave_common::catalog::{DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS}; -use risingwave_pb::catalog::PbDatabase; +use risingwave_pb::catalog::{PbDatabase, PbSchema}; use risingwave_pb::meta::subscribe_response::{ Info as NotificationInfo, Operation as NotificationOperation, }; @@ -84,21 +84,8 @@ impl CatalogController { } impl CatalogController { - pub async fn snapshot(&self) -> MetaResult> { - let inner = self.inner.read().await; - let dbs = Database::find() - .find_also_related(Object) - .all(&inner.db) - .await?; - let _tables = Table::find() - .find_also_related(Object) - .all(&inner.db) - .await?; - - Ok(dbs - .into_iter() - .map(|(db, obj)| ObjectModel(db, obj.unwrap()).into()) - .collect()) + pub async fn snapshot(&self) -> MetaResult<()> { + todo!("snapshot") } async fn create_object( @@ -347,6 +334,27 @@ impl CatalogController { version, )) } + + pub async fn create_schema(&self, schema: PbSchema) -> MetaResult { + let inner = self.inner.write().await; + let txn = inner.db.begin().await?; + let owner_id = schema.owner; + + // todo: whether to check existence of database and user, or let the database do it? + let schema_obj = Self::create_object(&txn, ObjectType::Schema, owner_id).await?; + let mut schema: schema::ActiveModel = schema.into(); + schema.schema_id = ActiveValue::Set(schema_obj.oid); + let schema = schema.insert(&txn).await?; + txn.commit().await?; + + let version = self + .notify_frontend( + NotificationOperation::Add, + NotificationInfo::Schema(ObjectModel(schema, schema_obj).into()), + ) + .await; + Ok(version) + } } #[cfg(test)] diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index be543838f540d..d9d8e1c0439d3 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -86,6 +86,16 @@ impl From for database::ActiveModel { } } +impl From for schema::ActiveModel { + fn from(schema: PbSchema) -> Self { + Self { + schema_id: ActiveValue::Set(schema.id as _), + name: ActiveValue::Set(schema.name), + database_id: ActiveValue::Set(schema.database_id as _), + } + } +} + impl From> for PbSchema { fn from(value: ObjectModel) -> Self { Self { From ec5cc08dabe1007fbe819dfc0cc98422b8e0852d Mon Sep 17 00:00:00 2001 From: August Date: Wed, 20 Sep 2023 14:16:40 +0800 Subject: [PATCH 16/30] update some fields --- src/meta/src/controller/catalog.rs | 22 ++++++++-------- .../migration/src/m20230908_072257_init.rs | 26 +++++++++---------- src/meta/src/model_v2/mod.rs | 2 ++ src/meta/src/model_v2/prelude.rs | 2 ++ src/meta/src/model_v2/sink.rs | 6 +++-- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index f28b687ccb07a..ca20a44a1018d 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -161,13 +161,13 @@ impl CatalogController { let base_query = SelectStatement::new() .column(object_dependency::Column::UsedBy) - .from(object_dependency::Entity) + .from(ObjectDependency) .and_where(object_dependency::Column::Oid.eq(obj_id)) .to_owned(); let cte_referencing = Query::select() - .column((object_dependency::Entity, object_dependency::Column::UsedBy)) - .from(object_dependency::Entity) + .column((ObjectDependency, object_dependency::Column::UsedBy)) + .from(ObjectDependency) .join( JoinType::InnerJoin, cte_alias.clone(), @@ -258,42 +258,42 @@ impl CatalogController { let mut query_schema = Query::select() .column(schema::Column::SchemaId) - .from(schema::Entity) + .from(Schema) .and_where(schema::Column::DatabaseId.eq(database_id as i32)) .to_owned(); let query_table = Query::select() .column(table::Column::TableId) - .from(table::Entity) + .from(Table) .and_where(table::Column::DatabaseId.eq(database_id as i32)) .to_owned(); let query_source = Query::select() .column(source::Column::SourceId) - .from(source::Entity) + .from(Source) .and_where(source::Column::DatabaseId.eq(database_id as i32)) .to_owned(); let query_sink = Query::select() .column(sink::Column::SinkId) - .from(sink::Entity) + .from(Sink) .and_where(sink::Column::DatabaseId.eq(database_id as i32)) .to_owned(); let query_index = Query::select() .column(index::Column::IndexId) - .from(index::Entity) + .from(Index) .and_where(index::Column::DatabaseId.eq(database_id as i32)) .to_owned(); let query_view = Query::select() .column(view::Column::ViewId) - .from(view::Entity) + .from(View) .and_where(view::Column::DatabaseId.eq(database_id as i32)) .to_owned(); let query_function = Query::select() .column(function::Column::FunctionId) - .from(function::Entity) + .from(Function) .and_where(function::Column::DatabaseId.eq(database_id as i32)) .to_owned(); let query_connection = Query::select() .column(connection::Column::ConnectionId) - .from(connection::Entity) + .from(Connection) .and_where(connection::Column::DatabaseId.eq(database_id as i32)) .to_owned(); let query_all = query_schema diff --git a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs index ae0106c91bfef..e5a07e8c87574 100644 --- a/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs +++ b/src/meta/src/model_v2/migration/src/m20230908_072257_init.rs @@ -538,10 +538,12 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Sink::PkColumnIds).json()) .col(ColumnDef::new(Sink::DistributionKey).json()) .col(ColumnDef::new(Sink::DownstreamPk).json()) - .col(ColumnDef::new(Sink::SinkType).string()) + .col(ColumnDef::new(Sink::SinkType).string().not_null()) .col(ColumnDef::new(Sink::Properties).json()) - .col(ColumnDef::new(Sink::Definition).string()) + .col(ColumnDef::new(Sink::Definition).string().not_null()) .col(ColumnDef::new(Sink::ConnectionId).integer()) + .col(ColumnDef::new(Sink::DbName).string().not_null()) + .col(ColumnDef::new(Sink::SinkFromName).string().not_null()) .foreign_key( &mut ForeignKey::create() .name("FK_sink_database_id") @@ -744,7 +746,12 @@ impl MigrationTrait for Migration { MigrationTable::create() .table(ElectionMember::Table) .col(ColumnDef::new(ElectionMember::Service).string().not_null()) - .col(ColumnDef::new(ElectionMember::Id).string().not_null()) + .col( + ColumnDef::new(ElectionMember::Id) + .string() + .primary_key() + .not_null(), + ) .col( ColumnDef::new(ElectionMember::LastHeartbeat) .timestamp() @@ -777,17 +784,6 @@ impl MigrationTrait for Migration { .to_owned(), ) .await?; - manager - .create_index( - MigrationIndex::create() - .table(ElectionMember::Table) - .name("election_member_pk") - .unique() - .col(ElectionMember::Service) - .col(ElectionMember::Id) - .to_owned(), - ) - .await?; // 4. initialize data. let insert_cluster_id = Query::insert() @@ -1058,6 +1054,8 @@ enum Sink { Properties, Definition, ConnectionId, + DbName, + SinkFromName, } #[derive(DeriveIden)] diff --git a/src/meta/src/model_v2/mod.rs b/src/meta/src/model_v2/mod.rs index 59c2876f15bd7..f0a009ec8ae9c 100644 --- a/src/meta/src/model_v2/mod.rs +++ b/src/meta/src/model_v2/mod.rs @@ -6,6 +6,8 @@ pub mod actor; pub mod cluster; pub mod connection; pub mod database; +pub mod election_leader; +pub mod election_member; pub mod fragment; pub mod function; pub mod index; diff --git a/src/meta/src/model_v2/prelude.rs b/src/meta/src/model_v2/prelude.rs index acdf40decd72f..60beff3a781e0 100644 --- a/src/meta/src/model_v2/prelude.rs +++ b/src/meta/src/model_v2/prelude.rs @@ -4,6 +4,8 @@ pub use super::actor::Entity as Actor; pub use super::cluster::Entity as Cluster; pub use super::connection::Entity as Connection; pub use super::database::Entity as Database; +pub use super::election_leader::Entity as ElectionLeader; +pub use super::election_member::Entity as ElectionMember; pub use super::fragment::Entity as Fragment; pub use super::function::Entity as Function; pub use super::index::Entity as Index; diff --git a/src/meta/src/model_v2/sink.rs b/src/meta/src/model_v2/sink.rs index 95229595b0c04..0ef354bdfc630 100644 --- a/src/meta/src/model_v2/sink.rs +++ b/src/meta/src/model_v2/sink.rs @@ -14,10 +14,12 @@ pub struct Model { pub pk_column_ids: Option, pub distribution_key: Option, pub downstream_pk: Option, - pub sink_type: Option, + pub sink_type: String, pub properties: Option, - pub definition: Option, + pub definition: String, pub connection_id: Option, + pub db_name: String, + pub sink_from_name: String, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] From 99d6354168db79f5abcd05e8030eaca04c85e5ab Mon Sep 17 00:00:00 2001 From: August Date: Wed, 20 Sep 2023 14:32:27 +0800 Subject: [PATCH 17/30] add election tables --- src/meta/src/model_v2/election_leader.rs | 17 +++++++++++++++++ src/meta/src/model_v2/election_member.rs | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 src/meta/src/model_v2/election_leader.rs create mode 100644 src/meta/src/model_v2/election_member.rs diff --git a/src/meta/src/model_v2/election_leader.rs b/src/meta/src/model_v2/election_leader.rs new file mode 100644 index 0000000000000..e5d82f84f54f8 --- /dev/null +++ b/src/meta/src/model_v2/election_leader.rs @@ -0,0 +1,17 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "election_leader")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub service: String, + pub id: String, + pub last_heartbeat: DateTime, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/meta/src/model_v2/election_member.rs b/src/meta/src/model_v2/election_member.rs new file mode 100644 index 0000000000000..d23a4a85a3be3 --- /dev/null +++ b/src/meta/src/model_v2/election_member.rs @@ -0,0 +1,17 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "election_member")] +pub struct Model { + pub service: String, + #[sea_orm(primary_key, auto_increment = false)] + pub id: String, + pub last_heartbeat: DateTime, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} From dcbb94eaeb9718e2060fa02c3b3406bbaf9f5da9 Mon Sep 17 00:00:00 2001 From: yezizp2012 Date: Wed, 20 Sep 2023 06:43:56 +0000 Subject: [PATCH 18/30] Fix "cargo-hakari" --- Cargo.lock | 8 ++++++++ src/workspace-hack/Cargo.toml | 33 +++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5168043f74203..da167e822b668 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10784,6 +10784,7 @@ dependencies = [ "num-traits", "opentelemetry_api", "opentelemetry_sdk", + "ordered-float 3.9.1", "parking_lot 0.12.1", "parking_lot_core 0.9.8", "petgraph", @@ -10803,6 +10804,9 @@ dependencies = [ "ring", "rust_decimal", "scopeguard", + "sea-orm", + "sea-query", + "sea-query-binder", "serde", "serde_json", "serde_with 3.3.0", @@ -10810,10 +10814,13 @@ dependencies = [ "sha2", "signature", "smallvec", + "socket2 0.5.4", + "sqlx", "sqlx-core", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", + "strum 0.25.0", "subtle", "syn 1.0.109", "syn 2.0.37", @@ -10835,6 +10842,7 @@ dependencies = [ "unicode-normalization", "url", "uuid", + "whoami", ] [[package]] diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml index 9a8ddc3de1173..11d3d76deb1b5 100644 --- a/src/workspace-hack/Cargo.toml +++ b/src/workspace-hack/Cargo.toml @@ -66,7 +66,7 @@ lexical-write-float = { version = "0.8", default-features = false, features = [" lexical-write-integer = { version = "0.8", default-features = false, features = ["format", "std"] } libc = { version = "0.2", features = ["extra_traits"] } lock_api = { version = "0.4", features = ["arc_lock"] } -log = { version = "0.4", default-features = false, features = ["std"] } +log = { version = "0.4", default-features = false, features = ["kv_unstable", "std"] } madsim-rdkafka = { git = "https://github.com/madsim-rs/madsim.git", rev = "fedb1e3", features = ["cmake-build", "gssapi", "ssl-vendored", "zstd"] } madsim-tokio = { version = "0.2", default-features = false, features = ["fs", "io-util", "macros", "net", "process", "rt", "rt-multi-thread", "signal", "sync", "time", "tracing"] } md-5 = { version = "0.10" } @@ -79,6 +79,7 @@ num-iter = { version = "0.1", default-features = false, features = ["i128", "std num-traits = { version = "0.2", features = ["i128", "libm"] } opentelemetry_api = { version = "0.20", features = ["logs", "metrics"] } opentelemetry_sdk = { version = "0.20", features = ["logs", "metrics"] } +ordered-float = { version = "3" } parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] } parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] } petgraph = { version = "0.6" } @@ -97,6 +98,9 @@ reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls"] } ring = { version = "0.16", features = ["std"] } rust_decimal = { version = "1", features = ["db-postgres", "maths"] } scopeguard = { version = "1" } +sea-orm = { version = "0.12", features = ["runtime-tokio-native-tls", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite"] } +sea-query = { version = "0.30", default-features = false, features = ["backend-mysql", "backend-postgres", "backend-sqlite", "derive", "hashable-value", "postgres-array", "thread-safe", "with-bigdecimal", "with-chrono", "with-json", "with-rust_decimal", "with-time", "with-uuid"] } +sea-query-binder = { version = "0.5", default-features = false, features = ["postgres-array", "runtime-tokio-native-tls", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "with-bigdecimal", "with-chrono", "with-json", "with-rust_decimal", "with-time", "with-uuid"] } serde = { version = "1", features = ["alloc", "derive", "rc"] } serde_json = { version = "1", features = ["alloc", "raw_value"] } serde_with = { version = "3", features = ["json"] } @@ -104,10 +108,13 @@ sha1 = { version = "0.10" } sha2 = { version = "0.10" } signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] } smallvec = { version = "1", default-features = false, features = ["serde", "union", "write"] } -sqlx-core = { version = "0.7", features = ["_rt-tokio", "any", "chrono", "json", "migrate", "offline"] } -sqlx-mysql = { version = "0.7", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } -sqlx-postgres = { version = "0.7", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } -sqlx-sqlite = { version = "0.7", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +socket2 = { version = "0.5", default-features = false, features = ["all"] } +sqlx = { version = "0.7", features = ["bigdecimal", "chrono", "mysql", "postgres", "runtime-tokio-native-tls", "rust_decimal", "sqlite", "time", "uuid"] } +sqlx-core = { version = "0.7", features = ["_rt-tokio", "_tls-native-tls", "any", "bigdecimal", "chrono", "json", "migrate", "offline", "rust_decimal", "time", "uuid"] } +sqlx-mysql = { version = "0.7", default-features = false, features = ["any", "bigdecimal", "chrono", "json", "migrate", "offline", "rust_decimal", "time", "uuid"] } +sqlx-postgres = { version = "0.7", default-features = false, features = ["any", "bigdecimal", "chrono", "json", "migrate", "offline", "rust_decimal", "time", "uuid"] } +sqlx-sqlite = { version = "0.7", default-features = false, features = ["any", "chrono", "json", "migrate", "offline", "time", "uuid"] } +strum = { version = "0.25", features = ["derive"] } subtle = { version = "2" } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } tinyvec = { version = "1", features = ["alloc", "grab_spare_slice", "rustc_1_55"] } @@ -126,6 +133,7 @@ unicode-bidi = { version = "0.3" } unicode-normalization = { version = "0.1" } url = { version = "2", features = ["serde"] } uuid = { version = "1", features = ["fast-rng", "serde", "v4"] } +whoami = { version = "1" } [build-dependencies] ahash = { version = "0.8" } @@ -157,7 +165,7 @@ itertools = { version = "0.10" } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits"] } lock_api = { version = "0.4", features = ["arc_lock"] } -log = { version = "0.4", default-features = false, features = ["std"] } +log = { version = "0.4", default-features = false, features = ["kv_unstable", "std"] } madsim-tokio = { version = "0.2", default-features = false, features = ["fs", "io-util", "macros", "net", "process", "rt", "rt-multi-thread", "signal", "sync", "time", "tracing"] } md-5 = { version = "0.10" } mio = { version = "0.8", features = ["net", "os-ext"] } @@ -172,6 +180,7 @@ parking_lot_core = { version = "0.9", default-features = false, features = ["dea petgraph = { version = "0.6" } phf = { version = "0.11", features = ["uncased"] } phf_shared = { version = "0.11", features = ["uncased"] } +postgres-types = { version = "0.2", default-features = false, features = ["derive", "with-chrono-0_4", "with-serde_json-1"] } proc-macro2 = { version = "1", features = ["span-locations"] } prost = { version = "0.11", features = ["no-recursion-limit"] } rand = { version = "0.8", features = ["small_rng"] } @@ -180,6 +189,7 @@ rand_core = { version = "0.6", default-features = false, features = ["std"] } regex = { version = "1" } regex-automata = { version = "0.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.7" } +rust_decimal = { version = "1", features = ["db-postgres", "maths"] } scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive", "rc"] } serde_json = { version = "1", features = ["alloc", "raw_value"] } @@ -187,10 +197,10 @@ sha1 = { version = "0.10" } sha2 = { version = "0.10" } signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] } smallvec = { version = "1", default-features = false, features = ["serde", "union", "write"] } -sqlx-core = { version = "0.7", features = ["_rt-tokio", "any", "chrono", "json", "migrate", "offline"] } -sqlx-mysql = { version = "0.7", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } -sqlx-postgres = { version = "0.7", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } -sqlx-sqlite = { version = "0.7", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] } +sqlx-core = { version = "0.7", features = ["_rt-tokio", "_tls-native-tls", "any", "bigdecimal", "chrono", "json", "migrate", "offline", "rust_decimal", "time", "uuid"] } +sqlx-mysql = { version = "0.7", default-features = false, features = ["any", "bigdecimal", "chrono", "json", "migrate", "offline", "rust_decimal", "time", "uuid"] } +sqlx-postgres = { version = "0.7", default-features = false, features = ["any", "bigdecimal", "chrono", "json", "migrate", "offline", "rust_decimal", "time", "uuid"] } +sqlx-sqlite = { version = "0.7", default-features = false, features = ["any", "chrono", "json", "migrate", "offline", "time", "uuid"] } subtle = { version = "2" } syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } @@ -199,6 +209,7 @@ time-macros = { version = "0.2", default-features = false, features = ["formatti tinyvec = { version = "1", features = ["alloc", "grab_spare_slice", "rustc_1_55"] } tokio = { version = "1", features = ["full", "stats", "tracing"] } tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "fe39bb8e", features = ["fs", "net"] } +tokio-util = { version = "0.7", features = ["codec", "io"] } toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } toml_edit = { version = "0.19", features = ["serde"] } tracing = { version = "0.1", features = ["log"] } @@ -206,5 +217,7 @@ tracing-core = { version = "0.1" } unicode-bidi = { version = "0.3" } unicode-normalization = { version = "0.1" } url = { version = "2", features = ["serde"] } +uuid = { version = "1", features = ["fast-rng", "serde", "v4"] } +whoami = { version = "1" } ### END HAKARI SECTION From db605b83777dca5be93c109e84f759ef506b58df Mon Sep 17 00:00:00 2001 From: August Date: Wed, 20 Sep 2023 14:51:19 +0800 Subject: [PATCH 19/30] ignore license checker for new models --- .licenserc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.licenserc.yaml b/.licenserc.yaml index e596f12143b13..2a041fe660926 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -17,5 +17,6 @@ header: - "**/*.d.ts" - "src/sqlparser/**/*.rs" - "java/connector-node/risingwave-source-cdc/src/main/java/com/risingwave/connector/cdc/debezium/internal/*.java" + - "src/meta/src/model_v2/**/*.rs" comment: on-failure From 11666217415ce869a8d906f1a8000997b352d467 Mon Sep 17 00:00:00 2001 From: yezizp2012 Date: Wed, 20 Sep 2023 07:58:44 +0000 Subject: [PATCH 20/30] Fix "cargo-hakari" --- Cargo.lock | 1 - src/workspace-hack/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 052e9448ac198..c701cfcf962a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10788,7 +10788,6 @@ dependencies = [ "sha2", "signature", "smallvec", - "socket2 0.5.3", "sqlx", "sqlx-core", "sqlx-mysql", diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml index d572324a20f37..601a997b6a3d6 100644 --- a/src/workspace-hack/Cargo.toml +++ b/src/workspace-hack/Cargo.toml @@ -108,7 +108,6 @@ sha1 = { version = "0.10" } sha2 = { version = "0.10" } signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] } smallvec = { version = "1", default-features = false, features = ["serde", "union", "write"] } -socket2 = { version = "0.5", default-features = false, features = ["all"] } sqlx = { version = "0.7", features = ["bigdecimal", "chrono", "mysql", "postgres", "runtime-tokio-native-tls", "rust_decimal", "sqlite", "time", "uuid"] } sqlx-core = { version = "0.7", features = ["_rt-tokio", "_tls-native-tls", "any", "bigdecimal", "chrono", "json", "migrate", "offline", "rust_decimal", "time", "uuid"] } sqlx-mysql = { version = "0.7", default-features = false, features = ["any", "bigdecimal", "chrono", "json", "migrate", "offline", "rust_decimal", "time", "uuid"] } @@ -208,6 +207,7 @@ time = { version = "0.3", features = ["local-offset", "macros", "serde-well-know time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] } tinyvec = { version = "1", features = ["alloc", "grab_spare_slice", "rustc_1_55"] } tokio = { version = "1", features = ["full", "stats", "tracing"] } +tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "ac00d88", features = ["with-chrono-0_4"] } tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "fe39bb8e", features = ["fs", "net"] } tokio-util = { version = "0.7", features = ["codec", "io"] } toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } From 7e35fb5d333af57d5ec5ab60d1399e9e7277a7c9 Mon Sep 17 00:00:00 2001 From: August Date: Wed, 20 Sep 2023 18:00:26 +0800 Subject: [PATCH 21/30] use enum and i32 array instead in the models --- .licenserc.yaml | 2 +- src/meta/src/model_v2/actor.rs | 18 ++++++++-- src/meta/src/model_v2/cluster.rs | 14 +++++++- src/meta/src/model_v2/connection.rs | 14 +++++++- src/meta/src/model_v2/database.rs | 14 +++++++- src/meta/src/model_v2/election_leader.rs | 14 +++++++- src/meta/src/model_v2/election_member.rs | 14 +++++++- src/meta/src/model_v2/fragment.rs | 24 ++++++++++--- src/meta/src/model_v2/function.rs | 14 +++++++- src/meta/src/model_v2/index.rs | 18 ++++++++-- src/meta/src/model_v2/mod.rs | 20 ++++++++++- src/meta/src/model_v2/object.rs | 14 +++++++- src/meta/src/model_v2/object_dependency.rs | 14 +++++++- src/meta/src/model_v2/prelude.rs | 14 +++++++- src/meta/src/model_v2/schema.rs | 14 +++++++- src/meta/src/model_v2/sink.rs | 33 ++++++++++++++--- src/meta/src/model_v2/source.rs | 14 +++++++- src/meta/src/model_v2/system_parameter.rs | 14 +++++++- src/meta/src/model_v2/table.rs | 26 ++++++++++---- src/meta/src/model_v2/user.rs | 14 +++++++- src/meta/src/model_v2/user_privilege.rs | 14 +++++++- src/meta/src/model_v2/view.rs | 14 +++++++- src/meta/src/model_v2/worker.rs | 41 ++++++++++++++++++++-- src/meta/src/model_v2/worker_property.rs | 18 ++++++++-- 24 files changed, 369 insertions(+), 41 deletions(-) diff --git a/.licenserc.yaml b/.licenserc.yaml index 2a041fe660926..c1745a4d1ad74 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -17,6 +17,6 @@ header: - "**/*.d.ts" - "src/sqlparser/**/*.rs" - "java/connector-node/risingwave-source-cdc/src/main/java/com/risingwave/connector/cdc/debezium/internal/*.java" - - "src/meta/src/model_v2/**/*.rs" + - "src/meta/src/model_v2/migration/**/*.rs" comment: on-failure diff --git a/src/meta/src/model_v2/actor.rs b/src/meta/src/model_v2/actor.rs index 92c18111f6a77..8fecb3046b1bc 100644 --- a/src/meta/src/model_v2/actor.rs +++ b/src/meta/src/model_v2/actor.rs @@ -1,7 +1,21 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; +use crate::model_v2::I32Array; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "actor")] pub struct Model { @@ -11,7 +25,7 @@ pub struct Model { pub status: Option, pub splits: Option, pub parallel_unit_id: i32, - pub upstream_actor_ids: Option, + pub upstream_actor_ids: Option, pub dispatchers: Option, pub vnode_bitmap: Option, } diff --git a/src/meta/src/model_v2/cluster.rs b/src/meta/src/model_v2/cluster.rs index be1e662f07913..36cdb449046bf 100644 --- a/src/meta/src/model_v2/cluster.rs +++ b/src/meta/src/model_v2/cluster.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/connection.rs b/src/meta/src/model_v2/connection.rs index e105ee1e4047a..6f56002780411 100644 --- a/src/meta/src/model_v2/connection.rs +++ b/src/meta/src/model_v2/connection.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/database.rs b/src/meta/src/model_v2/database.rs index c0fdeca4bb959..dca87c32b9f44 100644 --- a/src/meta/src/model_v2/database.rs +++ b/src/meta/src/model_v2/database.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/election_leader.rs b/src/meta/src/model_v2/election_leader.rs index e5d82f84f54f8..b56991ffd26f9 100644 --- a/src/meta/src/model_v2/election_leader.rs +++ b/src/meta/src/model_v2/election_leader.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/election_member.rs b/src/meta/src/model_v2/election_member.rs index d23a4a85a3be3..4c441d5aa0b73 100644 --- a/src/meta/src/model_v2/election_member.rs +++ b/src/meta/src/model_v2/election_member.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/fragment.rs b/src/meta/src/model_v2/fragment.rs index 0c116cf8929c5..9263dd99eabb8 100644 --- a/src/meta/src/model_v2/fragment.rs +++ b/src/meta/src/model_v2/fragment.rs @@ -1,7 +1,21 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; +use crate::model_v2::I32Array; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "fragment")] pub struct Model { @@ -12,11 +26,11 @@ pub struct Model { pub distribution_type: String, pub stream_node: Json, pub vnode_mapping: Option, - pub state_table_ids: Option, - pub upstream_fragment_id: Option, + pub state_table_ids: Option, + pub upstream_fragment_id: Option, pub dispatcher_type: Option, - pub dist_key_indices: Option, - pub output_indices: Option, + pub dist_key_indices: Option, + pub output_indices: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/function.rs b/src/meta/src/model_v2/function.rs index e0855613b71da..e60f9ea5a2bde 100644 --- a/src/meta/src/model_v2/function.rs +++ b/src/meta/src/model_v2/function.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/index.rs b/src/meta/src/model_v2/index.rs index 6211f4bf3c16b..4cd3cf212a03c 100644 --- a/src/meta/src/model_v2/index.rs +++ b/src/meta/src/model_v2/index.rs @@ -1,7 +1,21 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; +use crate::model_v2::I32Array; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "index")] pub struct Model { @@ -13,7 +27,7 @@ pub struct Model { pub index_table_id: i32, pub primary_table_id: i32, pub index_items: Option, - pub original_columns: Option, + pub original_columns: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/mod.rs b/src/meta/src/model_v2/mod.rs index f0a009ec8ae9c..8f496b56d1291 100644 --- a/src/meta/src/model_v2/mod.rs +++ b/src/meta/src/model_v2/mod.rs @@ -1,4 +1,19 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sea_orm::FromJsonQueryResult; +use serde::{Deserialize, Serialize}; pub mod prelude; @@ -23,3 +38,6 @@ pub mod user_privilege; pub mod view; pub mod worker; pub mod worker_property; + +#[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Eq, Serialize, Deserialize, Default)] +pub struct I32Array(pub Vec); diff --git a/src/meta/src/model_v2/object.rs b/src/meta/src/model_v2/object.rs index b6a22dcc54086..bdd4f47b03b2b 100644 --- a/src/meta/src/model_v2/object.rs +++ b/src/meta/src/model_v2/object.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/object_dependency.rs b/src/meta/src/model_v2/object_dependency.rs index c24e389bae106..b1a0164ff0f4b 100644 --- a/src/meta/src/model_v2/object_dependency.rs +++ b/src/meta/src/model_v2/object_dependency.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/prelude.rs b/src/meta/src/model_v2/prelude.rs index 60beff3a781e0..d929fa2a4aece 100644 --- a/src/meta/src/model_v2/prelude.rs +++ b/src/meta/src/model_v2/prelude.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. pub use super::actor::Entity as Actor; pub use super::cluster::Entity as Cluster; diff --git a/src/meta/src/model_v2/schema.rs b/src/meta/src/model_v2/schema.rs index 759ab4c0fc9fc..927989a1e6fb4 100644 --- a/src/meta/src/model_v2/schema.rs +++ b/src/meta/src/model_v2/schema.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/sink.rs b/src/meta/src/model_v2/sink.rs index 0ef354bdfc630..5501d05387edf 100644 --- a/src/meta/src/model_v2/sink.rs +++ b/src/meta/src/model_v2/sink.rs @@ -1,7 +1,32 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; +use crate::model_v2::I32Array; + +#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "String(None)")] +pub enum SinkType { + #[sea_orm(string_value = "APPEND_ONLY")] + AppendOnly, + #[sea_orm(string_value = "FORCE_APPEND_ONLY")] + ForceAppendOnly, + #[sea_orm(string_value = "UPSERT")] + Upsert, +} + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "sink")] pub struct Model { @@ -12,9 +37,9 @@ pub struct Model { pub database_id: i32, pub columns: Option, pub pk_column_ids: Option, - pub distribution_key: Option, - pub downstream_pk: Option, - pub sink_type: String, + pub distribution_key: Option, + pub downstream_pk: Option, + pub sink_type: SinkType, pub properties: Option, pub definition: String, pub connection_id: Option, diff --git a/src/meta/src/model_v2/source.rs b/src/meta/src/model_v2/source.rs index b65e9051a206f..f498bd643da8f 100644 --- a/src/meta/src/model_v2/source.rs +++ b/src/meta/src/model_v2/source.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/system_parameter.rs b/src/meta/src/model_v2/system_parameter.rs index 422e10f0722b2..366c3f743187b 100644 --- a/src/meta/src/model_v2/system_parameter.rs +++ b/src/meta/src/model_v2/system_parameter.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/table.rs b/src/meta/src/model_v2/table.rs index 8a9ad68bd6786..9b43272358d82 100644 --- a/src/meta/src/model_v2/table.rs +++ b/src/meta/src/model_v2/table.rs @@ -1,7 +1,21 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; +use crate::model_v2::I32Array; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "table")] pub struct Model { @@ -14,19 +28,19 @@ pub struct Model { pub table_type: Option, pub columns: Option, pub pk: Option, - pub distribution_key: Option, + pub distribution_key: Option, pub append_only: Option, pub properties: Option, pub fragment_id: Option, pub vnode_col_index: Option, - pub value_indices: Option, + pub value_indices: Option, pub definition: Option, pub handle_pk_conflict_behavior: Option, pub read_prefix_len_hint: Option, - pub watermark_indices: Option, - pub dist_key_in_pk: Option, + pub watermark_indices: Option, + pub dist_key_in_pk: Option, pub dml_fragment_id: Option, - pub cardinality: Option, + pub cardinality: Option, pub cleaned_by_watermark: Option, pub version: Option, } diff --git a/src/meta/src/model_v2/user.rs b/src/meta/src/model_v2/user.rs index d3b87f99a2c0b..b37316684470e 100644 --- a/src/meta/src/model_v2/user.rs +++ b/src/meta/src/model_v2/user.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/user_privilege.rs b/src/meta/src/model_v2/user_privilege.rs index 35e8644bc5383..4079323904994 100644 --- a/src/meta/src/model_v2/user_privilege.rs +++ b/src/meta/src/model_v2/user_privilege.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/view.rs b/src/meta/src/model_v2/view.rs index 9f520e73d9fa0..b0689e56afeef 100644 --- a/src/meta/src/model_v2/view.rs +++ b/src/meta/src/model_v2/view.rs @@ -1,4 +1,16 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; diff --git a/src/meta/src/model_v2/worker.rs b/src/meta/src/model_v2/worker.rs index 02d12d89d4457..4002b55323a36 100644 --- a/src/meta/src/model_v2/worker.rs +++ b/src/meta/src/model_v2/worker.rs @@ -1,16 +1,51 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; +#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "String(None)")] +pub enum WorkerType { + #[sea_orm(string_value = "FRONTEND")] + Frontend, + #[sea_orm(string_value = "COMPUTE_NODE")] + ComputeNode, + #[sea_orm(string_value = "RISE_CTL")] + RiseCtl, + #[sea_orm(string_value = "COMPACTOR")] + Compactor, + #[sea_orm(string_value = "META")] + Meta, +} + +#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "String(None)")] +pub enum WorkerStatus { + #[sea_orm(string_value = "STARTING")] + Starting, + #[sea_orm(string_value = "RUNNING")] + Running, +} #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "worker")] pub struct Model { #[sea_orm(primary_key)] pub worker_id: i32, - pub worker_type: String, + pub worker_type: WorkerType, pub host: String, pub port: i32, - pub status: String, + pub status: WorkerStatus, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/meta/src/model_v2/worker_property.rs b/src/meta/src/model_v2/worker_property.rs index 8d15af22eeee8..862733d6f8544 100644 --- a/src/meta/src/model_v2/worker_property.rs +++ b/src/meta/src/model_v2/worker_property.rs @@ -1,13 +1,27 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use sea_orm::entity::prelude::*; +use crate::model_v2::I32Array; + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "worker_property")] pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub worker_id: i32, - pub parallel_unit_ids: Json, + pub parallel_unit_ids: I32Array, pub is_streaming: bool, pub is_serving: bool, pub is_unschedulable: bool, From c93ec5c4e3c5114c666acbefebc5601907650fda Mon Sep 17 00:00:00 2001 From: August Date: Wed, 20 Sep 2023 18:36:03 +0800 Subject: [PATCH 22/30] refine object type --- src/meta/src/controller/catalog.rs | 5 +++-- src/meta/src/controller/mod.rs | 16 ---------------- src/meta/src/model_v2/object.rs | 25 ++++++++++++++++++++++++- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index ca20a44a1018d..d06485430253f 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -31,8 +31,9 @@ use sea_orm::{ }; use tokio::sync::RwLock; -use crate::controller::{ObjectModel, ObjectType}; +use crate::controller::ObjectModel; use crate::manager::{DatabaseId, MetaSrvEnv, NotificationVersion, UserId}; +use crate::model_v2::object::ObjectType; use crate::model_v2::prelude::*; use crate::model_v2::{ connection, database, function, index, object, object_dependency, schema, sink, source, table, @@ -94,7 +95,7 @@ impl CatalogController { owner_id: UserId, ) -> MetaResult { let mut active_db = object::ActiveModel::new(); - active_db.obj_type = ActiveValue::Set(obj_type.to_string()); + active_db.obj_type = ActiveValue::Set(obj_type); active_db.owner_id = ActiveValue::Set(owner_id as _); Ok(active_db.insert(txn).await?) } diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index d9d8e1c0439d3..9d3298fdbd5b7 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. use anyhow::anyhow; -use parse_display::Display; use risingwave_pb::catalog::{PbDatabase, PbSchema}; use sea_orm::{ActiveValue, DatabaseConnection, ModelTrait}; @@ -50,21 +49,6 @@ impl SqlMetaStore { } } -#[derive(Clone, Display, Debug)] -#[display(style = "UPPERCASE")] -#[allow(dead_code)] -enum ObjectType { - Database, - Schema, - Table, - Source, - Sink, - View, - Index, - Function, - Connection, -} - pub struct ObjectModel(M, object::Model); impl From> for PbDatabase { diff --git a/src/meta/src/model_v2/object.rs b/src/meta/src/model_v2/object.rs index bdd4f47b03b2b..a2bc1164b631d 100644 --- a/src/meta/src/model_v2/object.rs +++ b/src/meta/src/model_v2/object.rs @@ -14,12 +14,35 @@ use sea_orm::entity::prelude::*; +#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "String(None)")] +pub enum ObjectType { + #[sea_orm(string_value = "DATABASE")] + Database, + #[sea_orm(string_value = "SCHEMA")] + Schema, + #[sea_orm(string_value = "TABLE")] + Table, + #[sea_orm(string_value = "SOURCE")] + Source, + #[sea_orm(string_value = "SINK")] + Sink, + #[sea_orm(string_value = "VIEW")] + View, + #[sea_orm(string_value = "INDEX")] + Index, + #[sea_orm(string_value = "FUNCTION")] + Function, + #[sea_orm(string_value = "CONNECTION")] + Connection, +} + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "object")] pub struct Model { #[sea_orm(primary_key)] pub oid: i32, - pub obj_type: String, + pub obj_type: ObjectType, pub owner_id: i32, pub initialized_at: DateTime, pub created_at: DateTime, From eee3c592a2ae8c751c7e32898089e27fe6759a2f Mon Sep 17 00:00:00 2001 From: August Date: Thu, 21 Sep 2023 15:33:13 +0800 Subject: [PATCH 23/30] add README doc for defining and generate model files --- src/meta/src/model_v2/README.md | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/meta/src/model_v2/README.md diff --git a/src/meta/src/model_v2/README.md b/src/meta/src/model_v2/README.md new file mode 100644 index 0000000000000..a75ab6ef5da44 --- /dev/null +++ b/src/meta/src/model_v2/README.md @@ -0,0 +1,50 @@ +# How to define changes between versions and generate migration and model files + +- Generate a new migration file and apply it to the database, check [migration](./migration/README.md) for more details. Let's take a local PG database as an example(`postgres://postgres:@localhost:5432/postgres`): + ```sh + export DATABASE_URL=postgres://postgres:@localhost:5432/postgres; + cargo run -- generate MIGRATION_NAME + cargo run -- up + ``` + - Define tables, indexes, foreign keys in the file. The new generated file will include a sample migration script, + you can replace it with your own migration scripts, like defining or changing tables, indexes, foreign keys and other + dml operation to do data correctness etc. Check [writing-migration](https://www.sea-ql.org/SeaORM/docs/migration/writing-migration/) + for more details. + ```rust + #[async_trait::async_trait] + impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Replace the sample below with your own migration scripts + todo!(); + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Replace the sample below with your own migration scripts + todo!(); + } + } + ``` +- Apply migration, and generate model files for new tables and indexes from the database, so you don't need to write them manually, + ```sh + cargo run -- up + sea-orm-cli generate entity -u postgres://postgres:@localhost:5432/postgres -s public -o {target_dir} + cp {target_dir}/xxx.rs src/meta/src/model_v2/ + ``` +- Defines enum and array types in the model files, since they're basically only supported in PG, and we need to + define them in the model files manually. For example: + ```rust + // We define integer array typed fields as json and derive it using the follow one. + #[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Eq, Serialize, Deserialize, Default)] + pub struct I32Array(pub Vec); + + // We define enum typed fields as string and derive it using the follow one. + #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] + #[sea_orm(rs_type = "String", db_type = "String(None)")] + pub enum WorkerStatus { + #[sea_orm(string_value = "STARTING")] + Starting, + #[sea_orm(string_value = "RUNNING")] + Running, + } + ``` +- Define other helper functions in the model files if necessary. \ No newline at end of file From cde742e3a69eebc2b70c06f12802511a2b053edc Mon Sep 17 00:00:00 2001 From: August Date: Thu, 21 Sep 2023 15:42:53 +0800 Subject: [PATCH 24/30] fix --- src/meta/src/model_v2/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/meta/src/model_v2/README.md b/src/meta/src/model_v2/README.md index a75ab6ef5da44..25c22a4f566e1 100644 --- a/src/meta/src/model_v2/README.md +++ b/src/meta/src/model_v2/README.md @@ -6,9 +6,9 @@ cargo run -- generate MIGRATION_NAME cargo run -- up ``` - - Define tables, indexes, foreign keys in the file. The new generated file will include a sample migration script, - you can replace it with your own migration scripts, like defining or changing tables, indexes, foreign keys and other - dml operation to do data correctness etc. Check [writing-migration](https://www.sea-ql.org/SeaORM/docs/migration/writing-migration/) + - Define tables, indexes, foreign keys in the file. The new generated file will include a sample migration script, + you can replace it with your own migration scripts, like defining or changing tables, indexes, foreign keys and other + dml operation to do data correctness etc. Check [writing-migration](https://www.sea-ql.org/SeaORM/docs/migration/writing-migration/) for more details. ```rust #[async_trait::async_trait] @@ -30,13 +30,13 @@ sea-orm-cli generate entity -u postgres://postgres:@localhost:5432/postgres -s public -o {target_dir} cp {target_dir}/xxx.rs src/meta/src/model_v2/ ``` -- Defines enum and array types in the model files, since they're basically only supported in PG, and we need to +- Defines enum and array types in the model files, since they're basically only supported in PG, and we need to define them in the model files manually. For example: ```rust // We define integer array typed fields as json and derive it using the follow one. #[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Eq, Serialize, Deserialize, Default)] pub struct I32Array(pub Vec); - + // We define enum typed fields as string and derive it using the follow one. #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] #[sea_orm(rs_type = "String", db_type = "String(None)")] From 1965d4699c3d85d95dea48029ed8e6a59c84b140 Mon Sep 17 00:00:00 2001 From: August Date: Thu, 21 Sep 2023 15:49:40 +0800 Subject: [PATCH 25/30] fix --- src/meta/src/model_v2/migration/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meta/src/model_v2/migration/Cargo.toml b/src/meta/src/model_v2/migration/Cargo.toml index 30b61d5504926..d5d51d77da909 100644 --- a/src/meta/src/model_v2/migration/Cargo.toml +++ b/src/meta/src/model_v2/migration/Cargo.toml @@ -14,4 +14,4 @@ uuid = { version = "1", features = ["v4"] } [dependencies.sea-orm-migration] version = "0.12.0" -features = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls", "with-uuid" ] +features = ["sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls", "with-uuid"] From c3043ec4afa8c92062dcdc8f3ca2b4fb8b7da6bd Mon Sep 17 00:00:00 2001 From: August Date: Thu, 21 Sep 2023 16:10:37 +0800 Subject: [PATCH 26/30] auto upgrade in leader only --- src/meta/src/rpc/server.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/meta/src/rpc/server.rs b/src/meta/src/rpc/server.rs index 4ea4e2d45f2e9..82d306552a30b 100644 --- a/src/meta/src/rpc/server.rs +++ b/src/meta/src/rpc/server.rs @@ -146,11 +146,6 @@ pub async fn rpc_serve( .connect_timeout(Duration::from_secs(10)) .idle_timeout(Duration::from_secs(30)); let conn = sea_orm::Database::connect(options).await?; - // Try to upgrade if any new model changes are added. - Migrator::up(&conn, None) - .await - .expect("Failed to upgrade models in meta store"); - Some(SqlMetaStore::new(conn)) } None => None, @@ -383,6 +378,13 @@ pub async fn start_service_as_election_leader( mut svc_shutdown_rx: WatchReceiver<()>, ) -> MetaResult<()> { tracing::info!("Defining leader services"); + if let Some(sql_store) = &meta_store_sql { + // Try to upgrade if any new model changes are added. + Migrator::up(&sql_store.conn, None) + .await + .expect("Failed to upgrade models in meta store"); + } + let prometheus_endpoint = opts.prometheus_endpoint.clone(); let env = MetaSrvEnv::new( opts.clone(), From 3dd9fdd52fd5e06e27a1b90e335329a56e531c5b Mon Sep 17 00:00:00 2001 From: August Date: Thu, 21 Sep 2023 16:28:51 +0800 Subject: [PATCH 27/30] clippy --- src/meta/src/controller/catalog.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index d06485430253f..b354ac1749105 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -54,7 +54,7 @@ pub struct ReleaseContext { } impl CatalogController { - pub async fn new(env: MetaSrvEnv) -> MetaResult { + pub fn new(env: MetaSrvEnv) -> MetaResult { let meta_store = env .sql_meta_store() .expect("sql meta store is not initialized"); @@ -85,7 +85,7 @@ impl CatalogController { } impl CatalogController { - pub async fn snapshot(&self) -> MetaResult<()> { + pub fn snapshot(&self) -> MetaResult<()> { todo!("snapshot") } @@ -366,9 +366,7 @@ mod tests { #[tokio::test] async fn test_create_database() { - let mgr = CatalogController::new(MetaSrvEnv::for_test().await) - .await - .unwrap(); + let mgr = CatalogController::new(MetaSrvEnv::for_test().await).unwrap(); let db = PbDatabase { name: "test".to_string(), owner: DEFAULT_SUPER_USER_ID, From 235bcd18db53c9da1f520c359e713de906638bdf Mon Sep 17 00:00:00 2001 From: August Date: Thu, 21 Sep 2023 19:18:20 +0800 Subject: [PATCH 28/30] disabled in madsim --- src/meta/src/controller/catalog.rs | 1 + src/meta/src/controller/mod.rs | 1 + src/meta/src/controller/system_param.rs | 1 + src/meta/src/manager/env.rs | 30 ++++++++++++++++--------- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index b354ac1749105..3b6ddffd27338 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -365,6 +365,7 @@ mod tests { use super::*; #[tokio::test] + #[cfg(not(madsim))] async fn test_create_database() { let mgr = CatalogController::new(MetaSrvEnv::for_test().await).unwrap(); let db = PbDatabase { diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index 9d3298fdbd5b7..ccce77eba02f4 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -41,6 +41,7 @@ impl SqlMetaStore { } #[cfg(any(test, feature = "test"))] + #[cfg(not(madsim))] pub async fn for_test() -> Self { use model_migration::{Migrator, MigratorTrait}; let conn = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); diff --git a/src/meta/src/controller/system_param.rs b/src/meta/src/controller/system_param.rs index c91b0f2484366..0656da5ea9a46 100644 --- a/src/meta/src/controller/system_param.rs +++ b/src/meta/src/controller/system_param.rs @@ -256,6 +256,7 @@ mod tests { use crate::manager::MetaSrvEnv; #[tokio::test] + #[cfg(not(madsim))] async fn test_system_params() { let env = MetaSrvEnv::for_test().await; let meta_store = env.sql_meta_store().unwrap(); diff --git a/src/meta/src/manager/env.rs b/src/meta/src/manager/env.rs index b876fb71a9639..2b3ff0c525da1 100644 --- a/src/meta/src/manager/env.rs +++ b/src/meta/src/manager/env.rs @@ -357,7 +357,11 @@ impl MetaSrvEnv { pub async fn for_test_opts(opts: Arc) -> Self { // change to sync after refactor `IdGeneratorManager::new` sync. let meta_store = MemStore::default().into_ref(); - let sql_meta_store = SqlMetaStore::for_test().await; + #[cfg(madsim)] + let meta_store_sql: Option = None; + #[cfg(not(madsim))] + let meta_store_sql = Some(SqlMetaStore::for_test().await); + let id_gen_manager = Arc::new(IdGeneratorManager::new(meta_store.clone()).await); let notification_manager = Arc::new(NotificationManager::new(meta_store.clone()).await); let stream_client_pool = Arc::new(StreamClientPool::default()); @@ -373,20 +377,24 @@ impl MetaSrvEnv { .await .unwrap(), ); - let system_params_controller = Some(Arc::new( - SystemParamsController::new( - sql_meta_store.clone(), - notification_manager.clone(), - risingwave_common::system_param::system_params_for_test(), - ) - .await - .unwrap(), - )); + let system_params_controller = if let Some(store) = &meta_store_sql { + Some(Arc::new( + SystemParamsController::new( + store.clone(), + notification_manager.clone(), + risingwave_common::system_param::system_params_for_test(), + ) + .await + .unwrap(), + )) + } else { + None + }; Self { id_gen_manager, meta_store, - meta_store_sql: Some(sql_meta_store), + meta_store_sql, notification_manager, stream_client_pool, idle_manager, From 196867dd8e2b8adaa04d5c2e06b913f7415500e6 Mon Sep 17 00:00:00 2001 From: August Date: Mon, 25 Sep 2023 19:11:27 +0800 Subject: [PATCH 29/30] address comments --- src/common/src/system_param/mod.rs | 1 + src/meta/src/controller/catalog.rs | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs index 344501d8442ec..5215e1cc144ea 100644 --- a/src/common/src/system_param/mod.rs +++ b/src/common/src/system_param/mod.rs @@ -246,6 +246,7 @@ macro_rules! impl_default_from_other_params { macro_rules! impl_set_system_param { ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + /// Set a system parameter with the given value or default one, returns the new value. pub fn set_system_param(params: &mut PbSystemParams, key: &str, value: Option) -> Result { match key { $( diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index 3b6ddffd27338..233a556c70773 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -25,9 +25,9 @@ use sea_orm::sea_query::{ WithClause, }; use sea_orm::{ - ActiveModelBehavior, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, - DatabaseConnection, DatabaseTransaction, EntityTrait, Iden, JoinType, ModelTrait, Order, - QueryFilter, Statement, TransactionTrait, + ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, DatabaseConnection, + DatabaseTransaction, EntityTrait, Iden, JoinType, ModelTrait, Order, QueryFilter, Statement, + TransactionTrait, }; use tokio::sync::RwLock; @@ -94,9 +94,13 @@ impl CatalogController { obj_type: ObjectType, owner_id: UserId, ) -> MetaResult { - let mut active_db = object::ActiveModel::new(); - active_db.obj_type = ActiveValue::Set(obj_type); - active_db.owner_id = ActiveValue::Set(owner_id as _); + let active_db = object::ActiveModel { + oid: Default::default(), + obj_type: ActiveValue::Set(obj_type), + owner_id: ActiveValue::Set(owner_id as _), + initialized_at: Default::default(), + created_at: Default::default(), + }; Ok(active_db.insert(txn).await?) } @@ -112,13 +116,13 @@ impl CatalogController { let mut schemas = vec![]; for schema_name in iter::once(DEFAULT_SCHEMA_NAME).chain(SYSTEM_SCHEMAS) { - let mut schema = schema::ActiveModel::new(); let schema_obj = Self::create_object(&txn, ObjectType::Schema, owner_id).await?; - schema.schema_id = ActiveValue::Set(schema_obj.oid); - schema.database_id = ActiveValue::Set(db.database_id); - schema.name = ActiveValue::Set(schema_name.into()); + let schema = schema::ActiveModel { + schema_id: ActiveValue::Set(schema_obj.oid), + name: ActiveValue::Set(schema_name.into()), + database_id: ActiveValue::Set(db.database_id), + }; let schema = schema.insert(&txn).await?; - schemas.push(ObjectModel(schema, schema_obj).into()); } txn.commit().await?; From d1846b5517642dc9126ab0925533a8eb16b679da Mon Sep 17 00:00:00 2001 From: August Date: Tue, 26 Sep 2023 16:17:29 +0800 Subject: [PATCH 30/30] add doctest --- src/meta/src/controller/catalog.rs | 77 +++---------------------- src/meta/src/controller/mod.rs | 3 +- src/meta/src/controller/utils.rs | 93 ++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 71 deletions(-) create mode 100644 src/meta/src/controller/utils.rs diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs index 233a556c70773..2b992bab70f55 100644 --- a/src/meta/src/controller/catalog.rs +++ b/src/meta/src/controller/catalog.rs @@ -20,24 +20,20 @@ use risingwave_pb::catalog::{PbDatabase, PbSchema}; use risingwave_pb::meta::subscribe_response::{ Info as NotificationInfo, Operation as NotificationOperation, }; -use sea_orm::sea_query::{ - Alias, CommonTableExpression, Expr, Query, QueryStatementBuilder, SelectStatement, UnionType, - WithClause, -}; +use sea_orm::sea_query::{Expr, Query, QueryStatementBuilder, UnionType}; use sea_orm::{ ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, DatabaseConnection, - DatabaseTransaction, EntityTrait, Iden, JoinType, ModelTrait, Order, QueryFilter, Statement, - TransactionTrait, + DatabaseTransaction, EntityTrait, ModelTrait, QueryFilter, Statement, TransactionTrait, }; use tokio::sync::RwLock; +use crate::controller::utils::construct_obj_dependency_query; use crate::controller::ObjectModel; use crate::manager::{DatabaseId, MetaSrvEnv, NotificationVersion, UserId}; use crate::model_v2::object::ObjectType; use crate::model_v2::prelude::*; use crate::model_v2::{ - connection, database, function, index, object, object_dependency, schema, sink, source, table, - view, + connection, database, function, index, object, schema, sink, source, table, view, }; use crate::{MetaError, MetaResult}; @@ -142,70 +138,11 @@ impl CatalogController { Ok(version) } - /// This function will list all the objects that are using the given one. It runs a recursive CTE to find all the dependencies. - /// The cte and the query is as follows: - /// ```sql - /// WITH RECURSIVE used_by_object_ids (used_by) AS - /// ( - /// SELECT used_by FROM object_dependency WHERE object_dependency.oid = $1 - /// UNION ALL - /// ( - /// SELECT object_dependency.used_by - /// FROM object_dependency - /// INNER JOIN used_by_object_ids - /// ON used_by_object_ids.used_by = oid - /// ) - /// ) - /// SELECT DISTINCT used_by FROM used_by_object_ids ORDER BY used_by DESC; - /// ``` + /// List all objects that are using the given one. It runs a recursive CTE to find all the dependencies. async fn list_used_by(&self, obj_id: i32) -> MetaResult> { let inner = self.inner.read().await; - let cte_alias = Alias::new("used_by_object_ids"); - let cte_return_alias = Alias::new("used_by"); - - let base_query = SelectStatement::new() - .column(object_dependency::Column::UsedBy) - .from(ObjectDependency) - .and_where(object_dependency::Column::Oid.eq(obj_id)) - .to_owned(); - - let cte_referencing = Query::select() - .column((ObjectDependency, object_dependency::Column::UsedBy)) - .from(ObjectDependency) - .join( - JoinType::InnerJoin, - cte_alias.clone(), - Expr::col((cte_alias.clone(), cte_return_alias.clone())) - .equals(object_dependency::Column::Oid), - ) - .to_owned(); - - let common_table_expr = CommonTableExpression::new() - .query( - base_query - .clone() - .union(UnionType::All, cte_referencing) - .to_owned(), - ) - .column(cte_return_alias.clone()) - .table_name(cte_alias.clone()) - .to_owned(); - - let query = SelectStatement::new() - .distinct() - .column(cte_return_alias.clone()) - .from(cte_alias) - .order_by(cte_return_alias.clone(), Order::Desc) - .to_owned() - .with( - WithClause::new() - .recursive(true) - .cte(common_table_expr) - .to_owned(), - ) - .to_owned(); - + let query = construct_obj_dependency_query(obj_id, "used_by"); let (sql, values) = query.build_any(&*inner.db.get_database_backend().get_query_builder()); let res = inner .db @@ -218,7 +155,7 @@ impl CatalogController { let ids: Vec = res .into_iter() - .map(|row| row.try_get("", &cte_return_alias.to_string()).unwrap()) + .map(|row| row.try_get("", "user_by").unwrap()) .collect_vec(); Ok(ids) } diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs index ccce77eba02f4..968607df5e35d 100644 --- a/src/meta/src/controller/mod.rs +++ b/src/meta/src/controller/mod.rs @@ -20,8 +20,9 @@ use crate::model_v2::{database, object, schema}; use crate::MetaError; #[allow(dead_code)] -mod catalog; +pub mod catalog; pub mod system_param; +pub mod utils; // todo: refine the error transform. impl From for MetaError { diff --git a/src/meta/src/controller/utils.rs b/src/meta/src/controller/utils.rs new file mode 100644 index 0000000000000..9e2e75f9eb10b --- /dev/null +++ b/src/meta/src/controller/utils.rs @@ -0,0 +1,93 @@ +// Copyright 2023 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use model_migration::WithQuery; +use sea_orm::sea_query::{ + Alias, CommonTableExpression, Expr, Query, SelectStatement, UnionType, WithClause, +}; +use sea_orm::{ColumnTrait, JoinType, Order}; + +use crate::model_v2::object_dependency; +use crate::model_v2::prelude::*; + +/// This function will construct a query using recursive cte to find all objects that are used by the given object. +/// +/// # Examples +/// +/// ``` +/// use risingwave_meta::controller::utils::construct_obj_dependency_query; +/// use sea_orm::sea_query::*; +/// use sea_orm::*; +/// +/// let query = construct_obj_dependency_query(1, "used_by"); +/// +/// assert_eq!( +/// query.to_string(MysqlQueryBuilder), +/// r#"WITH RECURSIVE `used_by_object_ids` (`used_by`) AS (SELECT `used_by` FROM `object_dependency` WHERE `object_dependency`.`oid` = 1 UNION ALL (SELECT `object_dependency`.`used_by` FROM `object_dependency` INNER JOIN `used_by_object_ids` ON `used_by_object_ids`.`used_by` = `oid`)) SELECT DISTINCT `used_by` FROM `used_by_object_ids` ORDER BY `used_by` DESC"# +/// ); +/// assert_eq!( +/// query.to_string(PostgresQueryBuilder), +/// r#"WITH RECURSIVE "used_by_object_ids" ("used_by") AS (SELECT "used_by" FROM "object_dependency" WHERE "object_dependency"."oid" = 1 UNION ALL (SELECT "object_dependency"."used_by" FROM "object_dependency" INNER JOIN "used_by_object_ids" ON "used_by_object_ids"."used_by" = "oid")) SELECT DISTINCT "used_by" FROM "used_by_object_ids" ORDER BY "used_by" DESC"# +/// ); +/// assert_eq!( +/// query.to_string(SqliteQueryBuilder), +/// r#"WITH RECURSIVE "used_by_object_ids" ("used_by") AS (SELECT "used_by" FROM "object_dependency" WHERE "object_dependency"."oid" = 1 UNION ALL SELECT "object_dependency"."used_by" FROM "object_dependency" INNER JOIN "used_by_object_ids" ON "used_by_object_ids"."used_by" = "oid") SELECT DISTINCT "used_by" FROM "used_by_object_ids" ORDER BY "used_by" DESC"# +/// ); +/// ``` +pub fn construct_obj_dependency_query(obj_id: i32, column: &str) -> WithQuery { + let cte_alias = Alias::new("used_by_object_ids"); + let cte_return_alias = Alias::new(column); + + let base_query = SelectStatement::new() + .column(object_dependency::Column::UsedBy) + .from(ObjectDependency) + .and_where(object_dependency::Column::Oid.eq(obj_id)) + .to_owned(); + + let cte_referencing = Query::select() + .column((ObjectDependency, object_dependency::Column::UsedBy)) + .from(ObjectDependency) + .join( + JoinType::InnerJoin, + cte_alias.clone(), + Expr::col((cte_alias.clone(), cte_return_alias.clone())) + .equals(object_dependency::Column::Oid), + ) + .to_owned(); + + let common_table_expr = CommonTableExpression::new() + .query( + base_query + .clone() + .union(UnionType::All, cte_referencing) + .to_owned(), + ) + .column(cte_return_alias.clone()) + .table_name(cte_alias.clone()) + .to_owned(); + + SelectStatement::new() + .distinct() + .column(cte_return_alias.clone()) + .from(cte_alias) + .order_by(cte_return_alias.clone(), Order::Desc) + .to_owned() + .with( + WithClause::new() + .recursive(true) + .cte(common_table_expr) + .to_owned(), + ) + .to_owned() +}