From 2478845fa72964725a38824f5d132cf421863d22 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Wed, 9 Oct 2024 12:48:16 +0800 Subject: [PATCH] feat(meta): watch and reload license key from file (#18768) Signed-off-by: Bugen Zhao --- Cargo.lock | 113 +++++++++++++++++++++- src/cmd_all/src/standalone.rs | 1 + src/license/src/key.rs | 11 +++ src/license/src/manager.rs | 14 ++- src/meta/Cargo.toml | 3 + src/meta/node/src/lib.rs | 6 ++ src/meta/node/src/server.rs | 1 + src/meta/src/manager/env.rs | 5 + src/meta/src/manager/license.rs | 164 ++++++++++++++++++++++++++++++++ src/meta/src/manager/mod.rs | 1 + 10 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 src/meta/src/manager/license.rs diff --git a/Cargo.lock b/Cargo.lock index 37c8daf19c534..09c90448c7776 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4826,6 +4826,18 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox 0.1.3", + "windows-sys 0.59.0", +] + [[package]] name = "findshlibs" version = "0.10.2" @@ -5184,6 +5196,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "function_name" version = "0.3.0" @@ -6396,6 +6417,26 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "inout" version = "0.1.3" @@ -6669,6 +6710,26 @@ dependencies = [ "indexmap 1.9.3", ] +[[package]] +name = "kqueue" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "krb5-src" version = "0.3.2+1.19.2" @@ -6829,6 +6890,17 @@ dependencies = [ "redox_syscall 0.4.1", ] +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", + "redox_syscall 0.5.7", +] + [[package]] name = "libsqlite3-sys" version = "0.27.0" @@ -7738,6 +7810,24 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.6.0", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "walkdir", + "windows-sys 0.48.0", +] + [[package]] name = "npm_rs" version = "1.0.0" @@ -9827,6 +9917,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "redox_users" version = "0.4.4" @@ -9834,7 +9933,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ "getrandom", - "libredox", + "libredox 0.0.1", "thiserror", ] @@ -11366,6 +11465,7 @@ dependencies = [ "maplit", "memcomparable", "mime_guess", + "notify", "num-integer", "num-traits", "otlp-embedded", @@ -11396,6 +11496,7 @@ dependencies = [ "serde_json", "strum 0.26.3", "sync-point", + "tempfile", "thiserror", "thiserror-ext", "tokio-retry", @@ -11403,6 +11504,7 @@ dependencies = [ "tower", "tower-http", "tracing", + "tracing-subscriber", "url", "uuid", "workspace-hack", @@ -15973,6 +16075,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.42.2" diff --git a/src/cmd_all/src/standalone.rs b/src/cmd_all/src/standalone.rs index fd3e950f34d69..6946930a8b70a 100644 --- a/src/cmd_all/src/standalone.rs +++ b/src/cmd_all/src/standalone.rs @@ -468,6 +468,7 @@ mod test { dangerous_max_idle_secs: None, connector_rpc_endpoint: None, license_key: None, + license_key_file: None, temp_secret_file_dir: "./meta/secrets/", }, ), diff --git a/src/license/src/key.rs b/src/license/src/key.rs index f27ad22719e8b..2533c2a533c42 100644 --- a/src/license/src/key.rs +++ b/src/license/src/key.rs @@ -18,6 +18,17 @@ use std::str::FromStr; use serde::{Deserialize, Serialize}; /// A license key with the paid tier that only works in tests. +/// +/// The content is a JWT token with the following payload: +/// ```text +/// License { +/// sub: "rw-test", +/// iss: Test, +/// tier: Paid, +/// cpu_core_limit: None, +/// exp: 9999999999, +/// } +/// ``` pub(crate) const TEST_PAID_LICENSE_KEY_CONTENT: &str = "eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.\ eyJzdWIiOiJydy10ZXN0IiwidGllciI6InBhaWQiLCJpc3MiOiJ0ZXN0LnJpc2luZ3dhdmUuY29tIiwiZXhwIjo5OTk5OTk5OTk5fQ.\ diff --git a/src/license/src/manager.rs b/src/license/src/manager.rs index 5c1bc298388da..b6c1941ea1708 100644 --- a/src/license/src/manager.rs +++ b/src/license/src/manager.rs @@ -43,7 +43,7 @@ pub enum Tier { /// /// The issuer must be `prod.risingwave.com` in production, and can be `test.risingwave.com` in /// development. This will be validated when refreshing the license key. -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] pub enum Issuer { #[serde(rename = "prod.risingwave.com")] Prod, @@ -58,10 +58,13 @@ pub enum Issuer { /// The content of a license. /// /// We use JSON Web Token (JWT) to represent the license. This struct is the payload. +/// +/// Prefer calling [`crate::Feature::check_available`] to check the availability of a feature, +/// other than directly checking the content of the license. // TODO(license): Shall we add a version field? -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] #[serde(rename_all = "snake_case")] -pub(super) struct License { +pub struct License { /// Subject of the license. /// /// See . @@ -171,7 +174,10 @@ impl LicenseManager { /// Get the current license if it is valid. /// /// Since the license can expire, the returned license should not be cached by the caller. - pub(super) fn license(&self) -> Result { + /// + /// Prefer calling [`crate::Feature::check_available`] to check the availability of a feature, + /// other than directly calling this method and checking the content of the license. + pub fn license(&self) -> Result { let license = self.inner.read().unwrap().license.clone()?; // Check the expiration time additionally. diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml index ef4852a665f72..4b38734a70fa9 100644 --- a/src/meta/Cargo.toml +++ b/src/meta/Cargo.toml @@ -43,6 +43,7 @@ jsonbb = { workspace = true } maplit = "1.0.2" memcomparable = { version = "0.2" } mime_guess = "2" +notify = { version = "6", default-features = false, features = ["macos_fsevent"] } num-integer = "0.1" num-traits = "0.2" otlp-embedded = { workspace = true } @@ -105,6 +106,8 @@ expect-test = "1.5" rand = { workspace = true } risingwave_hummock_sdk = { workspace = true, features = ["test"] } risingwave_test_runner = { workspace = true } +tempfile = "3" +tracing-subscriber = "0.3" [features] test = [] diff --git a/src/meta/node/src/lib.rs b/src/meta/node/src/lib.rs index 9090d29a09fa6..1a8e8cf432add 100644 --- a/src/meta/node/src/lib.rs +++ b/src/meta/node/src/lib.rs @@ -17,6 +17,7 @@ mod server; +use std::path::PathBuf; use std::time::Duration; use clap::Parser; @@ -192,6 +193,10 @@ pub struct MetaNodeOpts { #[override_opts(path = system.license_key)] pub license_key: Option, + /// The path of the license key file to be watched and hot-reloaded. + #[clap(long, env = "RW_LICENSE_KEY_PATH")] + pub license_key_file: Option, + /// 128-bit AES key for secret store in HEX format. #[educe(Debug(ignore))] // TODO: use newtype to redact debug impl #[clap(long, hide = true, env = "RW_SECRET_STORE_PRIVATE_KEY_HEX")] @@ -465,6 +470,7 @@ pub fn start( .meta .developer .actor_cnt_per_worker_parallelism_soft_limit, + license_key_path: opts.license_key_file, }, config.system.into_init_system_params(), Default::default(), diff --git a/src/meta/node/src/server.rs b/src/meta/node/src/server.rs index 6a5914a942021..1d690cb176464 100644 --- a/src/meta/node/src/server.rs +++ b/src/meta/node/src/server.rs @@ -398,6 +398,7 @@ pub async fn start_service_as_election_leader( meta_store_impl, ) .await?; + let _ = env.may_start_watch_license_key_file()?; let system_params_reader = env.system_params_reader().await; let data_directory = system_params_reader.data_directory(); diff --git a/src/meta/src/manager/env.rs b/src/meta/src/manager/env.rs index 2bce7b5aeaa32..49c5b82d81108 100644 --- a/src/meta/src/manager/env.rs +++ b/src/meta/src/manager/env.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::ops::Deref; +use std::path::PathBuf; use std::sync::Arc; use risingwave_common::config::{ @@ -296,6 +297,8 @@ pub struct MetaOpts { // Cluster limits pub actor_cnt_per_worker_parallelism_hard_limit: usize, pub actor_cnt_per_worker_parallelism_soft_limit: usize, + + pub license_key_path: Option, } impl MetaOpts { @@ -362,6 +365,7 @@ impl MetaOpts { table_info_statistic_history_times: 240, actor_cnt_per_worker_parallelism_hard_limit: usize::MAX, actor_cnt_per_worker_parallelism_soft_limit: usize::MAX, + license_key_path: None, } } } @@ -523,6 +527,7 @@ impl MetaSrvEnv { } } }; + Ok(env) } diff --git a/src/meta/src/manager/license.rs b/src/meta/src/manager/license.rs new file mode 100644 index 0000000000000..f71f167a0dc5f --- /dev/null +++ b/src/meta/src/manager/license.rs @@ -0,0 +1,164 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::Context; +use notify::Watcher; +use risingwave_common::system_param::LICENSE_KEY_KEY; +use thiserror_ext::AsReport; +use tokio::sync::watch; +use tokio::task::JoinHandle; + +use super::{MetaSrvEnv, SystemParamsManagerImpl}; +use crate::MetaResult; + +impl MetaSrvEnv { + /// Spawn background tasks to watch the license key file and update the system parameter, + /// if configured. + pub fn may_start_watch_license_key_file(&self) -> MetaResult>> { + let Some(path) = self.opts.license_key_path.as_ref() else { + return Ok(None); + }; + + let (changed_tx, mut changed_rx) = watch::channel(()); + // Send an initial event to trigger the initial load. + changed_tx.send(()).unwrap(); + + let mut watcher = + notify::recommended_watcher(move |event: Result| { + if let Err(e) = event { + tracing::warn!( + error = %e.as_report(), + "error occurred while watching license key file" + ); + return; + } + // We don't check the event type but always notify the updater for simplicity. + let _ = changed_tx.send(()); + }) + .context("failed to create license key file watcher")?; + + // This will spawn a new thread to watch the file, so no need to be concerned about blocking. + watcher + .watch(path, notify::RecursiveMode::NonRecursive) + .context("failed to watch license key file")?; + + let updater = { + let mgr = self.system_params_manager_impl_ref(); + let path = path.to_path_buf(); + async move { + // Let the watcher live until the end of the updater to prevent dropping (then stopping). + let _watcher = watcher; + + // Read the file content and set the system parameter every time the file changes. + // Note that `changed()` will immediately resolves on the very first call, so we + // will do the initialization then. + while changed_rx.changed().await.is_ok() { + tracing::info!(path = %path.display(), "license key file changed, reloading..."); + + let content = match tokio::fs::read_to_string(&path).await { + Ok(v) => v, + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e.as_report(), + "failed to read license key file" + ); + continue; + } + }; + + // Trim the content and use it as the new license key value. + // + // It's always a `Some`, meaning that an empty license key file here is equivalent to + // `ALTER SYSTEM SET license_key TO ''`, instead of `... TO DEFAULT`. Please note + // the slight difference in behavior of debug build, where the default value of the + // `license_key` system parameter is a test key but not an empty string. + let value = Some(content.trim().to_owned()); + + let result = match &mgr { + SystemParamsManagerImpl::Kv(mgr) => { + mgr.set_param(LICENSE_KEY_KEY, value).await + } + SystemParamsManagerImpl::Sql(mgr) => { + mgr.set_param(LICENSE_KEY_KEY, value).await + } + }; + + if let Err(e) = result { + tracing::error!( + error = %e.as_report(), + "failed to set license key from file" + ); + } + } + } + }; + + let handle = tokio::spawn(updater); + Ok(Some(handle)) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use risingwave_license::{License, LicenseManager, Tier}; + + use super::*; + use crate::manager::MetaOpts; + + // License { + // sub: "rw-test", + // iss: Test, + // tier: Free, <- difference from the default license in debug build + // cpu_core_limit: None, + // exp: 9999999999, + // } + const INITIAL_KEY: &str = + "eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.\ + eyJzdWIiOiJydy10ZXN0IiwidGllciI6ImZyZWUiLCJpc3MiOiJ0ZXN0LnJpc2luZ3dhdmUuY29tIiwiZXhwIjo5OTk5OTk5OTk5fQ.\ + ALC3Kc9LI6u0S-jeMB1YTxg1k8Azxwvc750ihuSZgjA_e1OJC9moxMvpLrHdLZDzCXHjBYi0XJ_1lowmuO_0iPEuPqN5AFpDV1ywmzJvGmMCMtw3A2wuN7hhem9OsWbwe6lzdwrefZLipyo4GZtIkg5ZdwGuHzm33zsM-X5gl_Ns4P6axHKiorNSR6nTAyA6B32YVET_FAM2YJQrXqpwA61wn1XLfarZqpdIQyJ5cgyiC33BFBlUL3lcRXLMLeYe6TjYGeV4K63qARCjM9yeOlsRbbW5ViWeGtR2Yf18pN8ysPXdbaXm_P_IVhl3jCTDJt9ctPh6pUCbkt36FZqO9A"; + + #[cfg(not(madsim))] // `notify` will spawn system threads, which is not allowed in madsim + #[tokio::test] + #[cfg_attr(not(debug_assertions), ignore)] // skip in release build + async fn test_watch_license_key_file() { + tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .init(); + + let key_file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(key_file.path(), INITIAL_KEY).unwrap(); + + let srv = MetaSrvEnv::for_test_opts(MetaOpts { + license_key_path: Some(key_file.path().to_path_buf()), + ..MetaOpts::test(false) + }) + .await; + let _updater_handle = srv.may_start_watch_license_key_file().unwrap().unwrap(); + + // Since we've filled the key file with the initial key, the license should be loaded. + tokio::time::sleep(Duration::from_secs(1)).await; + let license = LicenseManager::get().license().unwrap(); + assert_eq!(license.sub, "rw-test"); + assert_eq!(license.tier, Tier::Free); + + // Update the key file with an empty content, which should reset the license to the default. + std::fs::write(key_file.path(), "").unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + let license = LicenseManager::get().license().unwrap(); + assert_eq!(license, License::default()); + } +} diff --git a/src/meta/src/manager/mod.rs b/src/meta/src/manager/mod.rs index eac1b15c8e699..63a564faa9042 100644 --- a/src/meta/src/manager/mod.rs +++ b/src/meta/src/manager/mod.rs @@ -19,6 +19,7 @@ mod env; pub mod event_log; mod id; mod idle; +mod license; mod metadata; mod notification; mod notification_version;