feat(meta): define model V2 for hummock metadata

risingwavelabs · Oct 8, 2023 · ea7b6a8 · ea7b6a8
1 parent 848b0a1
commit ea7b6a8
Show file tree

Hide file tree

Showing 17 changed files with 422 additions and 88 deletions.
diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs
@@ -51,8 +51,8 @@ macro_rules! for_all_params {
             { bloom_false_positive, f64, Some(0.001_f64), false },
             { state_store, String, None, false },
             { data_directory, String, None, false },
-            { backup_storage_url, String, Some("memory".to_string()), false },
-            { backup_storage_directory, String, Some("backup".to_string()), false },
+            { backup_storage_url, String, Some("memory".to_string()), true },
+            { backup_storage_directory, String, Some("backup".to_string()), true },
             { max_concurrent_creating_streaming_jobs, u32, Some(1_u32), true },
             { pause_on_next_bootstrap, bool, Some(false), true },
         }

diff --git a/src/meta/src/backup_restore/meta_snapshot_builder.rs b/src/meta/src/backup_restore/meta_snapshot_builder.rs
@@ -191,7 +191,6 @@ mod tests {
             let v_ = v.clone();
             async move { v_ }
         };
-        hummock_version.insert(&meta_store).await.unwrap();
         let err = builder
             .build(1, get_ckpt_builder(&hummock_version))
             .await

diff --git a/src/meta/src/backup_restore/restore.rs b/src/meta/src/backup_restore/restore.rs
@@ -152,7 +152,6 @@ async fn restore_default_cf<S: MetaStore>(
 
 async fn restore_metadata<S: MetaStore>(meta_store: S, snapshot: MetaSnapshot) -> BackupResult<()> {
     restore_default_cf(&meta_store, &snapshot).await?;
-    restore_metadata_model(&meta_store, &[snapshot.metadata.hummock_version]).await?;
     restore_metadata_model(&meta_store, &[snapshot.metadata.version_stats]).await?;
     restore_metadata_model(
         &meta_store,
@@ -290,7 +289,7 @@ mod tests {
     use itertools::Itertools;
     use risingwave_backup::meta_snapshot::{ClusterMetadata, MetaSnapshot};
     use risingwave_common::config::{MetaBackend, SystemConfig};
-    use risingwave_pb::hummock::HummockVersion;
+    use risingwave_pb::hummock::{HummockVersion, HummockVersionStats};
     use risingwave_pb::meta::SystemParams;
 
     use crate::backup_restore::restore::restore_impl;
@@ -331,8 +330,8 @@ mod tests {
         let backup_store = get_backup_store(opts.clone()).await.unwrap();
         let nonempty_meta_store = get_meta_store(opts.clone()).await.unwrap();
         dispatch_meta_store!(nonempty_meta_store.clone(), store, {
-            let hummock_version = HummockVersion::default();
-            hummock_version.insert(&store).await.unwrap();
+            let stats = HummockVersionStats::default();
+            stats.insert(&store).await.unwrap();
         });
         let empty_meta_store = get_meta_store(opts.clone()).await.unwrap();
         let system_param = get_system_params();
@@ -377,13 +376,6 @@ mod tests {
         .unwrap();
 
         dispatch_meta_store!(empty_meta_store, store, {
-            let restored_hummock_version = HummockVersion::list(&store)
-                .await
-                .unwrap()
-                .into_iter()
-                .next()
-                .unwrap();
-            assert_eq!(restored_hummock_version.id, 123);
             let restored_system_param = SystemParams::get(&store).await.unwrap().unwrap();
             assert_eq!(restored_system_param, system_param);
         });
@@ -547,7 +539,6 @@ mod tests {
         .unwrap();
 
         dispatch_meta_store!(empty_meta_store, store, {
-            assert!(HummockVersion::list(&store).await.unwrap().is_empty());
             assert!(SystemParams::get(&store).await.unwrap().is_none());
         });
     }

diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs
@@ -435,29 +435,16 @@ impl HummockManager {
                 .collect();
 
         let mut redo_state = if self.need_init().await? {
-            // For backward compatibility, try to read checkpoint from meta store.
-            let versions = HummockVersion::list(self.env.meta_store()).await?;
-            let checkpoint_version = if !versions.is_empty() {
-                let checkpoint = versions.into_iter().next().unwrap();
-                tracing::warn!(
-                    "read hummock version checkpoint from meta store: {:#?}",
-                    checkpoint
-                );
-                checkpoint
-            } else {
-                // As no record found in stores, create a initial version.
-                let default_compaction_config = self
-                    .compaction_group_manager
-                    .read()
-                    .await
-                    .default_compaction_config();
-                let checkpoint = create_init_version(default_compaction_config);
-                tracing::info!("init hummock version checkpoint");
-                HummockVersionStats::default()
-                    .insert(self.env.meta_store())
-                    .await?;
-                checkpoint
-            };
+            let default_compaction_config = self
+                .compaction_group_manager
+                .read()
+                .await
+                .default_compaction_config();
+            let checkpoint_version = create_init_version(default_compaction_config);
+            tracing::info!("init hummock version checkpoint");
+            HummockVersionStats::default()
+                .insert(self.env.meta_store())
+                .await?;
             versioning_guard.checkpoint = HummockVersionCheckpoint {
                 version: Some(checkpoint_version.clone()),
                 stale_objects: Default::default(),

diff --git a/src/meta/src/hummock/model/mod.rs b/src/meta/src/hummock/model/mod.rs
@@ -17,20 +17,17 @@ mod compaction_group_config;
 mod compaction_status;
 mod pinned_snapshot;
 mod pinned_version;
-mod version;
 mod version_delta;
 mod version_stats;
 
 pub use compaction_group_config::CompactionGroup;
 pub use compaction_status::*;
 pub use pinned_snapshot::*;
 pub use pinned_version::*;
-pub use version::*;
 pub use version_delta::*;
 
 /// Column family names for hummock.
 /// Deprecated `cf_name` should be reserved for backward compatibility.
-const HUMMOCK_VERSION_CF_NAME: &str = "cf/hummock_0";
 const HUMMOCK_VERSION_DELTA_CF_NAME: &str = "cf/hummock_1";
 const HUMMOCK_PINNED_VERSION_CF_NAME: &str = "cf/hummock_2";
 const HUMMOCK_PINNED_SNAPSHOT_CF_NAME: &str = "cf/hummock_3";

diff --git a/src/meta/src/hummock/model/version.rs b/src/meta/src/hummock/model/version.rs
diff --git a/src/meta/src/model_v2/compaction_config.rs b/src/meta/src/model_v2/compaction_config.rs
@@ -0,0 +1,17 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.3
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "compaction_config")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub compaction_group_id: i64,
+    #[sea_orm(column_type = "JsonBinary", nullable)]
+    pub config: Option<Json>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/src/meta/src/model_v2/compaction_status.rs b/src/meta/src/model_v2/compaction_status.rs
@@ -0,0 +1,17 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.3
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "compaction_status")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub compaction_group_id: i64,
+    #[sea_orm(column_type = "JsonBinary", nullable)]
+    pub status: Option<Json>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/src/meta/src/model_v2/compaction_task.rs b/src/meta/src/model_v2/compaction_task.rs
@@ -0,0 +1,18 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.3
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "compaction_task")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub id: i64,
+    #[sea_orm(column_type = "JsonBinary")]
+    pub task: Json,
+    pub context_id: i32,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/src/meta/src/model_v2/hummock_pinned_snapshot.rs b/src/meta/src/model_v2/hummock_pinned_snapshot.rs
@@ -0,0 +1,16 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.3
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "hummock_pinned_snapshot")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub context_id: i32,
+    pub min_pinned_snapshot: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/src/meta/src/model_v2/hummock_pinned_version.rs b/src/meta/src/model_v2/hummock_pinned_version.rs
@@ -0,0 +1,16 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.3
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "hummock_pinned_version")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub context_id: i32,
+    pub min_pinned_id: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/src/meta/src/model_v2/hummock_version_delta.rs b/src/meta/src/model_v2/hummock_version_delta.rs
@@ -0,0 +1,23 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.3
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "hummock_version_delta")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub id: i64,
+    pub prev_id: i64,
+    #[sea_orm(column_type = "JsonBinary", nullable)]
+    pub group_deltas: Option<Json>,
+    pub max_committed_epoch: i64,
+    pub safe_epoch: i64,
+    pub trivial_move: bool,
+    #[sea_orm(column_type = "JsonBinary", nullable)]
+    pub gc_object_ids: Option<Json>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/src/meta/src/model_v2/hummock_version_stats.rs b/src/meta/src/model_v2/hummock_version_stats.rs
@@ -0,0 +1,17 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.3
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "hummock_version_stats")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub id: i64,
+    #[sea_orm(column_type = "JsonBinary")]
+    pub stats: Json,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/src/meta/src/model_v2/migration/src/lib.rs b/src/meta/src/model_v2/migration/src/lib.rs
@@ -1,12 +1,16 @@
 pub use sea_orm_migration::prelude::*;
 
 mod m20230908_072257_init;
+mod m20231008_020431_hummock;
 
 pub struct Migrator;
 
 #[async_trait::async_trait]
 impl MigratorTrait for Migrator {
     fn migrations() -> Vec<Box<dyn MigrationTrait>> {
-        vec![Box::new(m20230908_072257_init::Migration)]
+        vec![
+            Box::new(m20230908_072257_init::Migration),
+            Box::new(m20231008_020431_hummock::Migration),
+        ]
     }
 }