feat(meta): define model V2 for hummock metadata (#12674)

src/common/src/system_param/mod.rs

@@ -51,8 +51,8 @@ macro_rules! for_all_params {
             { bloom_false_positive, f64, Some(0.001_f64), false },
             { state_store, String, None, false },
             { data_directory, String, None, false },
-            { backup_storage_url, String, Some("memory".to_string()), false },
-            { backup_storage_directory, String, Some("backup".to_string()), false },
+            { backup_storage_url, String, Some("memory".to_string()), true },
+            { backup_storage_directory, String, Some("backup".to_string()), true },
             { max_concurrent_creating_streaming_jobs, u32, Some(1_u32), true },
             { pause_on_next_bootstrap, bool, Some(false), true },
         }

src/meta/src/backup_restore/meta_snapshot_builder.rs

@@ -191,7 +191,6 @@ mod tests {
             let v_ = v.clone();
             async move { v_ }
         };
-        hummock_version.insert(&meta_store).await.unwrap();
         let err = builder
             .build(1, get_ckpt_builder(&hummock_version))
             .await

src/meta/src/backup_restore/restore.rs

@@ -152,7 +152,6 @@ async fn restore_default_cf<S: MetaStore>(
 
 async fn restore_metadata<S: MetaStore>(meta_store: S, snapshot: MetaSnapshot) -> BackupResult<()> {
     restore_default_cf(&meta_store, &snapshot).await?;
-    restore_metadata_model(&meta_store, &[snapshot.metadata.hummock_version]).await?;
     restore_metadata_model(&meta_store, &[snapshot.metadata.version_stats]).await?;
     restore_metadata_model(
         &meta_store,
@@ -290,7 +289,7 @@ mod tests {
     use itertools::Itertools;
     use risingwave_backup::meta_snapshot::{ClusterMetadata, MetaSnapshot};
     use risingwave_common::config::{MetaBackend, SystemConfig};
-    use risingwave_pb::hummock::HummockVersion;
+    use risingwave_pb::hummock::{HummockVersion, HummockVersionStats};
     use risingwave_pb::meta::SystemParams;
 
     use crate::backup_restore::restore::restore_impl;
@@ -331,8 +330,8 @@ mod tests {
         let backup_store = get_backup_store(opts.clone()).await.unwrap();
         let nonempty_meta_store = get_meta_store(opts.clone()).await.unwrap();
         dispatch_meta_store!(nonempty_meta_store.clone(), store, {
-            let hummock_version = HummockVersion::default();
-            hummock_version.insert(&store).await.unwrap();
+            let stats = HummockVersionStats::default();
+            stats.insert(&store).await.unwrap();
         });
         let empty_meta_store = get_meta_store(opts.clone()).await.unwrap();
         let system_param = get_system_params();
@@ -377,13 +376,6 @@ mod tests {
         .unwrap();
 
         dispatch_meta_store!(empty_meta_store, store, {
-            let restored_hummock_version = HummockVersion::list(&store)
-                .await
-                .unwrap()
-                .into_iter()
-                .next()
-                .unwrap();
-            assert_eq!(restored_hummock_version.id, 123);
             let restored_system_param = SystemParams::get(&store).await.unwrap().unwrap();
             assert_eq!(restored_system_param, system_param);
         });
@@ -547,7 +539,6 @@ mod tests {
         .unwrap();
 
         dispatch_meta_store!(empty_meta_store, store, {
-            assert!(HummockVersion::list(&store).await.unwrap().is_empty());
             assert!(SystemParams::get(&store).await.unwrap().is_none());
         });
     }

src/meta/src/hummock/manager/mod.rs

@@ -437,29 +437,16 @@ impl HummockManager {
                 .collect();
 
         let mut redo_state = if self.need_init().await? {
-            // For backward compatibility, try to read checkpoint from meta store.
-            let versions = HummockVersion::list(self.env.meta_store()).await?;
-            let checkpoint_version = if !versions.is_empty() {
-                let checkpoint = versions.into_iter().next().unwrap();
-                tracing::warn!(
-                    "read hummock version checkpoint from meta store: {:#?}",
-                    checkpoint
-                );
-                checkpoint
-            } else {
-                // As no record found in stores, create a initial version.
-                let default_compaction_config = self
-                    .compaction_group_manager
-                    .read()
-                    .await
-                    .default_compaction_config();
-                let checkpoint = create_init_version(default_compaction_config);
-                tracing::info!("init hummock version checkpoint");
-                HummockVersionStats::default()
-                    .insert(self.env.meta_store())
-                    .await?;
-                checkpoint
-            };
+            let default_compaction_config = self
+                .compaction_group_manager
+                .read()
+                .await
+                .default_compaction_config();
+            let checkpoint_version = create_init_version(default_compaction_config);
+            tracing::info!("init hummock version checkpoint");
+            HummockVersionStats::default()
+                .insert(self.env.meta_store())
+                .await?;
             versioning_guard.checkpoint = HummockVersionCheckpoint {
                 version: Some(checkpoint_version.clone()),
                 stale_objects: Default::default(),

src/meta/src/hummock/model/mod.rs

@@ -17,20 +17,17 @@ mod compaction_group_config;
 mod compaction_status;
 mod pinned_snapshot;
 mod pinned_version;
-mod version;
 mod version_delta;
 mod version_stats;
 
 pub use compaction_group_config::CompactionGroup;
 pub use compaction_status::*;
 pub use pinned_snapshot::*;
 pub use pinned_version::*;
-pub use version::*;
 pub use version_delta::*;
 
 /// Column family names for hummock.
 /// Deprecated `cf_name` should be reserved for backward compatibility.
-const HUMMOCK_VERSION_CF_NAME: &str = "cf/hummock_0";
 const HUMMOCK_VERSION_DELTA_CF_NAME: &str = "cf/hummock_1";
 const HUMMOCK_PINNED_VERSION_CF_NAME: &str = "cf/hummock_2";
 const HUMMOCK_PINNED_SNAPSHOT_CF_NAME: &str = "cf/hummock_3";

src/meta/src/model_v2/compaction_config.rs

@@ -0,0 +1,29 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "compaction_config")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub compaction_group_id: i64,
+    #[sea_orm(column_type = "JsonBinary", nullable)]
+    pub config: Option<Json>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}

src/meta/src/model_v2/compaction_status.rs

@@ -0,0 +1,29 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "compaction_status")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub compaction_group_id: i64,
+    #[sea_orm(column_type = "JsonBinary", nullable)]
+    pub status: Option<Json>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}

src/meta/src/model_v2/compaction_task.rs

@@ -0,0 +1,30 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "compaction_task")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub id: i64,
+    #[sea_orm(column_type = "JsonBinary")]
+    pub task: Json,
+    pub context_id: i32,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}

src/meta/src/model_v2/hummock_pinned_snapshot.rs

@@ -0,0 +1,28 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "hummock_pinned_snapshot")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub context_id: i32,
+    pub min_pinned_snapshot: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}

src/meta/src/model_v2/hummock_pinned_version.rs

@@ -0,0 +1,28 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "hummock_pinned_version")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub context_id: i32,
+    pub min_pinned_id: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}

src/meta/src/model_v2/hummock_version_delta.rs

@@ -0,0 +1,35 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "hummock_version_delta")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub id: i64,
+    pub prev_id: i64,
+    #[sea_orm(column_type = "JsonBinary", nullable)]
+    pub group_deltas: Option<Json>,
+    pub max_committed_epoch: i64,
+    pub safe_epoch: i64,
+    pub trivial_move: bool,
+    #[sea_orm(column_type = "JsonBinary", nullable)]
+    pub gc_object_ids: Option<Json>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}

src/meta/src/model_v2/hummock_version_stats.rs

@@ -0,0 +1,29 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "hummock_version_stats")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub id: i64,
+    #[sea_orm(column_type = "JsonBinary")]
+    pub stats: Json,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}