Merge branch 'main' into test/sqlness-remote-wal

.github/workflows/apidoc.yml

@@ -1,7 +1,7 @@
 on:
   push:
     branches:
-      - develop
+      - main
     paths-ignore:
       - 'docs/**'
       - 'config/**'

.github/workflows/develop.yml

@@ -11,7 +11,6 @@ on:
       - '.gitignore'
   push:
     branches:
-      - develop
       - main
     paths-ignore:
       - 'docs/**'

.github/workflows/docs.yml

@@ -11,7 +11,6 @@ on:
       - '.gitignore'
   push:
     branches:
-      - develop
       - main
     paths:
       - 'docs/**'

.github/workflows/license.yaml

@@ -3,7 +3,7 @@ name: License checker
 on:
   push:
     branches:
-    - develop
+    - main
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
 jobs:

CONTRIBUTING.md

@@ -10,7 +10,7 @@ Follow our [README](https://github.com/GreptimeTeam/greptimedb#readme) to get th
 
 It can feel intimidating to contribute to a complex project, but it can also be exciting and fun. These general notes will help everyone participate in this communal activity.
 
-- Follow the [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/develop/CODE_OF_CONDUCT.md)
+- Follow the [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/main/CODE_OF_CONDUCT.md)
 - Small changes make huge differences. We will happily accept a PR making a single character change if it helps move forward. Don't wait to have everything working.
 - Check the closed issues before opening your issue.
 - Try to follow the existing style of the code.
@@ -26,7 +26,7 @@ Pull requests are great, but we accept all kinds of other help if you like. Such
 
 ## Code of Conduct
 
-Also, there are things that we are not looking for because they don't match the goals of the product or benefit the community. Please read [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/develop/CODE_OF_CONDUCT.md); we hope everyone can keep good manners and become an honored member.
+Also, there are things that we are not looking for because they don't match the goals of the product or benefit the community. Please read [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/main/CODE_OF_CONDUCT.md); we hope everyone can keep good manners and become an honored member.
 
 ## License
 

README.md

@@ -1,8 +1,8 @@
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: light)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding.png">
-    <source media="(prefers-color-scheme: dark)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding-dark.png">
-    <img alt="GreptimeDB Logo" src="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding.png" width="400px">
+    <source media="(prefers-color-scheme: light)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@main/docs/logo-text-padding.png">
+    <source media="(prefers-color-scheme: dark)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@main/docs/logo-text-padding-dark.png">
+    <img alt="GreptimeDB Logo" src="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@main/docs/logo-text-padding.png" width="400px">
   </picture>
 </p>
 
@@ -12,11 +12,11 @@
 </h3>
 
 <p align="center">
-    <a href="https://codecov.io/gh/GrepTimeTeam/greptimedb"><img src="https://codecov.io/gh/GrepTimeTeam/greptimedb/branch/develop/graph/badge.svg?token=FITFDI3J3C"></img></a>
+    <a href="https://codecov.io/gh/GrepTimeTeam/greptimedb"><img src="https://codecov.io/gh/GrepTimeTeam/greptimedb/branch/main/graph/badge.svg?token=FITFDI3J3C"></img></a>
     &nbsp;
     <a href="https://github.com/GreptimeTeam/greptimedb/actions/workflows/develop.yml"><img src="https://github.com/GreptimeTeam/greptimedb/actions/workflows/develop.yml/badge.svg" alt="CI"></img></a>
     &nbsp;
-    <a href="https://github.com/greptimeTeam/greptimedb/blob/develop/LICENSE"><img src="https://img.shields.io/github/license/greptimeTeam/greptimedb"></a>
+    <a href="https://github.com/greptimeTeam/greptimedb/blob/main/LICENSE"><img src="https://img.shields.io/github/license/greptimeTeam/greptimedb"></a>
 </p>
 
 <p align="center">
@@ -168,7 +168,7 @@ In addition, you may:
 GreptimeDB uses the [Apache 2.0 license][1] to strike a balance between
 open contributions and allowing you to use the software however you want.
 
-[1]: <https://github.com/greptimeTeam/greptimedb/blob/develop/LICENSE>
+[1]: <https://github.com/greptimeTeam/greptimedb/blob/main/LICENSE>
 
 ## Contributing
 

src/catalog/src/information_schema.rs

@@ -51,6 +51,11 @@ lazy_static! {
         COLUMN_PRIVILEGES,
         COLUMN_STATISTICS,
         BUILD_INFO,
+        CHARACTER_SETS,
+        COLLATIONS,
+        COLLATION_CHARACTER_SET_APPLICABILITY,
+        CHECK_CONSTRAINTS,
+        EVENTS,
     ];
 }
 
@@ -125,7 +130,7 @@ impl InformationSchemaProvider {
 
         // Add memory tables
         for name in MEMORY_TABLES.iter() {
-            tables.insert((*name).to_string(), self.build_table(name).unwrap());
+            tables.insert((*name).to_string(), self.build_table(name).expect(name));
         }
 
         self.tables = tables;
@@ -156,6 +161,13 @@ impl InformationSchemaProvider {
             COLUMN_PRIVILEGES => setup_memory_table!(COLUMN_PRIVILEGES),
             COLUMN_STATISTICS => setup_memory_table!(COLUMN_STATISTICS),
             BUILD_INFO => setup_memory_table!(BUILD_INFO),
+            CHARACTER_SETS => setup_memory_table!(CHARACTER_SETS),
+            COLLATIONS => setup_memory_table!(COLLATIONS),
+            COLLATION_CHARACTER_SET_APPLICABILITY => {
+                setup_memory_table!(COLLATION_CHARACTER_SET_APPLICABILITY)
+            }
+            CHECK_CONSTRAINTS => setup_memory_table!(CHECK_CONSTRAINTS),
+            EVENTS => setup_memory_table!(EVENTS),
             _ => None,
         }
     }

src/catalog/src/information_schema/memory_table/tables.rs

@@ -17,7 +17,7 @@ use std::sync::Arc;
 use common_catalog::consts::MITO_ENGINE;
 use datatypes::prelude::{ConcreteDataType, VectorRef};
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
-use datatypes::vectors::StringVector;
+use datatypes::vectors::{Int64Vector, StringVector};
 
 use crate::information_schema::table_names::*;
 
@@ -97,6 +97,92 @@ pub fn get_schema_columns(table_name: &str) -> (SchemaRef, Vec<VectorRef>) {
             ],
         ),
 
+        CHARACTER_SETS => (
+            vec![
+                string_column("CHARACTER_SET_NAME"),
+                string_column("DEFAULT_COLLATE_NAME"),
+                string_column("DESCRIPTION"),
+                bigint_column("MAXLEN"),
+            ],
+            vec![
+                Arc::new(StringVector::from(vec!["utf8"])),
+                Arc::new(StringVector::from(vec!["utf8_bin"])),
+                Arc::new(StringVector::from(vec!["UTF-8 Unicode"])),
+                Arc::new(Int64Vector::from_slice([4])),
+            ],
+        ),
+
+        COLLATIONS => (
+            vec![
+                string_column("COLLATION_NAME"),
+                string_column("CHARACTER_SET_NAME"),
+                bigint_column("ID"),
+                string_column("IS_DEFAULT"),
+                string_column("IS_COMPILED"),
+                bigint_column("SORTLEN"),
+            ],
+            vec![
+                Arc::new(StringVector::from(vec!["utf8_bin"])),
+                Arc::new(StringVector::from(vec!["utf8"])),
+                Arc::new(Int64Vector::from_slice([1])),
+                Arc::new(StringVector::from(vec!["Yes"])),
+                Arc::new(StringVector::from(vec!["Yes"])),
+                Arc::new(Int64Vector::from_slice([1])),
+            ],
+        ),
+
+        COLLATION_CHARACTER_SET_APPLICABILITY => (
+            vec![
+                string_column("COLLATION_NAME"),
+                string_column("CHARACTER_SET_NAME"),
+            ],
+            vec![
+                Arc::new(StringVector::from(vec!["utf8_bin"])),
+                Arc::new(StringVector::from(vec!["utf8"])),
+            ],
+        ),
+
+        CHECK_CONSTRAINTS => (
+            string_columns(&[
+                "CONSTRAINT_CATALOG",
+                "CONSTRAINT_SCHEMA",
+                "CONSTRAINT_NAME",
+                "CHECK_CLAUSE",
+            ]),
+            // Not support check constraints yet
+            vec![],
+        ),
+
+        EVENTS => (
+            vec![
+                string_column("EVENT_CATALOG"),
+                string_column("EVENT_SCHEMA"),
+                string_column("EVENT_NAME"),
+                string_column("DEFINER"),
+                string_column("TIME_ZONE"),
+                string_column("EVENT_BODY"),
+                string_column("EVENT_DEFINITION"),
+                string_column("EVENT_TYPE"),
+                datetime_column("EXECUTE_AT"),
+                bigint_column("INTERVAL_VALUE"),
+                string_column("INTERVAL_FIELD"),
+                string_column("SQL_MODE"),
+                datetime_column("STARTS"),
+                datetime_column("ENDS"),
+                string_column("STATUS"),
+                string_column("ON_COMPLETION"),
+                datetime_column("CREATED"),
+                datetime_column("LAST_ALTERED"),
+                datetime_column("LAST_EXECUTED"),
+                string_column("EVENT_COMMENT"),
+                bigint_column("ORIGINATOR"),
+                string_column("CHARACTER_SET_CLIENT"),
+                string_column("COLLATION_CONNECTION"),
+                string_column("DATABASE_COLLATION"),
+            ],
+            vec![],
+        ),
+
         _ => unreachable!("Unknown table in information_schema: {}", table_name),
     };
 
@@ -115,6 +201,22 @@ fn string_column(name: &str) -> ColumnSchema {
     )
 }
 
+fn bigint_column(name: &str) -> ColumnSchema {
+    ColumnSchema::new(
+        str::to_lowercase(name),
+        ConcreteDataType::int64_datatype(),
+        false,
+    )
+}
+
+fn datetime_column(name: &str) -> ColumnSchema {
+    ColumnSchema::new(
+        str::to_lowercase(name),
+        ConcreteDataType::datetime_datatype(),
+        false,
+    )
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/catalog/src/information_schema/table_names.rs

@@ -20,3 +20,8 @@ pub const ENGINES: &str = "engines";
 pub const COLUMN_PRIVILEGES: &str = "column_privileges";
 pub const COLUMN_STATISTICS: &str = "column_statistics";
 pub const BUILD_INFO: &str = "build_info";
+pub const CHARACTER_SETS: &str = "character_sets";
+pub const COLLATIONS: &str = "collations";
+pub const COLLATION_CHARACTER_SET_APPLICABILITY: &str = "collation_character_set_applicability";
+pub const CHECK_CONSTRAINTS: &str = "check_constraints";
+pub const EVENTS: &str = "events";

src/client/src/stream_insert.rs

@@ -39,7 +39,7 @@ use crate::from_grpc_response;
 /// ```
 ///
 /// If you want to see a concrete usage example, please see
-/// [stream_inserter.rs](https://github.com/GreptimeTeam/greptimedb/blob/develop/src/client/examples/stream_ingest.rs).
+/// [stream_inserter.rs](https://github.com/GreptimeTeam/greptimedb/blob/main/src/client/examples/stream_ingest.rs).
 pub struct StreamInserter {
     sender: mpsc::Sender<GreptimeRequest>,
 

src/common/catalog/src/consts.rs

@@ -44,6 +44,16 @@ pub const INFORMATION_SCHEMA_COLUMN_PRIVILEGES_TABLE_ID: u32 = 6;
 pub const INFORMATION_SCHEMA_COLUMN_STATISTICS_TABLE_ID: u32 = 7;
 /// id for information_schema.build_info
 pub const INFORMATION_SCHEMA_BUILD_INFO_TABLE_ID: u32 = 8;
+/// id for information_schema.CHARACTER_SETS
+pub const INFORMATION_SCHEMA_CHARACTER_SETS_TABLE_ID: u32 = 9;
+/// id for information_schema.COLLATIONS
+pub const INFORMATION_SCHEMA_COLLATIONS_TABLE_ID: u32 = 10;
+/// id for information_schema.COLLATIONS
+pub const INFORMATION_SCHEMA_COLLATION_CHARACTER_SET_APPLICABILITY_TABLE_ID: u32 = 11;
+/// id for information_schema.CHECK_CONSTRAINTS
+pub const INFORMATION_SCHEMA_CHECK_CONSTRAINTS_TABLE_ID: u32 = 12;
+/// id for information_schema.EVENTS
+pub const INFORMATION_SCHEMA_EVENTS_TABLE_ID: u32 = 13;
 /// ----- End of information_schema tables -----
 
 pub const MITO_ENGINE: &str = "mito";

src/common/meta/src/wal/kafka/topic_manager.rs

@@ -135,7 +135,7 @@ impl TopicManager {
             .iter()
             .map(|i| async {
                 self.try_create_topic(&topics[*i], &control_client).await?;
-                self.try_create_noop_record(&topics[*i], &client).await?;
+                self.try_append_noop_record(&topics[*i], &client).await?;
                 Ok(())
             })
             .collect::<Vec<_>>();
@@ -154,7 +154,7 @@ impl TopicManager {
             .collect()
     }
 
-    async fn try_create_noop_record(&self, topic: &Topic, client: &Client) -> Result<()> {
+    async fn try_append_noop_record(&self, topic: &Topic, client: &Client) -> Result<()> {
         let partition_client = client
             .partition_client(topic, DEFAULT_PARTITION, UnknownTopicHandling::Retry)
             .await

src/index/src/inverted_index/format.rs

@@ -48,7 +48,7 @@
 //! More detailed information regarding the encoding of the inverted indices can be found in the [RFC].
 //!
 //! [`InvertedIndexMetas`]: https://github.com/GreptimeTeam/greptime-proto/blob/2aaee38de81047537dfa42af9df63bcfb866e06c/proto/greptime/v1/index/inverted_index.proto#L32-L64
-//! [RFC]: https://github.com/GreptimeTeam/greptimedb/blob/develop/docs/rfcs/2023-11-03-inverted-index.md
+//! [RFC]: https://github.com/GreptimeTeam/greptimedb/blob/main/docs/rfcs/2023-11-03-inverted-index.md
 
 pub mod reader;
 pub mod writer;

src/log-store/src/kafka/log_store.rs

@@ -186,6 +186,10 @@ impl LogStore for KafkaLogStore {
                     record_offset, ns_clone, high_watermark
                 );
 
+                // Ignores the noop record.
+                if record.record.value.is_none() {
+                    continue;
+                }
                 let entries = decode_from_record(record.record)?;
 
                 // Filters entries by region id.

src/meta-srv/src/cluster.rs

@@ -79,6 +79,17 @@ impl MetaPeerClient {
         to_stat_kv_map(kvs)
     }
 
+    // Get kv information from the leader's in_mem kv store.
+    pub async fn get(&self, key: Vec<u8>) -> Result<Option<KeyValue>> {
+        let mut kvs = self.range(key, vec![], false).await?;
+        Ok(if kvs.is_empty() {
+            None
+        } else {
+            debug_assert_eq!(kvs.len(), 1);
+            Some(kvs.remove(0))
+        })
+    }
+
     // Range kv information from the leader's in_mem kv store
     pub async fn range(
         &self,
@@ -228,7 +239,7 @@ impl MetaPeerClient {
 
     // Check if the meta node is a leader node.
     // Note: when self.election is None, we also consider the meta node is leader
-    fn is_leader(&self) -> bool {
+    pub(crate) fn is_leader(&self) -> bool {
         self.election
             .as_ref()
             .map(|election| election.is_leader())

src/meta-srv/src/error.rs

@@ -32,6 +32,9 @@ use crate::pubsub::Message;
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
+    #[snafu(display("The target peer is unavailable temporally: {}", peer_id))]
+    PeerUnavailable { location: Location, peer_id: u64 },
+
     #[snafu(display("Another migration procedure is running for region: {}", region_id))]
     MigrationRunning {
         location: Location,
@@ -650,7 +653,8 @@ impl ErrorExt for Error {
             | Error::Join { .. }
             | Error::WeightArray { .. }
             | Error::NotSetWeightArray { .. }
-            | Error::Unsupported { .. } => StatusCode::Internal,
+            | Error::Unsupported { .. }
+            | Error::PeerUnavailable { .. } => StatusCode::Internal,
             Error::TableAlreadyExists { .. } => StatusCode::TableAlreadyExists,
             Error::EmptyKey { .. }
             | Error::MissingRequiredParameter { .. }

src/meta-srv/src/handler.rs

@@ -229,7 +229,7 @@ impl HeartbeatHandlerGroup {
         let _ = self.pushers.insert(key.to_string(), pusher).await;
     }
 
-    pub async fn unregister(&self, key: impl AsRef<str>) -> Option<Pusher> {
+    pub async fn deregister(&self, key: impl AsRef<str>) -> Option<Pusher> {
         let key = key.as_ref();
         METRIC_META_HEARTBEAT_CONNECTION_NUM.dec();
         info!("Pusher unregister: {}", key);