feat(flow): render source/sink (#3903)

* feat(flow): render src/sink

* chore: add empty impl

* chore: typos

* refactor: according to review(WIP)

* refactor: reexport df_sbustrait&use to_sub_plan

* fix: add implict location to error enum

* fix: error handling unwrap query_ctx

Cargo.toml

@@ -206,6 +206,7 @@ common-wal = { path = "src/common/wal" }
 datanode = { path = "src/datanode" }
 datatypes = { path = "src/datatypes" }
 file-engine = { path = "src/file-engine" }
+flow = { path = "src/flow" }
 frontend = { path = "src/frontend" }
 index = { path = "src/index" }
 log-store = { path = "src/log-store" }

src/common/substrait/src/df_substrait.rs

@@ -69,14 +69,21 @@ impl SubstraitPlan for DFLogicalSubstraitConvertor {
 
     fn encode(&self, plan: &Self::Plan) -> Result<Bytes, Self::Error> {
         let mut buf = BytesMut::new();
+
+        let substrait_plan = self.to_sub_plan(plan)?;
+        substrait_plan.encode(&mut buf).context(EncodeRelSnafu)?;
+
+        Ok(buf.freeze())
+    }
+}
+
+impl DFLogicalSubstraitConvertor {
+    pub fn to_sub_plan(&self, plan: &LogicalPlan) -> Result<Box<Plan>, Error> {
         let session_state =
             SessionState::new_with_config_rt(SessionConfig::new(), Arc::new(RuntimeEnv::default()))
                 .with_serializer_registry(Arc::new(ExtensionSerializer));
         let context = SessionContext::new_with_state(session_state);
 
-        let substrait_plan = to_substrait_plan(plan, &context).context(EncodeDfPlanSnafu)?;
-        substrait_plan.encode(&mut buf).context(EncodeRelSnafu)?;
-
-        Ok(buf.freeze())
+        to_substrait_plan(plan, &context).context(EncodeDfPlanSnafu)
     }
 }

src/common/substrait/src/lib.rs

@@ -23,11 +23,14 @@ use async_trait::async_trait;
 use bytes::{Buf, Bytes};
 use datafusion::catalog::CatalogProviderList;
 use datafusion::execution::context::SessionState;
+/// Re-export the Substrait module of datafusion,
+/// note this is a different version of the `substrait_proto` crate
+pub use datafusion_substrait::substrait as substrait_proto_df;
+pub use datafusion_substrait::{logical_plan as df_logical_plan, variation_const};
 use session::context::QueryContextRef;
 pub use substrait_proto;
 
 pub use crate::df_substrait::DFLogicalSubstraitConvertor;
-
 #[async_trait]
 pub trait SubstraitPlan {
     type Error: std::error::Error;

src/flow/Cargo.toml

@@ -9,27 +9,41 @@ workspace = true
 
 [dependencies]
 api.workspace = true
+catalog.workspace = true
+common-base.workspace = true
 common-decimal.workspace = true
 common-error.workspace = true
+common-frontend.workspace = true
 common-macro.workspace = true
+common-runtime.workspace = true
 common-telemetry.workspace = true
 common-time.workspace = true
 datafusion-common.workspace = true
 datafusion-expr.workspace = true
-datafusion-substrait.workspace = true
 datatypes.workspace = true
 enum_dispatch = "0.3"
+futures = "0.3"
 # This fork is simply for keeping our dependency in our org, and pin the version
 # it is the same with upstream repo
+async-trait.workspace = true
+common-meta.workspace = true
+greptime-proto.workspace = true
 hydroflow = { git = "https://github.com/GreptimeTeam/hydroflow.git", branch = "main" }
 itertools.workspace = true
+minstant = "0.1.7"
+nom = "7.1.3"
 num-traits = "0.2"
+prost.workspace = true
+query.workspace = true
 serde.workspace = true
 servers.workspace = true
+session.workspace = true
 smallvec.workspace = true
 snafu.workspace = true
+store-api.workspace = true
 strum.workspace = true
 substrait.workspace = true
+table.workspace = true
 tokio.workspace = true
 tonic.workspace = true
 

src/flow/src/adapter.rs

@@ -16,3 +16,6 @@
 //! and communicating with other parts of the database
 
 pub(crate) mod error;
+pub(crate) mod node_context;
+
+pub(crate) use node_context::FlownodeContext;

src/flow/src/adapter/error.rs

@@ -16,12 +16,11 @@
 
 use std::any::Any;
 
+use common_error::ext::BoxedError;
 use common_macro::stack_trace_debug;
 use common_telemetry::common_error::ext::ErrorExt;
 use common_telemetry::common_error::status_code::StatusCode;
-use datatypes::data_type::ConcreteDataType;
 use datatypes::value::Value;
-use serde::{Deserialize, Serialize};
 use servers::define_into_tonic_status;
 use snafu::{Location, Snafu};
 
@@ -32,6 +31,16 @@ use crate::expr::EvalError;
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
+    #[snafu(display("External error"))]
+    External {
+        source: BoxedError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Internal error"))]
+    Internal { location: Location, reason: String },
+
     /// TODO(discord9): add detailed location of column
     #[snafu(display("Failed to eval stream"))]
     Eval {
@@ -47,6 +56,14 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Table not found: {msg}, meta error: {source}"))]
+    TableNotFoundMeta {
+        source: common_meta::error::Error,
+        msg: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Table already exist: {name}"))]
     TableAlreadyExist {
         name: String,
@@ -62,6 +79,27 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Invalid query plan: {source}"))]
+    InvalidQueryPlan {
+        source: query::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Invalid query: prost can't decode substrait plan: {inner}"))]
+    InvalidQueryProst {
+        inner: api::DecodeError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Invalid query, can't transform to substrait: {source}"))]
+    InvalidQuerySubstrait {
+        source: substrait::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Invalid query: {reason}"))]
     InvalidQuery {
         reason: String,
@@ -112,6 +150,13 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Unexpected: {reason}"))]
+    Unexpected {
+        reason: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 /// Result type for flow module
@@ -124,14 +169,21 @@ impl ErrorExt for Error {
                 StatusCode::Internal
             }
             &Self::TableAlreadyExist { .. } => StatusCode::TableAlreadyExists,
-            Self::TableNotFound { .. } => StatusCode::TableNotFound,
-            &Self::InvalidQuery { .. } | &Self::Plan { .. } | &Self::Datatypes { .. } => {
-                StatusCode::PlanQuery
+            Self::TableNotFound { .. } | Self::TableNotFoundMeta { .. } => {
+                StatusCode::TableNotFound
             }
-            Self::NoProtoType { .. } => StatusCode::Unexpected,
+            Self::InvalidQueryPlan { .. }
+            | Self::InvalidQuerySubstrait { .. }
+            | Self::InvalidQueryProst { .. }
+            | &Self::InvalidQuery { .. }
+            | &Self::Plan { .. }
+            | &Self::Datatypes { .. } => StatusCode::PlanQuery,
+            Self::NoProtoType { .. } | Self::Unexpected { .. } => StatusCode::Unexpected,
             &Self::NotImplemented { .. } | Self::UnsupportedTemporalFilter { .. } => {
                 StatusCode::Unsupported
             }
+            &Self::External { .. } => StatusCode::Unknown,
+            Self::Internal { .. } => StatusCode::Internal,
         }
     }