chore: add description in FieldDefinition for LLM semantics

getdozer · Apr 3, 2024 · d859ca7 · d859ca7
1 parent cb4c2fe
commit d859ca7
Show file tree

Hide file tree

Showing 18 changed files with 48 additions and 0 deletions.
diff --git a/dozer-ingestion/aerospike/src/connector.rs b/dozer-ingestion/aerospike/src/connector.rs
@@ -525,6 +525,7 @@ impl Connector for AerospikeConnector {
                                         },
                                         nullable: name != "PK",
                                         source: Default::default(),
+                                        description: None,
                                     })
                                     .collect(),
                                 primary_index,

diff --git a/dozer-ingestion/ethereum/src/log/helper.rs b/dozer-ingestion/ethereum/src/log/helper.rs
@@ -38,6 +38,7 @@ pub fn get_contract_event_schemas(
                     },
                     nullable: false,
                     source: SourceDefinition::Dynamic,
+                    description: None,
                 });
             }
 
@@ -231,72 +232,84 @@ pub fn get_eth_schema() -> Schema {
                 typ: FieldType::UInt,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "address".to_string(),
                 typ: FieldType::String,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "topics".to_string(),
                 typ: FieldType::String,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "data".to_string(),
                 typ: FieldType::Binary,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "block_hash".to_string(),
                 typ: FieldType::String,
                 nullable: true,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "block_number".to_string(),
                 typ: FieldType::UInt,
                 nullable: true,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "transaction_hash".to_string(),
                 typ: FieldType::String,
                 nullable: true,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "transaction_index".to_string(),
                 typ: FieldType::Int,
                 nullable: true,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "log_index".to_string(),
                 typ: FieldType::Int,
                 nullable: true,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "transaction_log_index".to_string(),
                 typ: FieldType::Int,
                 nullable: true,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "log_type".to_string(),
                 typ: FieldType::String,
                 nullable: true,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "removed".to_string(),
                 typ: FieldType::Boolean,
                 nullable: true,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
         ],
 

diff --git a/dozer-ingestion/ethereum/src/trace/helper.rs b/dozer-ingestion/ethereum/src/trace/helper.rs
@@ -93,48 +93,56 @@ pub fn get_trace_schema() -> Schema {
                 typ: FieldType::String,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "from".to_string(),
                 typ: FieldType::String,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "to".to_string(),
                 typ: FieldType::String,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "value".to_string(),
                 typ: FieldType::UInt,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "gas".to_string(),
                 typ: FieldType::UInt,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "gas_used".to_string(),
                 typ: FieldType::UInt,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "input".to_string(),
                 typ: FieldType::Text,
                 nullable: true,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             FieldDefinition {
                 name: "output".to_string(),
                 typ: FieldType::Text,
                 nullable: true,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
         ],
         primary_index: vec![],

diff --git a/dozer-ingestion/javascript/src/lib.rs b/dozer-ingestion/javascript/src/lib.rs
@@ -67,6 +67,7 @@ impl Connector for JavaScriptConnector {
                     typ: FieldType::Json,
                     nullable: false,
                     source: SourceDefinition::Dynamic,
+                    description: None,
                 }],
                 primary_index: vec![],
             },

diff --git a/dozer-ingestion/kafka/src/debezium/schema.rs b/dozer-ingestion/kafka/src/debezium/schema.rs
@@ -82,6 +82,7 @@ pub fn map_schema(
                                 typ,
                                 nullable: f.optional.map_or(false, |o| o),
                                 source: SourceDefinition::Dynamic,
+                                description: None,
                             })
                         })
                         .collect(),
@@ -196,12 +197,14 @@ mod tests {
                     typ: FieldType::Int,
                     nullable: false,
                     source: SourceDefinition::Dynamic,
+                    description: None,
                 },
                 FieldDefinition {
                     name: "name".to_string(),
                     typ: FieldType::String,
                     nullable: true,
                     source: SourceDefinition::Dynamic,
+                    description: None,
                 },
             ],
             primary_index: vec![0],

diff --git a/dozer-ingestion/kafka/src/debezium/schema_registry.rs b/dozer-ingestion/kafka/src/debezium/schema_registry.rs
@@ -133,6 +133,7 @@ impl SchemaRegistry {
                                 typ,
                                 nullable,
                                 source: SourceDefinition::Dynamic,
+                                description: None,
                             })
                         })
                         .collect();

diff --git a/dozer-ingestion/kafka/src/no_schema_registry_basic.rs b/dozer-ingestion/kafka/src/no_schema_registry_basic.rs
@@ -16,12 +16,14 @@ impl NoSchemaRegistryBasic {
                     typ: FieldType::String,
                     nullable: false,
                     source: SourceDefinition::Dynamic,
+                    description: None,
                 },
                 FieldDefinition {
                     name: "message".to_string(),
                     typ: FieldType::String,
                     nullable: true,
                     source: SourceDefinition::Dynamic,
+                    description: None,
                 },
             ],
             primary_index: vec![0],

diff --git a/dozer-ingestion/kafka/src/schema_registry_basic.rs b/dozer-ingestion/kafka/src/schema_registry_basic.rs
@@ -51,6 +51,7 @@ impl SchemaRegistryBasic {
                     typ,
                     nullable,
                     source: SourceDefinition::Dynamic,
+                    description: None,
                 })
             })
             .collect();

diff --git a/dozer-ingestion/mongodb/src/lib.rs b/dozer-ingestion/mongodb/src/lib.rs
@@ -501,12 +501,14 @@ impl Connector for MongodbConnector {
                                 typ: FieldType::Json,
                                 nullable: false,
                                 source: SourceDefinition::Dynamic,
+                                description: None,
                             },
                             FieldDefinition {
                                 name: "data".to_owned(),
                                 typ: FieldType::Json,
                                 nullable: false,
                                 source: SourceDefinition::Dynamic,
+                                description: None,
                             },
                         ],
                         primary_index: vec![0],

diff --git a/dozer-ingestion/mysql/src/connector.rs b/dozer-ingestion/mysql/src/connector.rs
@@ -177,6 +177,7 @@ impl Connector for MySQLConnector {
                                         typ,
                                         nullable,
                                         source: SourceDefinition::Dynamic,
+                                        description: None,
                                     }
                                 },
                             )

diff --git a/dozer-ingestion/object-store/src/schema_helper.rs b/dozer-ingestion/object-store/src/schema_helper.rs
@@ -52,6 +52,7 @@ pub fn map_schema_to_dozer<'a, I: Iterator<Item = &'a Arc<Field>>>(
                 typ: mapped_field_type,
                 nullable: field.is_nullable(),
                 source: SourceDefinition::Dynamic,
+                description: None,
             })
         })
         .collect()

diff --git a/dozer-ingestion/oracle/src/connector/mapping.rs b/dozer-ingestion/oracle/src/connector/mapping.rs
@@ -215,6 +215,7 @@ pub fn decide_schema(
                     connection: connection.to_string(),
                     name: table_name.clone(),
                 },
+                description: None,
             }),
             Err(err) => return Err(Error::DataType(err.clone())),
         }

diff --git a/dozer-ingestion/postgres/src/helper.rs b/dozer-ingestion/postgres/src/helper.rs
@@ -362,6 +362,7 @@ pub fn convert_column_to_field(column: &Column) -> Result<FieldDefinition, Postg
         typ,
         nullable: true,
         source: SourceDefinition::Dynamic,
+        description: None,
     })
 }
 

diff --git a/dozer-ingestion/snowflake/src/connection/client.rs b/dozer-ingestion/snowflake/src/connection/client.rs
@@ -369,6 +369,7 @@ impl<'env> Client<'env> {
                             typ,
                             nullable: *nullable,
                             source: SourceDefinition::Dynamic,
+                            description: None,
                         });
                     }
                 }

diff --git a/dozer-sink-clickhouse/src/metadata.rs b/dozer-sink-clickhouse/src/metadata.rs
@@ -29,6 +29,7 @@ impl ReplicationMetadata {
                         typ: FieldType::String,
                         nullable: false,
                         source: SourceDefinition::Dynamic,
+                        description: None,
                     },
                     true,
                 )
@@ -38,6 +39,7 @@ impl ReplicationMetadata {
                         typ: FieldType::UInt,
                         nullable: false,
                         source: SourceDefinition::Dynamic,
+                        description: None,
                     },
                     false,
                 )

diff --git a/dozer-sink-oracle/src/lib.rs b/dozer-sink-oracle/src/lib.rs
@@ -516,6 +516,7 @@ impl SinkFactory for OracleSinkFactory {
                 typ: FieldType::UInt,
                 nullable: true,
                 source: dozer_types::types::SourceDefinition::Dynamic,
+                description: None,
             },
             false,
         );
@@ -525,6 +526,7 @@ impl SinkFactory for OracleSinkFactory {
                 typ: FieldType::UInt,
                 nullable: true,
                 source: dozer_types::types::SourceDefinition::Dynamic,
+                description: None,
             },
             false,
         );
@@ -558,6 +560,7 @@ impl SinkFactory for OracleSinkFactory {
                         typ: FieldType::String,
                         nullable: false,
                         source: SourceDefinition::Dynamic,
+                        description: None,
                     },
                     true,
                 )
@@ -567,6 +570,7 @@ impl SinkFactory for OracleSinkFactory {
                         typ: FieldType::UInt,
                         nullable: false,
                         source: SourceDefinition::Dynamic,
+                        description: None,
                     },
                     false,
                 ),
@@ -944,6 +948,7 @@ mod tests {
                 typ: FieldType::UInt,
                 nullable: false,
                 source: SourceDefinition::Dynamic,
+                description: None,
             },
             true,
         );
@@ -976,6 +981,7 @@ mod tests {
             typ: FieldType::String,
             nullable: false,
             source: SourceDefinition::Dynamic,
+            description: None,
         }
     }
 }
diff --git a/dozer-types/src/arrow_types/from_arrow.rs b/dozer-types/src/arrow_types/from_arrow.rs
@@ -189,6 +189,7 @@ fn handle_with_dozer_schema(
             typ,
             nullable: field.is_nullable(),
             source: SourceDefinition::Dynamic,
+            description: None,
         });
     }
 

diff --git a/dozer-types/src/types/mod.rs b/dozer-types/src/types/mod.rs
@@ -53,6 +53,7 @@ pub struct FieldDefinition {
     pub nullable: bool,
     #[serde(default)]
     pub source: SourceDefinition,
+    pub description: Option<String>,
 }
 
 impl FieldDefinition {
@@ -62,6 +63,7 @@ impl FieldDefinition {
             typ,
             nullable,
             source,
+            description: None,
         }
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -177,6 +177,7 @@ impl Connector for MySQLConnector { @@
                                             typ,
                                             nullable,
                                             source: SourceDefinition::Dynamic,
+                                            description: None,
                                         }
                                     },
                                 )
@@ Expand Down @@