chore: include errors if table schemas are unavailable

getdozer · Oct 9, 2023 · ff8c2c4 · ff8c2c4
1 parent 3dc4067
commit ff8c2c4
Show file tree

Hide file tree

Showing 6 changed files with 143 additions and 13 deletions.
diff --git a/dozer-cli/src/live/state.rs b/dozer-cli/src/live/state.rs
@@ -1,4 +1,4 @@
-use std::{sync::Arc, thread::JoinHandle};
+use std::{collections::HashMap, sync::Arc, thread::JoinHandle};
 
 use clap::Parser;
 
@@ -204,6 +204,7 @@ impl LiveState {
         let contract = get_contract(&dozer)?;
         Ok(SchemasResponse {
             schemas: contract.get_endpoints_schemas(),
+            errors: HashMap::new(),
         })
     }
     pub async fn get_source_schemas(
@@ -217,7 +218,10 @@ impl LiveState {
         contract
             .get_source_schemas(&connection_name)
             .ok_or(LiveError::ConnectionNotFound(connection_name))
-            .map(|schemas| SchemasResponse { schemas })
+            .map(|schemas| SchemasResponse {
+                schemas,
+                errors: HashMap::new(),
+            })
     }
 
     pub async fn get_graph_schemas(&self) -> Result<SchemasResponse, LiveError> {
@@ -227,6 +231,7 @@ impl LiveState {
 
         Ok(SchemasResponse {
             schemas: contract.get_graph_schemas(),
+            errors: HashMap::new(),
         })
     }
 

diff --git a/dozer-ingestion/src/connectors/mod.rs b/dozer-ingestion/src/connectors/mod.rs
@@ -27,7 +27,7 @@ use crate::ingestion::Ingestor;
 use dozer_types::log::debug;
 use dozer_types::models::connection::Connection;
 use dozer_types::models::connection::ConnectionConfig;
-use dozer_types::models::ingestion_types::{default_grpc_adapter, EthProviderConfig};
+use dozer_types::models::ingestion_types::{default_grpc_adapter};
 use dozer_types::node::OpIdentifier;
 use dozer_types::tonic::async_trait;
 

diff --git a/dozer-types/protos/types.proto b/dozer-types/protos/types.proto
@@ -136,6 +136,7 @@ message Value {
 }
 message SchemasResponse {
   map<string, Schema> schemas = 1;
+  map<string, string> errors = 2;
 }
 
 message Schema  {

diff --git a/dozer-types/src/models/connection.rs b/dozer-types/src/models/connection.rs
@@ -57,12 +57,11 @@ impl SchemaExample for PostgresConfig {
     fn example() -> Self {
         Self {
             user: Some("postgres".to_string()),
-            password: None,
+            password: Some("postgres".to_string()),
             host: Some("localhost".to_string()),
             port: Some(5432),
             database: Some("postgres".to_string()),
-            sslmode: None,
-            connection_url: None,
+            ..Default::default()
         }
     }
 }

diff --git a/json_schemas/connections.json b/json_schemas/connections.json
@@ -10,7 +10,7 @@
           "connection_url": null,
           "database": "postgres",
           "host": "localhost",
-          "password": null,
+          "password": "postgres",
           "port": 5432,
           "sslmode": null,
           "user": "postgres"
@@ -78,6 +78,22 @@
     "schema": {
       "$schema": "http://json-schema.org/draft-07/schema#",
       "title": "EthConfig",
+      "examples": [
+        {
+          "provider": {
+            "Log": {
+              "contracts": [],
+              "filter": {
+                "addresses": [],
+                "from_block": 0,
+                "to_block": null,
+                "topics": []
+              },
+              "wss_url": ""
+            }
+          }
+        }
+      ],
       "type": "object",
       "required": [
         "provider"
@@ -111,6 +127,7 @@
           "type": "object",
           "properties": {
             "addresses": {
+              "default": [],
               "type": "array",
               "items": {
                 "type": "string"
@@ -133,6 +150,7 @@
               "minimum": 0.0
             },
             "topics": {
+              "default": [],
               "type": "array",
               "items": {
                 "type": "string"
@@ -147,6 +165,7 @@
           ],
           "properties": {
             "contracts": {
+              "default": [],
               "type": "array",
               "items": {
                 "$ref": "#/definitions/EthContract"
@@ -236,6 +255,16 @@
     "schema": {
       "$schema": "http://json-schema.org/draft-07/schema#",
       "title": "GrpcConfig",
+      "examples": [
+        {
+          "adapter": "arrow",
+          "host": "localhost",
+          "port": 50051,
+          "schemas": {
+            "Path": "schema.json"
+          }
+        }
+      ],
       "type": "object",
       "required": [
         "schemas"
@@ -302,6 +331,19 @@
     "schema": {
       "$schema": "http://json-schema.org/draft-07/schema#",
       "title": "SnowflakeConfig",
+      "examples": [
+        {
+          "database": "database",
+          "driver": "SnowflakeDSIIDriver",
+          "password": "password",
+          "port": "443",
+          "role": "role",
+          "schema": "schema",
+          "server": "<account_name>.<region_id>.snowflakecomputing.com",
+          "user": "bob",
+          "warehouse": "warehouse"
+        }
+      ],
       "type": "object",
       "required": [
         "database",
@@ -356,6 +398,12 @@
     "schema": {
       "$schema": "http://json-schema.org/draft-07/schema#",
       "title": "KafkaConfig",
+      "examples": [
+        {
+          "broker": "",
+          "schema_registry_url": ""
+        }
+      ],
       "type": "object",
       "required": [
         "broker"
@@ -378,6 +426,27 @@
     "schema": {
       "$schema": "http://json-schema.org/draft-07/schema#",
       "title": "S3Storage",
+      "examples": [
+        {
+          "details": {
+            "access_key_id": "",
+            "bucket_name": "",
+            "region": "",
+            "secret_access_key": ""
+          },
+          "tables": [
+            {
+              "config": {
+                "CSV": {
+                  "extension": ".csv",
+                  "path": "path/to/file"
+                }
+              },
+              "name": "table_name"
+            }
+          ]
+        }
+      ],
       "type": "object",
       "required": [
         "details",
@@ -474,12 +543,18 @@
         "Table": {
           "type": "object",
           "required": [
-            "config",
             "name"
           ],
           "properties": {
             "config": {
-              "$ref": "#/definitions/TableConfig"
+              "anyOf": [
+                {
+                  "$ref": "#/definitions/TableConfig"
+                },
+                {
+                  "type": "null"
+                }
+              ]
             },
             "name": {
               "type": "string"
@@ -534,6 +609,24 @@
     "schema": {
       "$schema": "http://json-schema.org/draft-07/schema#",
       "title": "LocalStorage",
+      "examples": [
+        {
+          "details": {
+            "path": "path"
+          },
+          "tables": [
+            {
+              "config": {
+                "CSV": {
+                  "extension": ".csv",
+                  "path": "path/to/table"
+                }
+              },
+              "name": "table_name"
+            }
+          ]
+        }
+      ],
       "type": "object",
       "required": [
         "details",
@@ -618,12 +711,18 @@
         "Table": {
           "type": "object",
           "required": [
-            "config",
             "name"
           ],
           "properties": {
             "config": {
-              "$ref": "#/definitions/TableConfig"
+              "anyOf": [
+                {
+                  "$ref": "#/definitions/TableConfig"
+                },
+                {
+                  "type": "null"
+                }
+              ]
             },
             "name": {
               "type": "string"
@@ -678,6 +777,16 @@
     "schema": {
       "$schema": "http://json-schema.org/draft-07/schema#",
       "title": "DeltaLakeConfig",
+      "examples": [
+        {
+          "tables": [
+            {
+              "name": "",
+              "path": ""
+            }
+          ]
+        }
+      ],
       "type": "object",
       "required": [
         "tables"
@@ -714,6 +823,11 @@
     "schema": {
       "$schema": "http://json-schema.org/draft-07/schema#",
       "title": "MongodbConfig",
+      "examples": [
+        {
+          "connection_string": "mongodb://localhost:27017/db_name"
+        }
+      ],
       "type": "object",
       "required": [
         "connection_string"
@@ -730,6 +844,12 @@
     "schema": {
       "$schema": "http://json-schema.org/draft-07/schema#",
       "title": "MySQLConfig",
+      "examples": [
+        {
+          "server_id": 1,
+          "url": "mysql://root:1234@localhost:3306/db_name"
+        }
+      ],
       "type": "object",
       "required": [
         "url"
@@ -760,7 +880,12 @@
       ],
       "properties": {
         "log_options": {
-          "$ref": "#/definitions/NestedDozerLogOptions"
+          "default": {},
+          "allOf": [
+            {
+              "$ref": "#/definitions/NestedDozerLogOptions"
+            }
+          ]
         },
         "url": {
           "type": "string"

diff --git a/json_schemas/dozer.json b/json_schemas/dozer.json
@@ -1233,7 +1233,7 @@
           "connection_url": null,
           "database": "postgres",
           "host": "localhost",
-          "password": null,
+          "password": "postgres",
           "port": 5432,
           "sslmode": null,
           "user": "postgres"