chore: remove option usage in connection configuration (#2126)

* chore: remove option from s3 config * chore: remove unnecessary option * chore: remove option
getdozer · Oct 5, 2023 · e01363b · e01363b
1 parent 4e934e3
commit e01363b
Show file tree

Hide file tree

Showing 10 changed files with 62 additions and 109 deletions.
diff --git a/dozer-cli/src/cli/init.rs b/dozer-cli/src/cli/init.rs
@@ -99,11 +99,11 @@ pub fn generate_connection(connection_name: &str) -> Connection {
                 topics: vec![],
             };
             let ethereum_config = EthConfig {
-                provider: Some(EthProviderConfig::Log(EthLogConfig {
+                provider: EthProviderConfig::Log(EthLogConfig {
                     wss_url: "wss://link".to_owned(),
                     filter: Some(eth_filter),
                     contracts: vec![],
-                })),
+                }),
             };
             let connection: Connection = Connection {
                 name: "ethereum".to_owned(),
@@ -130,7 +130,7 @@ pub fn generate_connection(connection_name: &str) -> Connection {
                 bucket_name: "<your_bucket_name>".to_owned(),
             };
             let s3_config = S3Storage {
-                details: Some(s3_details),
+                details: s3_details,
                 tables: vec![],
             };
             let connection: Connection = Connection {

diff --git a/dozer-ingestion/src/connectors/mod.rs b/dozer-ingestion/src/connectors/mod.rs
@@ -214,7 +214,7 @@ pub fn get_connector(connection: Connection) -> Result<Box<dyn Connector>, Conne
             Ok(Box::new(PostgresConnector::new(postgres_config)))
         }
         #[cfg(feature = "ethereum")]
-        ConnectionConfig::Ethereum(eth_config) => match eth_config.provider.unwrap() {
+        ConnectionConfig::Ethereum(eth_config) => match eth_config.provider {
             dozer_types::ingestion_types::EthProviderConfig::Log(log_config) => {
                 Ok(Box::new(EthLogConnector::new(log_config, connection.name)))
             }

diff --git a/dozer-ingestion/src/connectors/object_store/adapters.rs b/dozer-ingestion/src/connectors/object_store/adapters.rs
@@ -1,4 +1,4 @@
-use crate::errors::ObjectStoreObjectError::{MissingStorageDetails, TableDefinitionNotFound};
+use crate::errors::ObjectStoreObjectError::TableDefinitionNotFound;
 use crate::errors::{ConnectorError, ObjectStoreConnectorError};
 use dozer_types::ingestion_types::{LocalStorage, S3Storage, Table};
 use object_store::aws::{AmazonS3, AmazonS3Builder};
@@ -54,7 +54,7 @@ impl DozerObjectStore for S3Storage {
         &self,
         table: &Table,
     ) -> Result<DozerObjectStoreParams<Self::ObjectStore>, ConnectorError> {
-        let details = get_details(&self.details)?;
+        let details = &self.details;
 
         let retry_config = RetryConfig {
             backoff: BackoffConfig::default(),
@@ -116,7 +116,7 @@ impl DozerObjectStore for LocalStorage {
         &self,
         table: &Table,
     ) -> Result<DozerObjectStoreParams<Self::ObjectStore>, ConnectorError> {
-        let path = get_details(&self.details)?.path.as_str();
+        let path = &self.details.path.as_str();
 
         let object_store = LocalFileSystem::new_with_prefix(path)
             .map_err(|e| ConnectorError::InitializationError(e.to_string()))?;
@@ -158,11 +158,3 @@ impl DozerObjectStore for LocalStorage {
         &self.tables
     }
 }
-
-fn get_details<T>(details: &Option<T>) -> Result<&T, ObjectStoreConnectorError> {
-    details
-        .as_ref()
-        .ok_or(ObjectStoreConnectorError::DataFusionStorageObjectError(
-            MissingStorageDetails,
-        ))
-}
diff --git a/dozer-ingestion/src/connectors/object_store/tests/local_storage_tests.rs b/dozer-ingestion/src/connectors/object_store/tests/local_storage_tests.rs
@@ -388,9 +388,9 @@ fn test_unsupported_format() {
 #[tokio::test]
 async fn test_missing_directory() {
     let mut local_storage = get_local_storage_config("unsupported", "");
-    local_storage.details = Some(LocalDetails {
+    local_storage.details = LocalDetails {
         path: "not_existing_path".to_string(),
-    });
+    };
     let connector = ObjectStoreConnector::new(local_storage);
 
     let tables = connector

diff --git a/dozer-ingestion/src/connectors/object_store/tests/test_utils.rs b/dozer-ingestion/src/connectors/object_store/tests/test_utils.rs
@@ -8,9 +8,9 @@ pub fn get_local_storage_config(typ: &str, prefix: &str) -> LocalStorage {
     match typ {
         "parquet" => match prefix {
             "" => LocalStorage {
-                details: Some(LocalDetails {
+                details: LocalDetails {
                     path: p.to_str().unwrap().to_string(),
-                }),
+                },
                 tables: vec![Table {
                     config: Some(TableConfig::Parquet(ParquetConfig {
                         extension: typ.to_string(),
@@ -21,9 +21,9 @@ pub fn get_local_storage_config(typ: &str, prefix: &str) -> LocalStorage {
                 }],
             },
             &_ => LocalStorage {
-                details: Some(LocalDetails {
+                details: LocalDetails {
                     path: p.to_str().unwrap().to_string(),
-                }),
+                },
                 tables: vec![Table {
                     config: Some(TableConfig::Parquet(ParquetConfig {
                         extension: typ.to_string(),
@@ -36,9 +36,9 @@ pub fn get_local_storage_config(typ: &str, prefix: &str) -> LocalStorage {
         },
         "csv" => match prefix {
             "" => LocalStorage {
-                details: Some(LocalDetails {
+                details: LocalDetails {
                     path: p.to_str().unwrap().to_string(),
-                }),
+                },
                 tables: vec![Table {
                     config: Some(TableConfig::CSV(CsvConfig {
                         extension: typ.to_string(),
@@ -49,9 +49,9 @@ pub fn get_local_storage_config(typ: &str, prefix: &str) -> LocalStorage {
                 }],
             },
             &_ => LocalStorage {
-                details: Some(LocalDetails {
+                details: LocalDetails {
                     path: p.to_str().unwrap().to_string(),
-                }),
+                },
                 tables: vec![Table {
                     config: Some(TableConfig::CSV(CsvConfig {
                         extension: typ.to_string(),
@@ -63,9 +63,9 @@ pub fn get_local_storage_config(typ: &str, prefix: &str) -> LocalStorage {
             },
         },
         &_ => LocalStorage {
-            details: Some(LocalDetails {
+            details: LocalDetails {
                 path: p.to_str().unwrap().to_string(),
-            }),
+            },
             tables: vec![Table {
                 config: None,
                 name: String::new(),

diff --git a/dozer-ingestion/tests/test_suite/connectors/object_store/local_storage.rs b/dozer-ingestion/tests/test_suite/connectors/object_store/local_storage.rs
@@ -86,9 +86,9 @@ fn create_connector(
     writer.close().expect("Failed to close writer");
 
     let local_storage = LocalStorage {
-        details: Some(LocalDetails {
+        details: LocalDetails {
             path: temp_dir.path().to_str().expect("Non-UTF8 path").to_string(),
-        }),
+        },
         tables: vec![Table {
             config: Some(TableConfig::Parquet(ParquetConfig {
                 path: table_name.to_string(),

diff --git a/dozer-types/src/ingestion_types.rs b/dozer-types/src/ingestion_types.rs
@@ -77,7 +77,7 @@ pub enum GrpcConfigSchemas {
 
 #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema, Default)]
 pub struct EthConfig {
-    pub provider: Option<EthProviderConfig>,
+    pub provider: EthProviderConfig,
 }
 
 #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)]
@@ -86,6 +86,11 @@ pub enum EthProviderConfig {
 
     Trace(EthTraceConfig),
 }
+impl Default for EthProviderConfig {
+    fn default() -> Self {
+        EthProviderConfig::Log(EthLogConfig::default())
+    }
+}
 
 #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema, Default)]
 pub struct EthLogConfig {
@@ -117,8 +122,7 @@ impl EthConfig {
     pub fn convert_to_table(&self) -> PrettyTable {
         let mut table = table!();
 
-        let provider = self.provider.as_ref().expect("Must provide provider");
-        match provider {
+        match &self.provider {
             EthProviderConfig::Log(log) => {
                 table.add_row(row!["provider", "logs"]);
                 table.add_row(row!["wss_url", format!("{:?}", log.wss_url)]);
@@ -312,23 +316,18 @@ pub struct S3Details {
 
 #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)]
 pub struct S3Storage {
-    pub details: Option<S3Details>,
+    pub details: S3Details,
 
     pub tables: Vec<Table>,
 }
 
 impl S3Storage {
     pub fn convert_to_table(&self) -> PrettyTable {
-        self.details.as_ref().map_or_else(
-            || table!(),
-            |details| {
-                table!(
-                    ["access_key_id", details.access_key_id],
-                    ["secret_access_key", details.secret_access_key],
-                    ["region", details.region],
-                    ["bucket_name", details.bucket_name]
-                )
-            },
+        table!(
+            ["access_key_id", self.details.access_key_id],
+            ["secret_access_key", self.details.secret_access_key],
+            ["region", self.details.region],
+            ["bucket_name", self.details.bucket_name]
         )
     }
 }
@@ -340,16 +339,14 @@ pub struct LocalDetails {
 
 #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)]
 pub struct LocalStorage {
-    pub details: Option<LocalDetails>,
+    pub details: LocalDetails,
 
     pub tables: Vec<Table>,
 }
 
 impl LocalStorage {
     pub fn convert_to_table(&self) -> PrettyTable {
-        self.details
-            .as_ref()
-            .map_or_else(|| table!(), |details| table!(["path", details.path]))
+        table!(["path", self.details.path])
     }
 }
 

diff --git a/dozer-types/src/tests/eth_yaml_deserialize.rs b/dozer-types/src/tests/eth_yaml_deserialize.rs
@@ -22,13 +22,11 @@ fn standard() {
         topics: vec![],
     };
     let expected_eth_config = EthConfig {
-        provider: Some(crate::ingestion_types::EthProviderConfig::Log(
-            EthLogConfig {
-                filter: Some(expected_eth_filter),
-                wss_url: "wss://link".to_owned(),
-                contracts: vec![],
-            },
-        )),
+        provider: crate::ingestion_types::EthProviderConfig::Log(EthLogConfig {
+            filter: Some(expected_eth_filter),
+            wss_url: "wss://link".to_owned(),
+            contracts: vec![],
+        }),
     };
     let expected = ConnectionConfig::Ethereum(expected_eth_config);
     assert_eq!(expected, deserializer_result);
@@ -51,13 +49,11 @@ fn config_without_empty_array() {
         topics: vec![],
     };
     let expected_eth_config = EthConfig {
-        provider: Some(crate::ingestion_types::EthProviderConfig::Log(
-            EthLogConfig {
-                wss_url: "wss://link".to_owned(),
-                filter: Some(expected_eth_filter),
-                contracts: vec![],
-            },
-        )),
+        provider: crate::ingestion_types::EthProviderConfig::Log(EthLogConfig {
+            wss_url: "wss://link".to_owned(),
+            filter: Some(expected_eth_filter),
+            contracts: vec![],
+        }),
     };
     let expected = ConnectionConfig::Ethereum(expected_eth_config);
     assert_eq!(expected, deserializer_result);

diff --git a/json_schemas/connections.json b/json_schemas/connections.json
@@ -79,16 +79,12 @@
       "$schema": "http://json-schema.org/draft-07/schema#",
       "title": "EthConfig",
       "type": "object",
+      "required": [
+        "provider"
+      ],
       "properties": {
         "provider": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/EthProviderConfig"
-            },
-            {
-              "type": "null"
-            }
-          ]
+          "$ref": "#/definitions/EthProviderConfig"
         }
       },
       "definitions": {
@@ -387,18 +383,12 @@
       "title": "S3Storage",
       "type": "object",
       "required": [
+        "details",
         "tables"
       ],
       "properties": {
         "details": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/S3Details"
-            },
-            {
-              "type": "null"
-            }
-          ]
+          "$ref": "#/definitions/S3Details"
         },
         "tables": {
           "type": "array",
@@ -555,18 +545,12 @@
       "title": "LocalStorage",
       "type": "object",
       "required": [
+        "details",
         "tables"
       ],
       "properties": {
         "details": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/LocalDetails"
-            },
-            {
-              "type": "null"
-            }
-          ]
+          "$ref": "#/definitions/LocalDetails"
         },
         "tables": {
           "type": "array",

diff --git a/json_schemas/dozer.json b/json_schemas/dozer.json
@@ -713,16 +713,12 @@
     },
     "EthConfig": {
       "type": "object",
+      "required": [
+        "provider"
+      ],
       "properties": {
         "provider": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/EthProviderConfig"
-            },
-            {
-              "type": "null"
-            }
-          ]
+          "$ref": "#/definitions/EthProviderConfig"
         }
       }
     },
@@ -1051,18 +1047,12 @@
     "LocalStorage": {
       "type": "object",
       "required": [
+        "details",
         "tables"
       ],
       "properties": {
         "details": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/LocalDetails"
-            },
-            {
-              "type": "null"
-            }
-          ]
+          "$ref": "#/definitions/LocalDetails"
         },
         "tables": {
           "type": "array",
@@ -1406,18 +1396,12 @@
     "S3Storage": {
       "type": "object",
       "required": [
+        "details",
         "tables"
       ],
       "properties": {
         "details": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/S3Details"
-            },
-            {
-              "type": "null"
-            }
-          ]
+          "$ref": "#/definitions/S3Details"
         },
         "tables": {
           "type": "array",