diff --git a/dozer-api/src/generator/oapi/generator.rs b/dozer-api/src/generator/oapi/generator.rs index fcaf3300ee..a2d4428c01 100644 --- a/dozer-api/src/generator/oapi/generator.rs +++ b/dozer-api/src/generator/oapi/generator.rs @@ -95,7 +95,7 @@ impl<'a> OpenApiGenerator<'a> { parameters: vec![ReferenceOr::Item(Parameter::Path { parameter_data: ParameterData { name: "id".to_owned(), - description: Some(format!("Primary key of the document - {} ", self.endpoint.index.as_ref().map_or(String::new(), |index| index.primary_key.join(", ")))), + description: Some(format!("Primary key of the document - {} ", self.endpoint.index.primary_key.join(", "))), required: true, format: ParameterSchemaOrContent::Schema(ReferenceOr::Item(Schema { schema_data: SchemaData { diff --git a/dozer-api/src/grpc/client_server.rs b/dozer-api/src/grpc/client_server.rs index cde32be556..057d98b4ad 100644 --- a/dozer-api/src/grpc/client_server.rs +++ b/dozer-api/src/grpc/client_server.rs @@ -15,6 +15,8 @@ use dozer_types::grpc_types::{ common::common_grpc_service_server::CommonGrpcServiceServer, health::health_grpc_service_server::HealthGrpcServiceServer, }; +use dozer_types::models::api_config::{default_grpc_port, default_host}; +use dozer_types::models::flags::default_dynamic; use dozer_types::tonic::transport::server::TcpIncoming; use dozer_types::tonic::transport::Server; use dozer_types::tracing::Level; @@ -62,7 +64,7 @@ impl ApiServer { let security = get_api_security(self.security.to_owned()); // Service handling dynamic gRPC requests. - let typed_service = if self.flags.dynamic { + let typed_service = if self.flags.dynamic.unwrap_or_else(default_dynamic) { Some(TypedService::new( cache_endpoints, operations_receiver, @@ -78,8 +80,8 @@ impl ApiServer { pub fn new(grpc_config: GrpcApiOptions, security: Option, flags: Flags) -> Self { Self { - port: grpc_config.port as u16, - host: grpc_config.host, + port: grpc_config.port.unwrap_or_else(default_grpc_port), + host: grpc_config.host.unwrap_or_else(default_host), security, flags, } @@ -95,13 +97,14 @@ impl ApiServer { default_max_num_records: usize, ) -> Result>, ApiInitError> { + let grpc_web = self.flags.grpc_web.unwrap_or(true); // Create our services. let common_service = CommonGrpcServiceServer::new(CommonService::new( cache_endpoints.clone(), operations_receiver.as_ref().map(|r| r.resubscribe()), default_max_num_records, )); - let common_service = enable_grpc_web(common_service, self.flags.grpc_web); + let common_service = enable_grpc_web(common_service, grpc_web); let (typed_service, reflection_service) = self.get_dynamic_service( cache_endpoints, @@ -109,8 +112,8 @@ impl ApiServer { default_max_num_records, )?; let typed_service = - typed_service.map(|typed_service| enable_grpc_web(typed_service, self.flags.grpc_web)); - let reflection_service = enable_grpc_web(reflection_service, self.flags.grpc_web); + typed_service.map(|typed_service| enable_grpc_web(typed_service, grpc_web)); + let reflection_service = enable_grpc_web(reflection_service, grpc_web); let mut service_map: HashMap = HashMap::new(); service_map.insert("".to_string(), ServingStatus::Serving); @@ -123,7 +126,7 @@ impl ApiServer { let health_service = HealthGrpcServiceServer::new(HealthService { serving_status: service_map, }); - let health_service = enable_grpc_web(health_service, self.flags.grpc_web); + let health_service = enable_grpc_web(health_service, grpc_web); // Auth middleware. let security = get_api_security(self.security.to_owned()); @@ -134,7 +137,7 @@ impl ApiServer { let typed_service = typed_service.map(|typed_service| auth_middleware.layer(typed_service)); let mut authenticated_reflection_service = None; let mut unauthenticated_reflection_service = None; - if self.flags.authenticate_server_reflection { + if self.flags.authenticate_server_reflection.unwrap_or(false) { authenticated_reflection_service = Some(auth_middleware.layer(reflection_service)) } else { unauthenticated_reflection_service = Some(reflection_service); @@ -146,7 +149,7 @@ impl ApiServer { if security.is_some() { let service = enable_grpc_web( AuthGrpcServiceServer::new(AuthService::new(security.to_owned())), - self.flags.grpc_web, + grpc_web, ); auth_service = Some(auth_middleware.layer(service)); } diff --git a/dozer-api/src/grpc/internal/internal_pipeline_server.rs b/dozer-api/src/grpc/internal/internal_pipeline_server.rs index 1e7b4d9745..aa935d3a67 100644 --- a/dozer-api/src/grpc/internal/internal_pipeline_server.rs +++ b/dozer-api/src/grpc/internal/internal_pipeline_server.rs @@ -10,7 +10,9 @@ use dozer_types::grpc_types::internal::{ EndpointResponse, EndpointsResponse, LogRequest, LogResponse, StorageRequest, StorageResponse, }; use dozer_types::log::info; -use dozer_types::models::api_config::AppGrpcOptions; +use dozer_types::models::api_config::{ + default_app_grpc_host, default_app_grpc_port, AppGrpcOptions, +}; use dozer_types::models::api_endpoint::ApiEndpoint; use dozer_types::tonic::transport::server::TcpIncoming; use dozer_types::tonic::transport::Server; @@ -180,7 +182,11 @@ pub async fn start_internal_pipeline_server( let server = InternalPipelineServer::new(endpoints); // Start listening. - let addr = format!("{}:{}", options.host, options.port); + let addr = format!( + "{}:{}", + options.host.clone().unwrap_or_else(default_app_grpc_host), + options.port.unwrap_or_else(default_app_grpc_port) + ); info!("Starting Internal Server on {addr}"); let addr = addr .parse() diff --git a/dozer-api/src/lib.rs b/dozer-api/src/lib.rs index d841775fa7..76535c9a5d 100644 --- a/dozer-api/src/lib.rs +++ b/dozer-api/src/lib.rs @@ -52,11 +52,11 @@ impl CacheEndpoint { cache_labels(endpoint.name.clone(), log_reader_builder.build_name.clone()); cache_labels.extend(labels.labels().clone()); let schema = log_reader_builder.schema.clone(); - let conflict_resolution = endpoint.conflict_resolution.unwrap_or_default(); + let conflict_resolution = endpoint.conflict_resolution; let write_options = CacheWriteOptions { - insert_resolution: conflict_resolution.on_insert.unwrap_or_default(), - delete_resolution: conflict_resolution.on_delete.unwrap_or_default(), - update_resolution: conflict_resolution.on_update.unwrap_or_default(), + insert_resolution: conflict_resolution.on_insert, + delete_resolution: conflict_resolution.on_delete, + update_resolution: conflict_resolution.on_update, ..Default::default() }; let cache = open_or_create_cache( @@ -154,18 +154,15 @@ fn get_log_reader_options(endpoint: &ApiEndpoint) -> LogReaderOptions { endpoint: endpoint.name.clone(), batch_size: endpoint .log_reader_options - .as_ref() - .and_then(|options| options.batch_size) + .batch_size .unwrap_or_else(default_log_reader_batch_size), timeout_in_millis: endpoint .log_reader_options - .as_ref() - .and_then(|options| options.timeout_in_millis) + .timeout_in_millis .unwrap_or_else(default_log_reader_timeout_in_millis), buffer_size: endpoint .log_reader_options - .as_ref() - .and_then(|options| options.buffer_size) + .buffer_size .unwrap_or_else(default_log_reader_buffer_size), } } diff --git a/dozer-api/src/rest/mod.rs b/dozer-api/src/rest/mod.rs index 0c63f0948e..c1a05595b2 100644 --- a/dozer-api/src/rest/mod.rs +++ b/dozer-api/src/rest/mod.rs @@ -20,6 +20,7 @@ use actix_web::{ }; use actix_web_httpauth::middleware::HttpAuthentication; use dozer_tracing::LabelsAndProgress; +use dozer_types::models::api_config::{default_host, default_rest_port}; use dozer_types::{log::info, models::api_config::RestApiOptions}; use dozer_types::{ models::api_security::ApiSecurity, @@ -71,10 +72,10 @@ impl ApiServer { ) -> Self { Self { shutdown_timeout: 0, - port: rest_config.port as u16, + port: rest_config.port.unwrap_or_else(default_rest_port), cors: CorsOptions::Permissive, security, - host: rest_config.host, + host: rest_config.host.unwrap_or_else(default_host), default_max_num_records, } } diff --git a/dozer-api/src/test_utils.rs b/dozer-api/src/test_utils.rs index a46664cec2..404d5ee5e6 100644 --- a/dozer-api/src/test_utils.rs +++ b/dozer-api/src/test_utils.rs @@ -59,13 +59,13 @@ pub fn get_endpoint() -> ApiEndpoint { ApiEndpoint { name: "films".to_string(), path: "/films".to_string(), - index: Some(ApiIndex { + index: ApiIndex { primary_key: vec!["film_id".to_string()], - secondary: None, - }), + secondary: Default::default(), + }, table_name: "film".to_string(), - conflict_resolution: None, - log_reader_options: None, + conflict_resolution: Default::default(), + log_reader_options: Default::default(), version: None, } } diff --git a/dozer-cache/src/cache/lmdb/cache/main_environment/conflict_resolution_tests.rs b/dozer-cache/src/cache/lmdb/cache/main_environment/conflict_resolution_tests.rs index a7ddcb02c7..335e66bed2 100644 --- a/dozer-cache/src/cache/lmdb/cache/main_environment/conflict_resolution_tests.rs +++ b/dozer-cache/src/cache/lmdb/cache/main_environment/conflict_resolution_tests.rs @@ -12,9 +12,9 @@ use super::RwMainEnvironment; fn init_env(conflict_resolution: ConflictResolution) -> (RwMainEnvironment, Schema) { let schema = schema_multi_indices(); let write_options = CacheWriteOptions { - insert_resolution: conflict_resolution.on_insert.unwrap_or_default(), - delete_resolution: conflict_resolution.on_delete.unwrap_or_default(), - update_resolution: conflict_resolution.on_update.unwrap_or_default(), + insert_resolution: conflict_resolution.on_insert, + delete_resolution: conflict_resolution.on_delete, + update_resolution: conflict_resolution.on_update, ..Default::default() }; let main_env = @@ -25,9 +25,9 @@ fn init_env(conflict_resolution: ConflictResolution) -> (RwMainEnvironment, Sche #[test] fn ignore_insert_error_when_type_nothing() { let (mut env, schema) = init_env(ConflictResolution { - on_insert: Some(OnInsertResolutionTypes::Nothing(())), - on_update: None, - on_delete: None, + on_insert: OnInsertResolutionTypes::Nothing, + on_update: Default::default(), + on_delete: Default::default(), }); let initial_values = vec![Field::Int(1), Field::String("Film name old".to_string())]; @@ -56,9 +56,9 @@ fn ignore_insert_error_when_type_nothing() { #[test] fn update_after_insert_error_when_type_update() { let (mut env, schema) = init_env(ConflictResolution { - on_insert: Some(OnInsertResolutionTypes::Update(())), - on_update: None, - on_delete: None, + on_insert: OnInsertResolutionTypes::Update, + on_update: Default::default(), + on_delete: Default::default(), }); let initial_values = vec![Field::Int(1), Field::String("Film name old".to_string())]; @@ -102,9 +102,9 @@ fn update_after_insert_error_when_type_update() { #[test] fn return_insert_error_when_type_panic() { let (mut env, schema) = init_env(ConflictResolution { - on_insert: Some(OnInsertResolutionTypes::Panic(())), - on_update: None, - on_delete: None, + on_insert: OnInsertResolutionTypes::Panic, + on_update: Default::default(), + on_delete: Default::default(), }); let initial_values = vec![Field::Int(1), Field::String("Film name old".to_string())]; @@ -129,9 +129,9 @@ fn return_insert_error_when_type_panic() { #[test] fn ignore_update_error_when_type_nothing() { let (mut env, schema) = init_env(ConflictResolution { - on_insert: None, - on_update: Some(OnUpdateResolutionTypes::Nothing(())), - on_delete: None, + on_insert: Default::default(), + on_update: OnUpdateResolutionTypes::Nothing, + on_delete: Default::default(), }); let initial_values = vec![Field::Int(1), Field::Null]; @@ -159,9 +159,9 @@ fn ignore_update_error_when_type_nothing() { #[test] fn update_after_update_error_when_type_upsert() { let (mut env, schema) = init_env(ConflictResolution { - on_insert: None, - on_update: Some(OnUpdateResolutionTypes::Upsert(())), - on_delete: None, + on_insert: Default::default(), + on_update: OnUpdateResolutionTypes::Upsert, + on_delete: Default::default(), }); let initial_values = vec![Field::Int(1), Field::Null]; @@ -191,9 +191,9 @@ fn update_after_update_error_when_type_upsert() { #[test] fn return_update_error_when_type_panic() { let (mut env, _) = init_env(ConflictResolution { - on_insert: None, - on_update: Some(OnUpdateResolutionTypes::Panic(())), - on_delete: None, + on_insert: Default::default(), + on_update: OnUpdateResolutionTypes::Panic, + on_delete: Default::default(), }); let initial_values = vec![Field::Int(1), Field::Null]; @@ -219,9 +219,9 @@ fn return_update_error_when_type_panic() { #[test] fn ignore_delete_error_when_type_nothing() { let (mut env, _) = init_env(ConflictResolution { - on_insert: None, - on_update: None, - on_delete: Some(OnDeleteResolutionTypes::Nothing(())), + on_insert: Default::default(), + on_update: Default::default(), + on_delete: OnDeleteResolutionTypes::Nothing, }); let initial_values = vec![Field::Int(1), Field::Null]; @@ -242,9 +242,9 @@ fn ignore_delete_error_when_type_nothing() { #[test] fn return_delete_error_when_type_panic() { let (mut env, _) = init_env(ConflictResolution { - on_insert: None, - on_update: None, - on_delete: Some(OnDeleteResolutionTypes::Panic(())), + on_insert: Default::default(), + on_update: Default::default(), + on_delete: OnDeleteResolutionTypes::Panic, }); let initial_values = vec![Field::Int(1), Field::Null]; diff --git a/dozer-cache/src/cache/lmdb/cache/main_environment/mod.rs b/dozer-cache/src/cache/lmdb/cache/main_environment/mod.rs index 8c34f59f28..aa44051893 100644 --- a/dozer-cache/src/cache/lmdb/cache/main_environment/mod.rs +++ b/dozer-cache/src/cache/lmdb/cache/main_environment/mod.rs @@ -259,11 +259,11 @@ impl RwMainEnvironment { } else { // The record does not exist. Resolve the conflict. match self.write_options.delete_resolution { - OnDeleteResolutionTypes::Nothing(()) => { + OnDeleteResolutionTypes::Nothing => { warn!("Record (Key: {:?}) not found, ignoring delete", key); Ok(None) } - OnDeleteResolutionTypes::Panic(()) => Err(CacheError::PrimaryKeyNotFound), + OnDeleteResolutionTypes::Panic => Err(CacheError::PrimaryKeyNotFound), } } } @@ -320,8 +320,7 @@ impl RwMainEnvironment { meta, }) => { // Case 5, 6, 7. - if self.write_options.update_resolution - == OnUpdateResolutionTypes::Nothing(()) + if self.write_options.update_resolution == OnUpdateResolutionTypes::Nothing { // Case 5. warn!("Old record (Key: {:?}) and new record (Key: {:?}) both exist, ignoring update", get_key_fields(old, schema), get_key_fields(new, schema)); @@ -367,22 +366,22 @@ impl RwMainEnvironment { } else { // Case 2, 3, 4, 9, 10, 11, 12, 13. match self.write_options.update_resolution { - OnUpdateResolutionTypes::Nothing(()) => { + OnUpdateResolutionTypes::Nothing => { // Case 2, 9, 12. warn!("Old record (Key: {:?}) not found, ignoring update", old_key); Ok(UpsertResult::Ignored) } - OnUpdateResolutionTypes::Upsert(()) => { + OnUpdateResolutionTypes::Upsert => { // Case 3, 10, 13. insert_impl( operation_log, txn, &self.common.schema.0, new, - OnInsertResolutionTypes::Panic(()), + OnInsertResolutionTypes::Panic, ) } - OnUpdateResolutionTypes::Panic(()) => { + OnUpdateResolutionTypes::Panic => { // Case 4, 11, 14. Err(CacheError::PrimaryKeyNotFound) } @@ -501,19 +500,19 @@ fn insert_impl( } else { // Resolve the conflict. match insert_resolution { - OnInsertResolutionTypes::Nothing(()) => { + OnInsertResolutionTypes::Nothing => { warn!( "Record (Key: {:?}) already exist, ignoring insert", get_key_fields(record, schema) ); Ok(UpsertResult::Ignored) } - OnInsertResolutionTypes::Panic(()) => Err(CacheError::PrimaryKeyExists { + OnInsertResolutionTypes::Panic => Err(CacheError::PrimaryKeyExists { key: get_key_fields(record, schema), meta, insert_operation_id, }), - OnInsertResolutionTypes::Update(()) => { + OnInsertResolutionTypes::Update => { let new_meta = operation_log.update( txn, key.as_ref(), diff --git a/dozer-cache/src/cache/plan/planner.rs b/dozer-cache/src/cache/plan/planner.rs index 94c9c489e3..90a9d1ee1c 100644 --- a/dozer-cache/src/cache/plan/planner.rs +++ b/dozer-cache/src/cache/plan/planner.rs @@ -1,8 +1,6 @@ use crate::cache::expression::{FilterExpression, Operator, SortDirection, SortOptions}; use crate::errors::PlanError; -use dozer_types::models::api_endpoint::{ - CreateSecondaryIndex, FullText, SecondaryIndex, SortedInverted, -}; +use dozer_types::models::api_endpoint::{FullText, SecondaryIndex, SortedInverted}; use dozer_types::types::{Field, FieldDefinition, Schema}; use dozer_types::types::{FieldType, IndexDefinition}; use dozer_types::{json_value_to_field, serde_yaml}; @@ -268,9 +266,7 @@ fn describe_index_configuration( match index { IndexScanKind::FullText { filter } => { let field = field_definitions[filter.field_index].name.clone(); - creates.push(CreateSecondaryIndex { - index: Some(SecondaryIndex::FullText(FullText { field })), - }); + creates.push(SecondaryIndex::FullText(FullText { field })); } IndexScanKind::SortedInverted { eq_filters, @@ -285,9 +281,7 @@ fn describe_index_configuration( let field = field_definitions[range_query.field_index].name.clone(); fields.push(field); } - creates.push(CreateSecondaryIndex { - index: Some(SecondaryIndex::SortedInverted(SortedInverted { fields })), - }); + creates.push(SecondaryIndex::SortedInverted(SortedInverted { fields })); } } } diff --git a/dozer-cli/src/cli/helper.rs b/dozer-cli/src/cli/helper.rs index 97de459600..a55111f985 100644 --- a/dozer-cli/src/cli/helper.rs +++ b/dozer-cli/src/cli/helper.rs @@ -235,10 +235,10 @@ mod tests { app_name: "test_override_nested".to_string(), ..Default::default() }; - config.api = Some(ApiConfig { + config.api = ApiConfig { api_security: Some(ApiSecurity::Jwt("secret1".to_string())), ..Default::default() - }); + }; let api_security = ApiSecurity::Jwt("secret2".to_string()); let config = apply_overrides( &config, @@ -248,6 +248,6 @@ mod tests { )], ) .unwrap(); - assert_eq!(config.api.unwrap().api_security.unwrap(), api_security); + assert_eq!(config.api.api_security.unwrap(), api_security); } } diff --git a/dozer-cli/src/cli/init.rs b/dozer-cli/src/cli/init.rs index 5bedb02644..f82d148d42 100644 --- a/dozer-cli/src/cli/init.rs +++ b/dozer-cli/src/cli/init.rs @@ -86,7 +86,7 @@ pub fn generate_connection(connection_name: &str) -> Connection { }; let connection: Connection = Connection { name: "snowflake".to_owned(), - config: Some(ConnectionConfig::Snowflake(snowflake_config)), + config: ConnectionConfig::Snowflake(snowflake_config), }; connection } @@ -106,7 +106,7 @@ pub fn generate_connection(connection_name: &str) -> Connection { }; let connection: Connection = Connection { name: "ethereum".to_owned(), - config: Some(ConnectionConfig::Ethereum(ethereum_config)), + config: ConnectionConfig::Ethereum(ethereum_config), }; connection } @@ -117,7 +117,7 @@ pub fn generate_connection(connection_name: &str) -> Connection { }; let connection: Connection = Connection { name: "mysql".to_owned(), - config: Some(ConnectionConfig::MySQL(mysql_config)), + config: ConnectionConfig::MySQL(mysql_config), }; connection } @@ -134,7 +134,7 @@ pub fn generate_connection(connection_name: &str) -> Connection { }; let connection: Connection = Connection { name: "s3".to_owned(), - config: Some(ConnectionConfig::S3Storage(s3_config)), + config: ConnectionConfig::S3Storage(s3_config), }; connection } @@ -145,7 +145,7 @@ pub fn generate_connection(connection_name: &str) -> Connection { }; let connection: Connection = Connection { name: "mongodb".to_owned(), - config: Some(ConnectionConfig::MongoDB(mongo_config)), + config: ConnectionConfig::MongoDB(mongo_config), }; connection } @@ -161,7 +161,7 @@ pub fn generate_connection(connection_name: &str) -> Connection { }; let connection: Connection = Connection { name: "postgres".to_owned(), - config: Some(ConnectionConfig::Postgres(postgres_config)), + config: ConnectionConfig::Postgres(postgres_config), }; connection } @@ -197,11 +197,11 @@ pub fn generate_config_repl() -> Result<(), OrchestrationError> { format!("question: Home directory ({:}): ", default_home_dir()), Box::new(move |(home_dir, config)| { if home_dir.is_empty() { - config.home_dir = default_home_dir(); - config.cache_dir = default_cache_dir(); + config.home_dir = Some(default_home_dir()); + config.cache_dir = Some(default_cache_dir()); } else { - config.home_dir = home_dir; - config.cache_dir = get_cache_dir(&config.home_dir); + config.cache_dir = Some(get_cache_dir(&home_dir)); + config.home_dir = Some(home_dir); } Ok(()) }), diff --git a/dozer-cli/src/cloud_app_context.rs b/dozer-cli/src/cloud_app_context.rs index c9dab5dc1a..bf9710e016 100644 --- a/dozer-cli/src/cloud_app_context.rs +++ b/dozer-cli/src/cloud_app_context.rs @@ -32,11 +32,8 @@ impl CloudAppContext { Ok(()) } - pub fn get_app_id(config: Option<&Cloud>) -> Result { - match &config { - None => Err(AppIdNotFound), - Some(cloud_config) => cloud_config.app_id.clone().ok_or(AppIdNotFound), - } + pub fn get_app_id(config: &Cloud) -> Result { + config.app_id.clone().ok_or(AppIdNotFound) } pub fn save_app_id(app_id: String) -> Result<(), CloudContextError> { diff --git a/dozer-cli/src/live/state.rs b/dozer-cli/src/live/state.rs index d941d808e6..89ad323330 100644 --- a/dozer-cli/src/live/state.rs +++ b/dozer-cli/src/live/state.rs @@ -7,20 +7,15 @@ use dozer_core::{app::AppPipeline, dag_schemas::DagSchemas, Dag}; use dozer_sql::builder::statement_to_pipeline; use dozer_tracing::{Labels, LabelsAndProgress}; use dozer_types::{ - constants::DEFAULT_DEFAULT_MAX_NUM_RECORDS, grpc_types::{ contract::{DotResponse, ProtoResponse, SchemasResponse}, live::{BuildResponse, BuildStatus, ConnectResponse, LiveApp, LiveResponse, RunRequest}, }, log::info, models::{ - api_config::{ - default_api_grpc, default_api_rest, default_app_grpc, ApiConfig, AppGrpcOptions, - GrpcApiOptions, RestApiOptions, - }, + api_config::{ApiConfig, AppGrpcOptions, GrpcApiOptions, RestApiOptions}, api_endpoint::ApiEndpoint, api_security::ApiSecurity, - app_config::AppConfig, flags::Flags, }, }; @@ -180,12 +175,7 @@ impl LiveState { .ok() .map(ApiSecurity::Jwt) .as_ref() - .or(dozer - .dozer - .config - .api - .as_ref() - .and_then(|f| f.api_security.as_ref())) + .or(dozer.dozer.config.api.api_security.as_ref()) .is_some(); LiveApp { app_name: dozer.dozer.config.app_name.clone(), @@ -390,7 +380,7 @@ fn get_dozer_run_instance( Some(dozer_types::grpc_types::live::run_request::Request::Sql(req)) => { let context = statement_to_pipeline( &req.sql, - &mut AppPipeline::new(dozer.config.flags.clone().unwrap_or_default().into()), + &mut AppPipeline::new(dozer.config.flags.clone().into()), None, dozer.config.udfs.clone(), ) @@ -425,31 +415,14 @@ fn get_dozer_run_instance( None => {} }; - if let Some(app) = dozer.config.app.as_mut() { - app.max_num_records_before_persist = Some(usize::MAX as u64); - app.max_interval_before_persist_in_seconds = Some(u64::MAX); - } else { - dozer.config.app = Some(AppConfig { - max_num_records_before_persist: Some(usize::MAX as u64), - max_interval_before_persist_in_seconds: Some(u64::MAX), - ..Default::default() - }) - } + let app = &mut dozer.config.app; + app.max_num_records_before_persist = Some(usize::MAX as u64); + app.max_interval_before_persist_in_seconds = Some(u64::MAX); - if let Some(api) = dozer.config.api.as_mut() { - override_api_config(api); - } else { - dozer.config.api = Some(ApiConfig { - api_security: None, - rest: Some(default_rest_config_for_live()), - grpc: Some(default_grpc_config_for_live()), - app_grpc: Some(default_app_grpc_config_for_live()), - default_max_num_records: DEFAULT_DEFAULT_MAX_NUM_RECORDS as u32, - }) - } + override_api_config(&mut dozer.config.api); - dozer.config.home_dir = temp_dir.to_string(); - dozer.config.cache_dir = AsRef::::as_ref(temp_dir).join("cache").into(); + dozer.config.home_dir = Some(temp_dir.to_string()); + dozer.config.cache_dir = Some(AsRef::::as_ref(temp_dir).join("cache").into()); dozer.labels = LabelsAndProgress::new(labels, false); @@ -457,59 +430,27 @@ fn get_dozer_run_instance( } fn override_api_config(api: &mut ApiConfig) { - if let Some(rest) = api.rest.as_mut() { - override_rest_config(rest); - } else { - api.rest = Some(default_rest_config_for_live()); - } - - if let Some(grpc) = api.grpc.as_mut() { - override_grpc_config(grpc); - } else { - api.grpc = Some(default_grpc_config_for_live()); - } - - if let Some(app_grpc) = api.app_grpc.as_mut() { - override_app_grpc_config(app_grpc); - } else { - api.app_grpc = Some(default_app_grpc_config_for_live()); - } + override_rest_config(&mut api.rest); + override_grpc_config(&mut api.grpc); + override_app_grpc_config(&mut api.app_grpc); } fn override_rest_config(rest: &mut RestApiOptions) { - rest.host = "0.0.0.0".to_string(); - rest.port = 62996; - rest.cors = true; - rest.enabled = true; -} - -fn default_rest_config_for_live() -> RestApiOptions { - let mut rest = default_api_rest(); - override_rest_config(&mut rest); - rest + rest.host = Some("0.0.0.0".to_string()); + rest.port = Some(62996); + rest.cors = Some(true); + rest.enabled = Some(true); } fn override_grpc_config(grpc: &mut GrpcApiOptions) { - grpc.host = "0.0.0.0".to_string(); - grpc.port = 62998; - grpc.cors = true; - grpc.web = true; - grpc.enabled = true; -} - -fn default_grpc_config_for_live() -> GrpcApiOptions { - let mut grpc = default_api_grpc(); - override_grpc_config(&mut grpc); - grpc + grpc.host = Some("0.0.0.0".to_string()); + grpc.port = Some(62998); + grpc.cors = Some(true); + grpc.web = Some(true); + grpc.enabled = Some(true); } fn override_app_grpc_config(app_grpc: &mut AppGrpcOptions) { - app_grpc.port = 62997; - app_grpc.host = "0.0.0.0".to_string(); -} - -fn default_app_grpc_config_for_live() -> AppGrpcOptions { - let mut app_grpc = default_app_grpc(); - override_app_grpc_config(&mut app_grpc); - app_grpc + app_grpc.port = Some(62997); + app_grpc.host = Some("0.0.0.0".to_string()); } diff --git a/dozer-cli/src/main.rs b/dozer-cli/src/main.rs index 2e94f4b0bd..3f48b1bd16 100644 --- a/dozer-cli/src/main.rs +++ b/dozer-cli/src/main.rs @@ -131,26 +131,26 @@ fn run() -> Result<(), OrchestrationError> { set_ctrl_handler(shutdown_sender); // Now we have access to telemetry configuration. Telemetry must be initialized in tokio runtime. - let app_name = dozer.config.app_name.clone(); let app_id = dozer .config .cloud - .as_ref() - .map(|cloud| cloud.app_id.clone().unwrap_or(app_name)); + .app_id + .as_deref() + .unwrap_or(&dozer.config.app_name); // We always enable telemetry when running live. let telemetry_config = if matches!(cli.cmd, Commands::Live(_)) { - Some(TelemetryConfig { + TelemetryConfig { trace: None, - metrics: Some(TelemetryMetricsConfig::Prometheus(())), - }) + metrics: Some(TelemetryMetricsConfig::Prometheus), + } } else { dozer.config.telemetry.clone() }; let _telemetry = dozer .runtime - .block_on(async { Telemetry::new(app_id.as_deref(), telemetry_config) }); + .block_on(async { Telemetry::new(Some(app_id), &telemetry_config) }); set_panic_hook(); @@ -251,58 +251,66 @@ fn run() -> Result<(), OrchestrationError> { // Some commands dont need to initialize the orchestrator // This function is used to run those commands fn parse_and_generate() -> Result { - dozer_tracing::init_telemetry_closure(None, None, || -> Result { - let cli = Cli::parse(); - - if let Commands::Init = cli.cmd { - Telemetry::new(None, None); - if let Err(e) = generate_config_repl() { - error!("{}", e); - Err(e) + dozer_tracing::init_telemetry_closure( + None, + &Default::default(), + || -> Result { + let cli = Cli::parse(); + + if let Commands::Init = cli.cmd { + Telemetry::new(None, &Default::default()); + if let Err(e) = generate_config_repl() { + error!("{}", e); + Err(e) + } else { + // We need to exit here, otherwise the orchestrator will be initialized + process::exit(0); + } } else { - // We need to exit here, otherwise the orchestrator will be initialized - process::exit(0); + Ok(cli) } - } else { - Ok(cli) - } - }) + }, + ) } fn init_orchestrator(cli: &Cli) -> Result { - dozer_tracing::init_telemetry_closure(None, None, || -> Result { - let runtime = Arc::new(Runtime::new().map_err(CliError::FailedToCreateTokioRuntime)?); - let res = runtime.block_on(init_dozer( - runtime.clone(), - cli.config_paths.clone(), - cli.config_token.clone(), - cli.config_overrides.clone(), - cli.ignore_pipe, - LabelsAndProgress::new(Default::default(), cli.enable_progress), - )); - - match res { - Ok(dozer) => { - dozer.runtime.spawn(check_update()); - Ok(dozer) - } - Err(e) => { - if let CliError::FailedToFindConfigurationFiles(_) = &e { - let description = "Dozer was not able to find configuration files. \n\n\ + dozer_tracing::init_telemetry_closure( + None, + &Default::default(), + || -> Result { + let runtime = Arc::new(Runtime::new().map_err(CliError::FailedToCreateTokioRuntime)?); + let res = runtime.block_on(init_dozer( + runtime.clone(), + cli.config_paths.clone(), + cli.config_token.clone(), + cli.config_overrides.clone(), + cli.ignore_pipe, + LabelsAndProgress::new(Default::default(), cli.enable_progress), + )); + + match res { + Ok(dozer) => { + dozer.runtime.spawn(check_update()); + Ok(dozer) + } + Err(e) => { + if let CliError::FailedToFindConfigurationFiles(_) = &e { + let description = "Dozer was not able to find configuration files. \n\n\ Please use \"dozer init\" to create project or \"dozer -c {path}\" with path to your configuration.\n\ Configuration documentation can be found in https://getdozer.io/docs/configuration"; - let mut command = Cli::command(); - command = command.about(format!("\n\n\n{} \n {}", LOGO, description)); + let mut command = Cli::command(); + command = command.about(format!("\n\n\n{} \n {}", LOGO, description)); - println!("{}", command.render_help()); - } + println!("{}", command.render_help()); + } - error!("{}", e); - Err(e) + error!("{}", e); + Err(e) + } } - } - }) + }, + ) } fn display_error(e: &OrchestrationError) { @@ -320,7 +328,7 @@ fn display_error(e: &OrchestrationError) { struct Telemetry(); impl Telemetry { - fn new(app_name: Option<&str>, config: Option) -> Self { + fn new(app_name: Option<&str>, config: &TelemetryConfig) -> Self { dozer_tracing::init_telemetry(app_name, config); Self() } diff --git a/dozer-cli/src/pipeline/builder.rs b/dozer-cli/src/pipeline/builder.rs index 4229804321..4b3996c26d 100644 --- a/dozer-cli/src/pipeline/builder.rs +++ b/dozer-cli/src/pipeline/builder.rs @@ -93,7 +93,7 @@ impl<'a> PipelineBuilder<'a> { for connection in self.connections { let connector = get_connector(connection.clone())?; - if let Ok(info_table) = get_connector_info_table(connection) { + if let Some(info_table) = get_connector_info_table(connection) { info!("[{}] Connection parameters\n{info_table}", connection.name); } diff --git a/dozer-cli/src/pipeline/tests/builder.rs b/dozer-cli/src/pipeline/tests/builder.rs index 69f8050c21..46c050aebc 100644 --- a/dozer-cli/src/pipeline/tests/builder.rs +++ b/dozer-cli/src/pipeline/tests/builder.rs @@ -12,10 +12,12 @@ use dozer_types::models::source::Source; fn get_default_config() -> Config { let schema_str = include_str!("./schemas.json"); let grpc_conn = Connection { - config: Some(ConnectionConfig::Grpc(GrpcConfig { - schemas: Some(GrpcConfigSchemas::Inline(schema_str.to_string())), - ..Default::default() - })), + config: ConnectionConfig::Grpc(GrpcConfig { + host: None, + port: None, + adapter: None, + schemas: GrpcConfigSchemas::Inline(schema_str.to_string()), + }), name: "grpc_conn".to_string(), }; @@ -32,7 +34,7 @@ fn get_default_config() -> Config { columns: vec!["id".to_string(), "name".to_string()], connection: grpc_conn.name.clone(), schema: None, - refresh_config: None, + refresh_config: Default::default(), }, Source { name: "grpc_conn_customers".to_string(), @@ -40,7 +42,7 @@ fn get_default_config() -> Config { columns: vec!["id".to_string(), "name".to_string()], connection: grpc_conn.name, schema: None, - refresh_config: None, + refresh_config: Default::default(), }, ], ..Default::default() diff --git a/dozer-cli/src/simple/build/contract/mod.rs b/dozer-cli/src/simple/build/contract/mod.rs index 648d5f10d2..7f85b07ccb 100644 --- a/dozer-cli/src/simple/build/contract/mod.rs +++ b/dozer-cli/src/simple/build/contract/mod.rs @@ -108,18 +108,17 @@ impl Contract { .find(|connection| connection.name == node.handle.id) .ok_or(BuildError::MissingConnection(node.handle.id.clone()))?; let typ = match &connection.config { - None => "None", - Some(ConnectionConfig::Postgres(_)) => "Postgres", - Some(ConnectionConfig::Ethereum(_)) => "Ethereum", - Some(ConnectionConfig::Grpc(_)) => "Grpc", - Some(ConnectionConfig::Snowflake(_)) => "Snowflake", - Some(ConnectionConfig::Kafka(_)) => "Kafka", - Some(ConnectionConfig::S3Storage(_)) => "S3Storage", - Some(ConnectionConfig::LocalStorage(_)) => "LocalStorage", - Some(ConnectionConfig::DeltaLake(_)) => "DeltaLake", - Some(ConnectionConfig::MongoDB(_)) => "MongoDB", - Some(ConnectionConfig::MySQL(_)) => "MySQL", - Some(ConnectionConfig::Dozer(_)) => "Dozer", + ConnectionConfig::Postgres(_) => "Postgres", + ConnectionConfig::Ethereum(_) => "Ethereum", + ConnectionConfig::Grpc(_) => "Grpc", + ConnectionConfig::Snowflake(_) => "Snowflake", + ConnectionConfig::Kafka(_) => "Kafka", + ConnectionConfig::S3Storage(_) => "S3Storage", + ConnectionConfig::LocalStorage(_) => "LocalStorage", + ConnectionConfig::DeltaLake(_) => "DeltaLake", + ConnectionConfig::MongoDB(_) => "MongoDB", + ConnectionConfig::MySQL(_) => "MySQL", + ConnectionConfig::Dozer(_) => "Dozer", }; source_types.insert(node_index, typ); } diff --git a/dozer-cli/src/simple/build/contract/modify_schema.rs b/dozer-cli/src/simple/build/contract/modify_schema.rs index 68e248dc5f..56f301cc4c 100644 --- a/dozer-cli/src/simple/build/contract/modify_schema.rs +++ b/dozer-cli/src/simple/build/contract/modify_schema.rs @@ -1,5 +1,3 @@ -use std::borrow::Cow; - use dozer_types::{ models::api_endpoint::{ ApiEndpoint, FullText, SecondaryIndex, SecondaryIndexConfig, SortedInverted, @@ -15,13 +13,7 @@ pub fn modify_schema( ) -> Result { let mut schema = schema.clone(); // Generated Cache index based on api_index - let configured_index = create_primary_indexes( - &schema.fields, - api_endpoint - .index - .as_ref() - .map(|index| index.primary_key.as_slice()), - )?; + let configured_index = create_primary_indexes(&schema.fields, &api_endpoint.index.primary_key)?; // Generated schema in SQL let upstream_index = schema.primary_index.clone(); @@ -43,8 +35,8 @@ pub fn modify_schema( schema.primary_index = index; - let secondary_index_config = get_secondary_index_config(api_endpoint); - let secondary_indexes = generate_secondary_indexes(&schema.fields, &secondary_index_config)?; + let secondary_indexes = + generate_secondary_indexes(&schema.fields, &api_endpoint.index.secondary)?; Ok((schema, secondary_indexes)) } @@ -56,29 +48,14 @@ fn get_field_names(schema: &Schema, indexes: &[usize]) -> Vec { .collect() } -fn get_secondary_index_config(api_endpoint: &ApiEndpoint) -> Cow { - if let Some(config) = api_endpoint - .index - .as_ref() - .and_then(|index| index.secondary.as_ref()) - { - Cow::Borrowed(config) - } else { - Cow::Owned(SecondaryIndexConfig::default()) - } -} - fn create_primary_indexes( field_definitions: &[FieldDefinition], - primary_key: Option<&[String]>, + primary_key: &[String], ) -> Result, BuildError> { - let mut primary_index = Vec::new(); - if let Some(primary_key) = primary_key { - for name in primary_key { - primary_index.push(field_index_from_field_name(field_definitions, name)?); - } - } - Ok(primary_index) + primary_key + .iter() + .map(|name| field_index_from_field_name(field_definitions, name)) + .collect() } fn generate_secondary_indexes( @@ -119,20 +96,18 @@ fn generate_secondary_indexes( } // Create requested indexes. - for create in &config.create { - if let Some(index) = &create.index { - match index { - SecondaryIndex::SortedInverted(SortedInverted { fields }) => { - let fields = fields - .iter() - .map(|field| field_index_from_field_name(field_definitions, field)) - .collect::, _>>()?; - result.push(IndexDefinition::SortedInverted(fields)); - } - SecondaryIndex::FullText(FullText { field }) => { - let field = field_index_from_field_name(field_definitions, field)?; - result.push(IndexDefinition::FullText(field)); - } + for index in &config.create { + match index { + SecondaryIndex::SortedInverted(SortedInverted { fields }) => { + let fields = fields + .iter() + .map(|field| field_index_from_field_name(field_definitions, field)) + .collect::, _>>()?; + result.push(IndexDefinition::SortedInverted(fields)); + } + SecondaryIndex::FullText(FullText { field }) => { + let field = field_index_from_field_name(field_definitions, field)?; + result.push(IndexDefinition::FullText(field)); } } } diff --git a/dozer-cli/src/simple/cloud/monitor.rs b/dozer-cli/src/simple/cloud/monitor.rs index 09a200bb2c..78306935c9 100644 --- a/dozer-cli/src/simple/cloud/monitor.rs +++ b/dozer-cli/src/simple/cloud/monitor.rs @@ -13,7 +13,7 @@ use tokio::runtime::Runtime; pub fn monitor_app( cloud: &Cloud, - cloud_config: Option<&dozer_types::models::cloud::Cloud>, + cloud_config: &dozer_types::models::cloud::Cloud, runtime: Arc, ) -> Result<(), CloudError> { let app_id = cloud diff --git a/dozer-cli/src/simple/cloud_orchestrator.rs b/dozer-cli/src/simple/cloud_orchestrator.rs index 52fa85c436..11a88bf43c 100644 --- a/dozer-cli/src/simple/cloud_orchestrator.rs +++ b/dozer-cli/src/simple/cloud_orchestrator.rs @@ -35,12 +35,12 @@ use tower::ServiceBuilder; use super::cloud::login::LoginSvc; use super::cloud::version::{get_version_status, version_is_up_to_date, version_status_table}; -pub async fn get_cloud_client( +async fn establish_cloud_service_channel( cloud: &Cloud, - cloud_config: Option<&dozer_types::models::cloud::Cloud>, -) -> Result, CloudError> { + cloud_config: &dozer_types::models::cloud::Cloud, +) -> Result { let profile_name = match &cloud.profile { - None => cloud_config.as_ref().and_then(|c| c.profile.clone()), + None => cloud_config.profile.clone(), Some(_) => cloud.profile.clone(), }; let credential = CredentialInfo::load(profile_name)?; @@ -55,31 +55,23 @@ pub async fn get_cloud_client( let channel = ServiceBuilder::new() .layer_fn(|channel| TokenLayer::new(channel, credential.clone())) .service(channel); - let client = DozerCloudClient::new(channel); + Ok(channel) +} + +pub async fn get_cloud_client( + cloud: &Cloud, + cloud_config: &dozer_types::models::cloud::Cloud, +) -> Result, CloudError> { + let client = DozerCloudClient::new(establish_cloud_service_channel(cloud, cloud_config).await?); Ok(client) } pub async fn get_explorer_client( cloud: &Cloud, - cloud_config: Option<&dozer_types::models::cloud::Cloud>, + cloud_config: &dozer_types::models::cloud::Cloud, ) -> Result, CloudError> { - let profile_name = match &cloud.profile { - None => cloud_config.as_ref().and_then(|c| c.profile.clone()), - Some(_) => cloud.profile.clone(), - }; - let credential = CredentialInfo::load(profile_name)?; - let target_url = cloud - .target_url - .as_ref() - .unwrap_or(&credential.target_url) - .clone(); - - let endpoint = Endpoint::from_shared(target_url.to_owned())?; - let channel = Endpoint::connect(&endpoint).await?; - let channel = ServiceBuilder::new() - .layer_fn(|channel| TokenLayer::new(channel, credential.clone())) - .service(channel); - let client = ApiExplorerServiceClient::new(channel); + let client = + ApiExplorerServiceClient::new(establish_cloud_service_channel(cloud, cloud_config).await?); Ok(client) } @@ -91,21 +83,15 @@ impl CloudOrchestrator for SimpleOrchestrator { deploy: DeployCommandArgs, config_paths: Vec, ) -> Result<(), OrchestrationError> { - let app_id = if cloud.app_id.is_some() { - cloud.app_id.clone() - } else { - let app_id_from_context = CloudAppContext::get_app_id(self.config.cloud.as_ref()); - match app_id_from_context { - Ok(id) => Some(id), - Err(_) => None, - } - }; + let app_id = cloud + .app_id + .clone() + .or(CloudAppContext::get_app_id(&self.config.cloud).ok()); - let cloud_config = self.config.cloud.as_ref(); + let cloud_config = &self.config.cloud; let lockfile_path = self.lockfile_path(); self.runtime.block_on(async move { let mut client = get_cloud_client(&cloud, cloud_config).await?; - // let mut explorer_client = get_explorer_client(&cloud, cloud_config).await?; let mut files = list_files(config_paths)?; if deploy.locked { let lockfile_contents = tokio::fs::read_to_string(lockfile_path) @@ -147,16 +133,13 @@ impl CloudOrchestrator for SimpleOrchestrator { let (app_id, delete_cloud_file) = if let Some(app_id) = cloud.app_id.clone() { // if the app_id on command line is equal to the one in the cloud config file then file can be deleted - if app_id == CloudAppContext::get_app_id(self.config.cloud.as_ref())? { + if app_id == CloudAppContext::get_app_id(&self.config.cloud)? { (app_id, true) } else { (app_id, false) } } else { - ( - CloudAppContext::get_app_id(self.config.cloud.as_ref())?, - true, - ) + (CloudAppContext::get_app_id(&self.config.cloud)?, true) }; let mut double_check = String::new(); @@ -173,7 +156,7 @@ impl CloudOrchestrator for SimpleOrchestrator { return Ok(()); } - let cloud_config = self.config.cloud.as_ref(); + let cloud_config = &self.config.cloud; self.runtime.block_on(async move { let mut client = get_cloud_client(&cloud, cloud_config).await?; @@ -203,7 +186,7 @@ impl CloudOrchestrator for SimpleOrchestrator { } fn list(&mut self, cloud: Cloud, list: ListCommandArgs) -> Result<(), OrchestrationError> { - let cloud_config = self.config.cloud.as_ref(); + let cloud_config = &self.config.cloud; self.runtime.block_on(async move { let mut client = get_cloud_client(&cloud, cloud_config).await?; let response = client @@ -244,8 +227,8 @@ impl CloudOrchestrator for SimpleOrchestrator { let app_id = cloud .app_id .clone() - .unwrap_or(CloudAppContext::get_app_id(self.config.cloud.as_ref())?); - let cloud_config = self.config.cloud.as_ref(); + .unwrap_or(CloudAppContext::get_app_id(&self.config.cloud)?); + let cloud_config = &self.config.cloud; self.runtime.block_on(async move { let mut client = get_cloud_client(&cloud, cloud_config).await?; let response = client @@ -314,7 +297,7 @@ impl CloudOrchestrator for SimpleOrchestrator { } fn monitor(&mut self, cloud: Cloud) -> Result<(), OrchestrationError> { - monitor_app(&cloud, self.config.cloud.as_ref(), self.runtime.clone()) + monitor_app(&cloud, &self.config.cloud, self.runtime.clone()) .map_err(crate::errors::OrchestrationError::CloudError) } @@ -322,8 +305,8 @@ impl CloudOrchestrator for SimpleOrchestrator { let app_id = cloud .app_id .clone() - .unwrap_or(CloudAppContext::get_app_id(self.config.cloud.as_ref())?); - let cloud_config = self.config.cloud.as_ref(); + .unwrap_or(CloudAppContext::get_app_id(&self.config.cloud)?); + let cloud_config = &self.config.cloud; self.runtime.block_on(async move { let mut client = get_cloud_client(&cloud, cloud_config).await?; @@ -425,8 +408,8 @@ impl CloudOrchestrator for SimpleOrchestrator { let app_id = cloud .app_id .clone() - .unwrap_or(CloudAppContext::get_app_id(self.config.cloud.as_ref())?); - let cloud_config = self.config.cloud.as_ref(); + .unwrap_or(CloudAppContext::get_app_id(&self.config.cloud)?); + let cloud_config = &self.config.cloud; self.runtime.block_on(async move { let mut client = get_cloud_client(&cloud, cloud_config).await?; @@ -506,9 +489,9 @@ impl SimpleOrchestrator { let app_id = cloud .app_id .clone() - .unwrap_or(CloudAppContext::get_app_id(self.config.cloud.as_ref())?); + .unwrap_or(CloudAppContext::get_app_id(&self.config.cloud)?); - let cloud_config = self.config.cloud.as_ref(); + let cloud_config = &self.config.cloud; self.runtime.block_on(async move { let mut client = get_cloud_client(&cloud, cloud_config).await?; @@ -614,8 +597,8 @@ impl SimpleOrchestrator { let app_id = cloud .app_id .clone() - .unwrap_or(CloudAppContext::get_app_id(self.config.cloud.as_ref())?); - let cloud_config = self.config.cloud.as_ref(); + .unwrap_or(CloudAppContext::get_app_id(&self.config.cloud)?); + let cloud_config = &self.config.cloud; self.runtime.block_on(async move { let mut client = get_cloud_client(&cloud, cloud_config).await?; let mut explorer_client = get_explorer_client(&cloud, cloud_config).await?; diff --git a/dozer-cli/src/simple/helper.rs b/dozer-cli/src/simple/helper.rs index e1bb96b691..fae9cd86e6 100644 --- a/dozer-cli/src/simple/helper.rs +++ b/dozer-cli/src/simple/helper.rs @@ -4,14 +4,19 @@ use crate::errors::OrchestrationError; use dozer_types::log::info; use dozer_types::models::api_config::ApiConfig; use dozer_types::models::api_endpoint::ApiEndpoint; +use dozer_types::models::config::default_home_dir; use dozer_types::models::config::Config; use dozer_types::prettytable::{row, Table}; pub fn validate_config(config: &Config) -> Result<(), OrchestrationError> { - info!("Home dir: {}", get_colored_text(&config.home_dir, PURPLE)); - if let Some(api_config) = &config.api { - print_api_config(api_config) - } + info!( + "Home dir: {}", + get_colored_text( + &config.home_dir.clone().unwrap_or_else(default_home_dir), + PURPLE + ) + ); + print_api_config(&config.api); validate_endpoints(&config.endpoints)?; @@ -31,13 +36,27 @@ fn print_api_config(api_config: &ApiConfig) { let mut table_parent = Table::new(); table_parent.add_row(row!["Type", "IP", "Port"]); - if let Some(rest_config) = &api_config.rest { - table_parent.add_row(row!["REST", rest_config.host, rest_config.port]); - } + table_parent.add_row(row![ + "REST", + api_config.rest.host.as_deref().unwrap_or("-"), + api_config + .rest + .port + .as_ref() + .map(ToString::to_string) + .unwrap_or_else(|| "-".to_string()) + ]); - if let Some(grpc_config) = &api_config.grpc { - table_parent.add_row(row!["GRPC", grpc_config.host, grpc_config.port]); - } + table_parent.add_row(row![ + "GRPC", + api_config.grpc.host.as_deref().unwrap_or("-"), + api_config + .grpc + .port + .as_ref() + .map(ToString::to_string) + .unwrap_or_else(|| "-".to_string()) + ]); info!( "[API] {}\n{}", get_colored_text("Configuration", PURPLE), diff --git a/dozer-cli/src/simple/orchestrator.rs b/dozer-cli/src/simple/orchestrator.rs index b7694721b8..ec3e3fe93e 100644 --- a/dozer-cli/src/simple/orchestrator.rs +++ b/dozer-cli/src/simple/orchestrator.rs @@ -6,9 +6,8 @@ use crate::shutdown::ShutdownReceiver; use crate::simple::build; use crate::simple::helper::validate_config; use crate::utils::{ - get_api_security_config, get_app_grpc_config, get_cache_manager_options, - get_checkpoint_factory_options, get_default_max_num_records, get_executor_options, - get_grpc_config, get_rest_config, + get_cache_manager_options, get_checkpoint_factory_options, get_default_max_num_records, + get_executor_options, }; use crate::{flatten_join_handle, join_handle_map_err}; @@ -22,7 +21,8 @@ use dozer_core::app::AppPipeline; use dozer_core::dag_schemas::DagSchemas; use dozer_tracing::LabelsAndProgress; use dozer_types::constants::LOCK_FILE; -use dozer_types::models::flags::default_push_events; +use dozer_types::models::api_config::{default_app_grpc_host, default_app_grpc_port}; +use dozer_types::models::flags::{default_dynamic, default_push_events}; use tokio::select; use crate::console_helper::get_colored_text; @@ -34,7 +34,7 @@ use dozer_ingestion::connectors::{get_connector, SourceSchema, TableInfo}; use dozer_sql::builder::statement_to_pipeline; use dozer_sql::errors::PipelineError; use dozer_types::log::info; -use dozer_types::models::config::Config; +use dozer_types::models::config::{default_cache_dir, default_home_dir, Config}; use dozer_types::tracing::error; use futures::stream::FuturesUnordered; use futures::{FutureExt, StreamExt, TryFutureExt}; @@ -84,18 +84,25 @@ impl SimpleOrchestrator { let mut futures = FuturesUnordered::new(); // Open `RoCacheEndpoint`s. Streaming operations if necessary. - let flags = self.config.flags.clone().unwrap_or_default(); - let (operations_sender, operations_receiver) = if flags.dynamic { - let (sender, receiver) = broadcast::channel(16); - (Some(sender), Some(receiver)) - } else { - (None, None) - }; + let flags = self.config.flags.clone(); + let (operations_sender, operations_receiver) = + if flags.dynamic.unwrap_or_else(default_dynamic) { + let (sender, receiver) = broadcast::channel(16); + (Some(sender), Some(receiver)) + } else { + (None, None) + }; - let internal_grpc_config = get_app_grpc_config(&self.config); + let internal_grpc_config = &self.config.api.app_grpc; let app_server_addr = format!( "http://{}:{}", - internal_grpc_config.host, internal_grpc_config.port + internal_grpc_config + .host + .clone() + .unwrap_or_else(default_app_grpc_host), + internal_grpc_config + .port + .unwrap_or_else(default_app_grpc_port) ); let cache_manager = Arc::new( LmdbRwCacheManager::new(get_cache_manager_options(&self.config)) @@ -129,9 +136,9 @@ impl SimpleOrchestrator { } // Initialize API Server - let rest_config = get_rest_config(&self.config); - let rest_handle = if rest_config.enabled { - let security = get_api_security_config(&self.config).cloned(); + let rest_config = self.config.api.rest.clone(); + let rest_handle = if rest_config.enabled.unwrap_or(true) { + let security = self.config.api.api_security.clone(); let cache_endpoints_for_rest = cache_endpoints.clone(); let shutdown_for_rest = shutdown.create_shutdown_future(); let api_server = @@ -149,9 +156,9 @@ impl SimpleOrchestrator { }; // Initialize gRPC Server - let grpc_config = get_grpc_config(&self.config); - let grpc_handle = if grpc_config.enabled { - let api_security = get_api_security_config(&self.config).cloned(); + let grpc_config = self.config.api.grpc.clone(); + let grpc_handle = if grpc_config.enabled.unwrap_or(true) { + let api_security = self.config.api.api_security.clone(); let grpc_server = grpc::ApiServer::new(grpc_config, api_security, flags); let shutdown = shutdown.create_shutdown_future(); let grpc_server = grpc_server @@ -187,11 +194,21 @@ impl SimpleOrchestrator { } pub fn home_dir(&self) -> Utf8PathBuf { - self.base_directory.join(&self.config.home_dir) + self.base_directory.join( + self.config + .home_dir + .clone() + .unwrap_or_else(default_home_dir), + ) } pub fn cache_dir(&self) -> Utf8PathBuf { - self.base_directory.join(&self.config.cache_dir) + self.base_directory.join( + self.config + .cache_dir + .clone() + .unwrap_or_else(default_cache_dir), + ) } pub fn lockfile_path(&self) -> Utf8PathBuf { @@ -221,15 +238,15 @@ impl SimpleOrchestrator { &self.runtime, get_executor_options(&self.config), shutdown.clone(), - self.config.flags.clone().unwrap_or_default(), + self.config.flags.clone(), ))?; - let app_grpc_config = get_app_grpc_config(&self.config); + let app_grpc_config = &self.config.api.app_grpc; let internal_server_future = self .runtime .block_on(start_internal_pipeline_server( endpoint_and_logs, - &app_grpc_config, + app_grpc_config, shutdown.create_shutdown_future(), )) .map_err(OrchestrationError::InternalServerFailed)?; @@ -276,8 +293,7 @@ impl SimpleOrchestrator { } pub fn generate_token(&self, ttl_in_secs: Option) -> Result { - if let Some(api_security) = get_api_security(get_api_security_config(&self.config).cloned()) - { + if let Some(api_security) = get_api_security(self.config.api.api_security.clone()) { match api_security { dozer_types::models::api_security::ApiSecurity::Jwt(secret) => { let auth = Authorizer::new(&secret, None, None); @@ -325,7 +341,7 @@ impl SimpleOrchestrator { self.config.sql.as_deref(), endpoint_and_logs, self.labels.clone(), - self.config.flags.clone().unwrap_or_default(), + self.config.flags.clone(), &self.config.udfs, ); let dag = self @@ -335,17 +351,11 @@ impl SimpleOrchestrator { let dag_schemas = DagSchemas::new(dag)?; // Get current contract. - let enable_token = self - .config - .api - .as_ref() - .map(|api| api.api_security.is_some()) - .unwrap_or(false); + let enable_token = self.config.api.api_security.is_some(); let enable_on_event = self .config .flags - .as_ref() - .map(|flags| flags.push_events) + .push_events .unwrap_or_else(default_push_events); let version = self.config.version as usize; diff --git a/dozer-cli/src/utils.rs b/dozer-cli/src/utils.rs index a80719e372..bab603b024 100644 --- a/dozer-cli/src/utils.rs +++ b/dozer-cli/src/utils.rs @@ -5,17 +5,12 @@ use dozer_core::{ use dozer_types::{ constants::DEFAULT_DEFAULT_MAX_NUM_RECORDS, models::{ - api_config::{ - default_api_grpc, default_api_rest, default_app_grpc, AppGrpcOptions, GrpcApiOptions, - RestApiOptions, - }, - api_security::ApiSecurity, app_config::{ default_app_buffer_size, default_commit_size, default_commit_timeout, default_error_threshold, default_max_interval_before_persist_in_seconds, - default_max_num_records_before_persist, default_persist_queue_capacity, DataStorage, + default_max_num_records_before_persist, default_persist_queue_capacity, }, - config::{default_cache_max_map_size, Config}, + config::{default_cache_dir, default_cache_max_map_size, Config}, }, }; use std::time::Duration; @@ -30,8 +25,7 @@ fn get_commit_time_threshold(config: &Config) -> Duration { Duration::from_millis( config .app - .as_ref() - .and_then(|app| app.commit_timeout) + .commit_timeout .unwrap_or_else(default_commit_timeout), ) } @@ -39,89 +33,43 @@ fn get_commit_time_threshold(config: &Config) -> Duration { fn get_buffer_size(config: &Config) -> u32 { config .app - .as_ref() - .and_then(|app| app.app_buffer_size) + .app_buffer_size .unwrap_or_else(default_app_buffer_size) } fn get_commit_size(config: &Config) -> u32 { - config - .app - .as_ref() - .and_then(|app| app.commit_size) - .unwrap_or_else(default_commit_size) + config.app.commit_size.unwrap_or_else(default_commit_size) } fn get_error_threshold(config: &Config) -> u32 { config .app - .as_ref() - .and_then(|app| app.error_threshold) + .error_threshold .unwrap_or_else(default_error_threshold) } fn get_max_num_records_before_persist(config: &Config) -> usize { config .app - .as_ref() - .and_then(|app| app.max_num_records_before_persist) + .max_num_records_before_persist .unwrap_or_else(default_max_num_records_before_persist) as usize } fn get_max_interval_before_persist_in_seconds(config: &Config) -> u64 { config .app - .as_ref() - .and_then(|app| app.max_interval_before_persist_in_seconds) + .max_interval_before_persist_in_seconds .unwrap_or_else(default_max_interval_before_persist_in_seconds) } -pub fn get_storage_config(config: &Config) -> DataStorage { - let app = config.app.as_ref(); - app.and_then(|app| app.data_storage.clone()) - .unwrap_or_default() -} - -pub fn get_grpc_config(config: &Config) -> GrpcApiOptions { - config - .api - .as_ref() - .and_then(|api| api.grpc.clone()) - .unwrap_or_else(default_api_grpc) -} - -pub fn get_rest_config(config: &Config) -> RestApiOptions { - config - .api - .as_ref() - .and_then(|api| api.rest.clone()) - .unwrap_or_else(default_api_rest) -} - -pub fn get_app_grpc_config(config: &Config) -> AppGrpcOptions { - config - .api - .as_ref() - .and_then(|api| api.app_grpc.clone()) - .unwrap_or_else(default_app_grpc) -} - -pub fn get_api_security_config(config: &Config) -> Option<&ApiSecurity> { - config - .api - .as_ref() - .and_then(|api| api.api_security.as_ref()) -} - pub fn get_checkpoint_factory_options(config: &Config) -> CheckpointFactoryOptions { CheckpointFactoryOptions { persist_queue_capacity: config .app - .as_ref() - .and_then(|app| app.persist_queue_capacity) + .persist_queue_capacity .unwrap_or_else(default_persist_queue_capacity) as usize, - storage_config: get_storage_config(config), + storage_config: config.app.data_storage.clone(), } } @@ -142,7 +90,13 @@ pub fn get_executor_options(config: &Config) -> ExecutorOptions { pub fn get_cache_manager_options(config: &Config) -> CacheManagerOptions { CacheManagerOptions { - path: Some(config.cache_dir.clone().into()), + path: Some( + config + .cache_dir + .clone() + .unwrap_or_else(default_cache_dir) + .into(), + ), max_size: get_cache_max_map_size(config) as usize, ..CacheManagerOptions::default() } @@ -151,7 +105,6 @@ pub fn get_cache_manager_options(config: &Config) -> CacheManagerOptions { pub fn get_default_max_num_records(config: &Config) -> usize { config .api - .as_ref() - .map(|api| api.default_max_num_records as usize) - .unwrap_or_else(|| DEFAULT_DEFAULT_MAX_NUM_RECORDS) + .default_max_num_records + .unwrap_or(DEFAULT_DEFAULT_MAX_NUM_RECORDS) } diff --git a/dozer-core/src/app.rs b/dozer-core/src/app.rs index d45887bc90..36a04d90e7 100644 --- a/dozer-core/src/app.rs +++ b/dozer-core/src/app.rs @@ -109,10 +109,7 @@ pub struct PipelineFlags { impl From<&Flags> for PipelineFlags { fn from(flags: &Flags) -> Self { Self { - enable_probabilistic_optimizations: flags - .enable_probabilistic_optimizations - .clone() - .unwrap_or_default(), + enable_probabilistic_optimizations: flags.enable_probabilistic_optimizations.clone(), } } } diff --git a/dozer-core/src/checkpoint/mod.rs b/dozer-core/src/checkpoint/mod.rs index 8afda79b92..b7302bb6f2 100644 --- a/dozer-core/src/checkpoint/mod.rs +++ b/dozer-core/src/checkpoint/mod.rs @@ -39,7 +39,7 @@ pub struct CheckpointFactoryOptions { impl Default for CheckpointFactoryOptions { fn default() -> Self { Self { - storage_config: DataStorage::Local(()), + storage_config: DataStorage::Local, persist_queue_capacity: 100, } } diff --git a/dozer-ingestion/src/connectors/dozer/connector.rs b/dozer-ingestion/src/connectors/dozer/connector.rs index c11e64b22b..dd14b467da 100644 --- a/dozer-ingestion/src/connectors/dozer/connector.rs +++ b/dozer-ingestion/src/connectors/dozer/connector.rs @@ -4,7 +4,6 @@ use dozer_log::{ reader::{LogReaderBuilder, LogReaderOptions}, replication::LogOperation, }; -use dozer_types::tonic::{async_trait, transport::Channel}; use dozer_types::{ errors::types::DeserializationError, grpc_types::internal::{ @@ -12,11 +11,16 @@ use dozer_types::{ DescribeApplicationRequest, DescribeApplicationResponse, }, ingestion_types::{ - default_log_options, IngestionMessage, NestedDozerConfig, NestedDozerLogOptions, + default_buffer_size, default_timeout, IngestionMessage, NestedDozerConfig, + NestedDozerLogOptions, }, serde_json, types::{Operation, Record, Schema}, }; +use dozer_types::{ + ingestion_types::default_log_batch_size, + tonic::{async_trait, transport::Channel}, +}; use tokio::{ sync::mpsc::{channel, Sender}, task::JoinSet, @@ -162,12 +166,11 @@ impl NestedDozerConnector { Self { config } } async fn get_client(&self) -> Result, ConnectorError> { - let app_server_addr = self.get_server_addr()?; - let client = InternalPipelineServiceClient::connect(app_server_addr.clone()) + let client = InternalPipelineServiceClient::connect(self.config.url.clone()) .await .map_err(|e| { ConnectorError::NestedDozerConnectorError( - NestedDozerConnectorError::ConnectionError(app_server_addr, e), + NestedDozerConnectorError::ConnectionError(self.config.url.clone(), e), ) })?; Ok(client) @@ -188,21 +191,12 @@ impl NestedDozerConnector { Ok(response.into_inner()) } - fn get_server_addr(&self) -> Result { - let config = self - .config - .grpc - .as_ref() - .ok_or(NestedDozerConnectorError::MissingGrpcConfig)?; - Ok(format!("http://{}:{}", &config.host, &config.port)) - } - fn get_log_options(endpoint: String, value: NestedDozerLogOptions) -> LogReaderOptions { LogReaderOptions { endpoint, - batch_size: value.batch_size, - timeout_in_millis: value.timeout_in_millis, - buffer_size: value.buffer_size, + batch_size: value.batch_size.unwrap_or_else(default_log_batch_size), + timeout_in_millis: value.timeout_in_millis.unwrap_or_else(default_timeout), + buffer_size: value.buffer_size.unwrap_or_else(default_buffer_size), } } @@ -210,14 +204,8 @@ impl NestedDozerConnector { &self, endpoint: String, ) -> Result { - let app_server_addr = self.get_server_addr()?; - - let log_options = match self.config.log_options.as_ref() { - Some(opts) => opts.clone(), - None => default_log_options(), - }; - let log_options = Self::get_log_options(endpoint, log_options); - let log_reader_builder = LogReaderBuilder::new(app_server_addr, log_options) + let log_options = Self::get_log_options(endpoint, self.config.log_options.clone()); + let log_reader_builder = LogReaderBuilder::new(self.config.url.clone(), log_options) .await .map_err(NestedDozerConnectorError::ReaderBuilderError)?; Ok(log_reader_builder) diff --git a/dozer-ingestion/src/connectors/ethereum/log/tests/helper.rs b/dozer-ingestion/src/connectors/ethereum/log/tests/helper.rs index 0391c51517..c208dd8ee8 100644 --- a/dozer-ingestion/src/connectors/ethereum/log/tests/helper.rs +++ b/dozer-ingestion/src/connectors/ethereum/log/tests/helper.rs @@ -82,7 +82,7 @@ pub async fn run_eth_sample( wss_url: String, my_account: H160, ) -> (Contract, Vec) { - dozer_tracing::init_telemetry(None, None); + dozer_tracing::init_telemetry(None, &Default::default()); let orig_hook = std::panic::take_hook(); std::panic::set_hook(Box::new(move |panic_info| { // invoke the default handler and exit the process diff --git a/dozer-ingestion/src/connectors/ethereum/trace/connector.rs b/dozer-ingestion/src/connectors/ethereum/trace/connector.rs index 149a379bbd..b04f12ff65 100644 --- a/dozer-ingestion/src/connectors/ethereum/trace/connector.rs +++ b/dozer-ingestion/src/connectors/ethereum/trace/connector.rs @@ -5,7 +5,7 @@ use crate::connectors::{ TableToIngest, }; use crate::{connectors::TableInfo, errors::ConnectorError, ingestion::Ingestor}; -use dozer_types::ingestion_types::{EthTraceConfig, IngestionMessage}; +use dozer_types::ingestion_types::{default_batch_size, EthTraceConfig, IngestionMessage}; use dozer_types::log::{error, info, warn}; use dozer_types::tonic::async_trait; @@ -128,7 +128,11 @@ pub async fn run( "Starting Eth Trace connector: {} from block {}", conn_name, config.from_block ); - let batch_iter = BatchIterator::new(config.from_block, config.to_block, config.batch_size); + let batch_iter = BatchIterator::new( + config.from_block, + config.to_block, + config.batch_size.unwrap_or_else(default_batch_size), + ); let mut errors: Vec = vec![]; for batch in batch_iter { diff --git a/dozer-ingestion/src/connectors/ethereum/trace/tests.rs b/dozer-ingestion/src/connectors/ethereum/trace/tests.rs index 4167eff88e..4adf68b3c0 100644 --- a/dozer-ingestion/src/connectors/ethereum/trace/tests.rs +++ b/dozer-ingestion/src/connectors/ethereum/trace/tests.rs @@ -43,7 +43,7 @@ fn test_trace_iterator() { let runtime = create_test_runtime(); let https_url = env::var("ETH_HTTPS_URL").unwrap(); - dozer_tracing::init_telemetry(None, None); + dozer_tracing::init_telemetry(None, &Default::default()); let orig_hook = std::panic::take_hook(); std::panic::set_hook(Box::new(move |panic_info| { // invoke the default handler and exit the process @@ -57,7 +57,7 @@ fn test_trace_iterator() { https_url, from_block: 1000000, to_block: Some(1000001), - batch_size: 100, + batch_size: Some(100), }, "test".to_string(), ); diff --git a/dozer-ingestion/src/connectors/grpc/connector.rs b/dozer-ingestion/src/connectors/grpc/connector.rs index 564ac8be51..e9738a4ee9 100644 --- a/dozer-ingestion/src/connectors/grpc/connector.rs +++ b/dozer-ingestion/src/connectors/grpc/connector.rs @@ -8,7 +8,7 @@ use crate::connectors::{ }; use crate::{connectors::TableInfo, errors::ConnectorError, ingestion::Ingestor}; use dozer_types::grpc_types::ingest::ingest_service_server::IngestServiceServer; -use dozer_types::ingestion_types::GrpcConfig; +use dozer_types::ingestion_types::{default_ingest_host, default_ingest_port, GrpcConfig}; use dozer_types::log::{info, warn}; use dozer_types::tonic::async_trait; use dozer_types::tonic::transport::Server; @@ -41,14 +41,7 @@ where where T: IngestAdapter, { - let schemas = config.schemas.as_ref().map_or_else( - || { - Err(ConnectorError::InitializationError( - "schemas not found".to_string(), - )) - }, - Ok, - )?; + let schemas = &config.schemas; let schemas_str = match schemas { dozer_types::ingestion_types::GrpcConfigSchemas::Inline(schemas_str) => { schemas_str.clone() @@ -68,10 +61,10 @@ where ingestor: &Ingestor, tables: Vec, ) -> Result<(), ConnectorError> { - let host = &self.config.host; - let port = self.config.port; + let host = self.config.host.clone().unwrap_or_else(default_ingest_host); + let port = self.config.port.unwrap_or_else(default_ingest_port); - let addr = format!("{host:}:{port:}").parse().map_err(|e| { + let addr = format!("{host}:{port}").parse().map_err(|e| { ConnectorError::InitializationError(format!("Failed to parse address: {e}")) })?; diff --git a/dozer-ingestion/src/connectors/grpc/tests.rs b/dozer-ingestion/src/connectors/grpc/tests.rs index 6a9a5b6d6f..d008b6ef65 100644 --- a/dozer-ingestion/src/connectors/grpc/tests.rs +++ b/dozer-ingestion/src/connectors/grpc/tests.rs @@ -43,10 +43,10 @@ fn ingest_grpc( let grpc_connector = GrpcConnector::::new( "grpc".to_string(), GrpcConfig { - schemas: Some(GrpcConfigSchemas::Inline(schemas.to_string())), - adapter, - port, - ..Default::default() + schemas: GrpcConfigSchemas::Inline(schemas.to_string()), + adapter: Some(adapter), + port: Some(port), + host: None, }, ) .unwrap(); diff --git a/dozer-ingestion/src/connectors/mod.rs b/dozer-ingestion/src/connectors/mod.rs index 57cc3e296c..f11fb79730 100644 --- a/dozer-ingestion/src/connectors/mod.rs +++ b/dozer-ingestion/src/connectors/mod.rs @@ -24,6 +24,7 @@ use crate::connectors::postgres::connector::{PostgresConfig, PostgresConnector}; use crate::errors::ConnectorError; use crate::ingestion::Ingestor; +use dozer_types::ingestion_types::default_grpc_adapter; use dozer_types::log::debug; use dozer_types::models::connection::Connection; use dozer_types::models::connection::ConnectionConfig; @@ -50,7 +51,6 @@ use self::grpc::{ArrowAdapter, DefaultAdapter}; use self::mysql::connector::{mysql_connection_opts_from_url, MySQLConnector}; #[cfg(feature = "snowflake")] use crate::connectors::snowflake::connector::SnowflakeConnector; -use crate::errors::ConnectorError::{MissingConfiguration, WrongConnectionConfiguration}; #[derive(Clone, Copy, Serialize, Deserialize, Debug, Eq, PartialEq, Default)] #[serde(crate = "dozer_types::serde")] @@ -199,9 +199,7 @@ impl TableToIngest { } pub fn get_connector(connection: Connection) -> Result, ConnectorError> { - let config = connection - .config - .ok_or_else(|| ConnectorError::MissingConfiguration(connection.name.clone()))?; + let config = connection.config; match config { ConnectionConfig::Postgres(_) => { let config = map_connection_config(&config)?; @@ -226,20 +224,27 @@ pub fn get_connector(connection: Connection) -> Result, Conne }, #[cfg(not(feature = "ethereum"))] ConnectionConfig::Ethereum(_) => Err(ConnectorError::EthereumFeatureNotEnabled), - ConnectionConfig::Grpc(grpc_config) => match grpc_config.adapter.as_str() { - "arrow" => Ok(Box::new(GrpcConnector::::new( - connection.name, - grpc_config, - )?)), - "default" => Ok(Box::new(GrpcConnector::::new( - connection.name, - grpc_config, - )?)), - _ => Err(ConnectorError::UnsupportedGrpcAdapter( - connection.name, - grpc_config.adapter, - )), - }, + ConnectionConfig::Grpc(grpc_config) => { + match grpc_config + .adapter + .clone() + .unwrap_or_else(default_grpc_adapter) + .as_str() + { + "arrow" => Ok(Box::new(GrpcConnector::::new( + connection.name, + grpc_config, + )?)), + "default" => Ok(Box::new(GrpcConnector::::new( + connection.name, + grpc_config, + )?)), + _ => Err(ConnectorError::UnsupportedGrpcAdapter( + connection.name, + grpc_config.adapter, + )), + } + } #[cfg(feature = "snowflake")] ConnectionConfig::Snowflake(snowflake) => { let snowflake_config = snowflake; @@ -285,18 +290,18 @@ pub fn get_connector(connection: Connection) -> Result, Conne } } -pub fn get_connector_info_table(connection: &Connection) -> Result { +pub fn get_connector_info_table(connection: &Connection) -> Option { match &connection.config { - Some(ConnectionConfig::Postgres(config)) => match config.replenish() { - Ok(conf) => Ok(conf.convert_to_table()), - Err(e) => Err(WrongConnectionConfiguration(e)), + ConnectionConfig::Postgres(config) => match config.replenish() { + Ok(conf) => Some(conf.convert_to_table()), + Err(_) => None, }, - Some(ConnectionConfig::Ethereum(config)) => Ok(config.convert_to_table()), - Some(ConnectionConfig::Snowflake(config)) => Ok(config.convert_to_table()), - Some(ConnectionConfig::Kafka(config)) => Ok(config.convert_to_table()), - Some(ConnectionConfig::S3Storage(config)) => Ok(config.convert_to_table()), - Some(ConnectionConfig::LocalStorage(config)) => Ok(config.convert_to_table()), - _ => Err(MissingConfiguration(connection.name.clone())), + ConnectionConfig::Ethereum(config) => Some(config.convert_to_table()), + ConnectionConfig::Snowflake(config) => Some(config.convert_to_table()), + ConnectionConfig::Kafka(config) => Some(config.convert_to_table()), + ConnectionConfig::S3Storage(config) => Some(config.convert_to_table()), + ConnectionConfig::LocalStorage(config) => Some(config.convert_to_table()), + _ => None, } } diff --git a/dozer-ingestion/src/connectors/object_store/csv/csv_table.rs b/dozer-ingestion/src/connectors/object_store/csv/csv_table.rs index 870458a20a..171fa0afaf 100644 --- a/dozer-ingestion/src/connectors/object_store/csv/csv_table.rs +++ b/dozer-ingestion/src/connectors/object_store/csv/csv_table.rs @@ -75,10 +75,6 @@ impl CsvTable { // Get the table state after snapshot let mut update_state = self.update_state.clone(); let extension = self.table_config.extension.clone(); - let marker_extension = match self.table_config.marker_file { - true => self.table_config.marker_extension.clone(), - false => String::new(), - }; loop { // List objects in the S3 bucket with the specified prefix @@ -127,30 +123,34 @@ impl CsvTable { name: base_path.clone() + new_path_str, last_modified: object.last_modified.timestamp(), }); - if marker_extension.is_empty() { + if self.table_config.marker_extension.is_none() { update_state.insert(object.location, object.last_modified); } - } else if file_path.ends_with(marker_extension.as_str()) - && !marker_extension.is_empty() + } else if let Some(marker_extension) = + self.table_config.marker_extension.as_deref() { - // Scenario 3: New marker file added - info!( - "Source Object Marker has been added: {:?}, {:?}", - object.location, object.last_modified - ); - - // Remove base folder from relative path - let path = Path::new(&file_path); - let new_path = path - .strip_prefix(path.components().next().unwrap()) - .unwrap(); - let new_path_str = new_path.to_str().unwrap(); - - new_marker_files.push(FileInfo { - name: base_path.clone() + new_path_str, - last_modified: object.last_modified.timestamp(), - }); - update_state.insert(object.location, object.last_modified); + if file_path.ends_with(marker_extension) { + // Scenario 3: New marker file added + info!( + "Source Object Marker has been added: {:?}, {:?}", + object.location, object.last_modified + ); + + // Remove base folder from relative path + let path = Path::new(&file_path); + let new_path = path + .strip_prefix(path.components().next().unwrap()) + .unwrap(); + let new_path_str = new_path.to_str().unwrap(); + + new_marker_files.push(FileInfo { + name: base_path.clone() + new_path_str, + last_modified: object.last_modified.timestamp(), + }); + update_state.insert(object.location, object.last_modified); + } else { + continue; + } } else { // Skip files that do not match the extension nor marker extension continue; @@ -161,8 +161,7 @@ impl CsvTable { new_files.sort(); for file in &new_files { let marker_file_exist = is_marker_file_exist(new_marker_files.clone(), file); - let use_marker_file = marker_extension.is_empty(); - if !marker_file_exist && !use_marker_file { + if !marker_file_exist && self.table_config.marker_extension.is_some() { continue; } else { let file_path = ListingTableUrl::parse(&file.name) @@ -225,10 +224,7 @@ impl TableWatcher for CsvTable { // Get the table state after snapshot let mut update_state = self.update_state.clone(); let extension = self.table_config.extension.clone(); - let marker_extension = match self.table_config.marker_file { - true => self.table_config.marker_extension.clone(), - false => String::new(), - }; + let marker_extension = self.table_config.marker_extension.clone(); let h = tokio::spawn(async move { // List objects in the S3 bucket with the specified prefix @@ -277,31 +273,33 @@ impl TableWatcher for CsvTable { name: base_path.clone() + new_path_str, last_modified: object.last_modified.timestamp(), }); - if marker_extension.is_empty() { + if marker_extension.is_none() { update_state.insert(object.location, object.last_modified); } - } else if file_path.ends_with(marker_extension.as_str()) - && !marker_extension.is_empty() - { - // Scenario 3: New marker file added - info!( - "Source Object Marker has been added: {:?}, {:?}", - object.location, object.last_modified - ); - - // Remove base folder from relative path - let path = Path::new(&file_path); - let new_path = path - .strip_prefix(path.components().next().unwrap()) - .unwrap(); - let new_path_str = new_path.to_str().unwrap(); - - new_marker_files.push(FileInfo { - name: base_path.clone() + new_path_str, - last_modified: object.last_modified.timestamp(), - }); + } else if let Some(marker_extension) = marker_extension.as_deref() { + if file_path.ends_with(marker_extension) { + // Scenario 3: New marker file added + info!( + "Source Object Marker has been added: {:?}, {:?}", + object.location, object.last_modified + ); + + // Remove base folder from relative path + let path = Path::new(&file_path); + let new_path = path + .strip_prefix(path.components().next().unwrap()) + .unwrap(); + let new_path_str = new_path.to_str().unwrap(); + + new_marker_files.push(FileInfo { + name: base_path.clone() + new_path_str, + last_modified: object.last_modified.timestamp(), + }); - update_state.insert(object.location, object.last_modified); + update_state.insert(object.location, object.last_modified); + } else { + continue; + } } else { // Skip files that do not match the extension nor marker extension continue; @@ -312,8 +310,7 @@ impl TableWatcher for CsvTable { new_files.sort(); for file in &new_files { let marker_file_exist = is_marker_file_exist(new_marker_files.clone(), file); - let use_marker_file = marker_extension.is_empty(); - if !marker_file_exist && !use_marker_file { + if !marker_file_exist && marker_extension.is_some() { continue; } else { let file_path = ListingTableUrl::parse(&file.name) diff --git a/dozer-ingestion/src/connectors/object_store/parquet/parquet_table.rs b/dozer-ingestion/src/connectors/object_store/parquet/parquet_table.rs index b50a6d87ec..eb7ca99777 100644 --- a/dozer-ingestion/src/connectors/object_store/parquet/parquet_table.rs +++ b/dozer-ingestion/src/connectors/object_store/parquet/parquet_table.rs @@ -74,10 +74,6 @@ impl ParquetTable { // Get the table state after snapshot let mut update_state = self.update_state.clone(); let extension = self.table_config.extension.clone(); - let marker_extension = match self.table_config.marker_file { - true => self.table_config.marker_extension.clone(), - false => String::new(), - }; loop { // List objects in the S3 bucket with the specified prefix @@ -132,30 +128,34 @@ impl ParquetTable { name: base_path.clone() + new_path_str, last_modified: object.last_modified.timestamp(), }); - if marker_extension.is_empty() { + if self.table_config.marker_extension.is_none() { update_state.insert(object.location, object.last_modified); } - } else if file_path.ends_with(marker_extension.as_str()) - && !marker_extension.is_empty() + } else if let Some(marker_extension) = + self.table_config.marker_extension.as_deref() { - // Scenario 3: New marker file added - info!( - "Source Object Marker has been added: {:?}, {:?}", - object.location, object.last_modified - ); - - // Remove base folder from relative path - let path = Path::new(&file_path); - let new_path = path - .strip_prefix(path.components().next().unwrap()) - .unwrap(); - let new_path_str = new_path.to_str().unwrap(); - - new_marker_files.push(FileInfo { - name: base_path.clone() + new_path_str, - last_modified: object.last_modified.timestamp(), - }); - update_state.insert(object.location, object.last_modified); + if file_path.ends_with(marker_extension) { + // Scenario 3: New marker file added + info!( + "Source Object Marker has been added: {:?}, {:?}", + object.location, object.last_modified + ); + + // Remove base folder from relative path + let path = Path::new(&file_path); + let new_path = path + .strip_prefix(path.components().next().unwrap()) + .unwrap(); + let new_path_str = new_path.to_str().unwrap(); + + new_marker_files.push(FileInfo { + name: base_path.clone() + new_path_str, + last_modified: object.last_modified.timestamp(), + }); + update_state.insert(object.location, object.last_modified); + } else { + continue; + } } else { // Skip files that do not match the extension nor marker extension continue; @@ -166,8 +166,7 @@ impl ParquetTable { new_files.sort(); for file in &new_files { let marker_file_exist = is_marker_file_exist(new_marker_files.clone(), file); - let use_marker_file = marker_extension.is_empty(); - if !marker_file_exist && !use_marker_file { + if !marker_file_exist && self.table_config.marker_extension.is_some() { continue; } else { let file_path = ListingTableUrl::parse(&file.name) @@ -230,10 +229,7 @@ impl TableWatcher for ParquetTable { // Get the table state after snapshot let mut update_state = self.update_state.clone(); let extension = self.table_config.extension.clone(); - let marker_extension = match self.table_config.marker_file { - true => self.table_config.marker_extension.clone(), - false => String::new(), - }; + let marker_extension = self.table_config.marker_extension.clone(); let h = tokio::spawn(async move { // List objects in the S3 bucket with the specified prefix @@ -288,31 +284,33 @@ impl TableWatcher for ParquetTable { name: base_path.clone() + new_path_str, last_modified: object.last_modified.timestamp(), }); - if marker_extension.is_empty() { + if marker_extension.is_none() { update_state.insert(object.location, object.last_modified); } - } else if file_path.ends_with(marker_extension.as_str()) - && !marker_extension.is_empty() - { - // Scenario 3: New marker file added - info!( - "Source Object Marker has been added: {:?}, {:?}", - object.location, object.last_modified - ); - - // Remove base folder from relative path - let path = Path::new(&file_path); - let new_path = path - .strip_prefix(path.components().next().unwrap()) - .unwrap(); - let new_path_str = new_path.to_str().unwrap(); - - new_marker_files.push(FileInfo { - name: base_path.clone() + new_path_str, - last_modified: object.last_modified.timestamp(), - }); + } else if let Some(marker_extension) = marker_extension.as_deref() { + if file_path.ends_with(marker_extension) { + // Scenario 3: New marker file added + info!( + "Source Object Marker has been added: {:?}, {:?}", + object.location, object.last_modified + ); + + // Remove base folder from relative path + let path = Path::new(&file_path); + let new_path = path + .strip_prefix(path.components().next().unwrap()) + .unwrap(); + let new_path_str = new_path.to_str().unwrap(); + + new_marker_files.push(FileInfo { + name: base_path.clone() + new_path_str, + last_modified: object.last_modified.timestamp(), + }); - update_state.insert(object.location, object.last_modified); + update_state.insert(object.location, object.last_modified); + } else { + continue; + } } else { // Skip files that do not match the extension nor marker extension continue; @@ -323,8 +321,7 @@ impl TableWatcher for ParquetTable { new_files.sort(); for file in &new_files { let marker_file_exist = is_marker_file_exist(new_marker_files.clone(), file); - let use_marker_file = marker_extension.is_empty(); - if !marker_file_exist && !use_marker_file { + if !marker_file_exist && marker_extension.is_some() { continue; } else { let file_path = ListingTableUrl::parse(&file.name) diff --git a/dozer-ingestion/src/connectors/object_store/tests/test_utils.rs b/dozer-ingestion/src/connectors/object_store/tests/test_utils.rs index 559bddf86a..78a538ee7c 100644 --- a/dozer-ingestion/src/connectors/object_store/tests/test_utils.rs +++ b/dozer-ingestion/src/connectors/object_store/tests/test_utils.rs @@ -15,8 +15,7 @@ pub fn get_local_storage_config(typ: &str, prefix: &str) -> LocalStorage { config: Some(TableConfig::Parquet(ParquetConfig { extension: typ.to_string(), path: format!("all_types_{typ}"), - marker_file: false, - marker_extension: String::new(), + marker_extension: None, })), name: format!("all_types_{typ}"), }], @@ -29,8 +28,7 @@ pub fn get_local_storage_config(typ: &str, prefix: &str) -> LocalStorage { config: Some(TableConfig::Parquet(ParquetConfig { extension: typ.to_string(), path: format!("{prefix}_{typ}"), - marker_file: true, - marker_extension: String::from(".marker"), + marker_extension: Some(String::from(".marker")), })), name: format!("{prefix}_{typ}"), }], @@ -45,8 +43,7 @@ pub fn get_local_storage_config(typ: &str, prefix: &str) -> LocalStorage { config: Some(TableConfig::CSV(CsvConfig { extension: typ.to_string(), path: format!("all_types_{typ}"), - marker_file: false, - marker_extension: String::new(), + marker_extension: None, })), name: format!("all_types_{typ}"), }], @@ -59,8 +56,7 @@ pub fn get_local_storage_config(typ: &str, prefix: &str) -> LocalStorage { config: Some(TableConfig::CSV(CsvConfig { extension: typ.to_string(), path: format!("{prefix}_{typ}"), - marker_file: true, - marker_extension: String::from(".marker"), + marker_extension: Some(String::from(".marker")), })), name: format!("{prefix}_{typ}"), }], diff --git a/dozer-ingestion/src/connectors/postgres/snapshotter.rs b/dozer-ingestion/src/connectors/postgres/snapshotter.rs index a1d056adc3..ab237531b2 100644 --- a/dozer-ingestion/src/connectors/postgres/snapshotter.rs +++ b/dozer-ingestion/src/connectors/postgres/snapshotter.rs @@ -173,13 +173,7 @@ mod tests { #[serial] async fn test_connector_snapshotter_sync_tables_successfully_1_requested_table() { run_connector_test("postgres", |app_config| async move { - let config = app_config - .connections - .get(0) - .unwrap() - .config - .as_ref() - .unwrap(); + let config = &app_config.connections[0].config; let mut test_client = TestPostgresClient::new(config).await; @@ -229,13 +223,7 @@ mod tests { #[serial] async fn test_connector_snapshotter_sync_tables_successfully_not_match_table() { run_connector_test("postgres", |app_config| async move { - let config = app_config - .connections - .get(0) - .unwrap() - .config - .as_ref() - .unwrap(); + let config = &app_config.connections[0].config; let mut test_client = TestPostgresClient::new(config).await; @@ -281,13 +269,7 @@ mod tests { #[serial] async fn test_connector_snapshotter_sync_tables_successfully_table_not_exist() { run_connector_test("postgres", |app_config| async move { - let config = app_config - .connections - .get(0) - .unwrap() - .config - .as_ref() - .unwrap(); + let config = &app_config.connections[0].config; let mut rng = rand::thread_rng(); let table_name = format!("test_table_{}", rng.gen::()); diff --git a/dozer-ingestion/src/connectors/postgres/test_utils.rs b/dozer-ingestion/src/connectors/postgres/test_utils.rs index 90a97e1f26..44a2237ee1 100644 --- a/dozer-ingestion/src/connectors/postgres/test_utils.rs +++ b/dozer-ingestion/src/connectors/postgres/test_utils.rs @@ -11,14 +11,7 @@ use tokio_postgres::{error::DbError, Error as PostgresError, SimpleQueryMessage} use super::connection::client::Client; pub async fn get_client(app_config: Config) -> TestPostgresClient { - let config = app_config - .connections - .get(0) - .unwrap() - .config - .as_ref() - .unwrap(); - + let config = &app_config.connections[0].config; TestPostgresClient::new(config).await } @@ -62,9 +55,7 @@ pub async fn retry_drop_active_slot( } pub fn get_config(app_config: Config) -> tokio_postgres::Config { - if let Some(ConnectionConfig::Postgres(connection)) = - &app_config.connections.get(0).unwrap().config - { + if let ConnectionConfig::Postgres(connection) = &app_config.connections.get(0).unwrap().config { let config_replenished = connection.replenish().unwrap(); let mut config = tokio_postgres::Config::new(); config diff --git a/dozer-ingestion/src/connectors/postgres/tests/continue_replication_tests.rs b/dozer-ingestion/src/connectors/postgres/tests/continue_replication_tests.rs index 1406ac9712..a0efd36249 100644 --- a/dozer-ingestion/src/connectors/postgres/tests/continue_replication_tests.rs +++ b/dozer-ingestion/src/connectors/postgres/tests/continue_replication_tests.rs @@ -21,13 +21,7 @@ mod tests { #[serial] async fn test_connector_continue_replication() { run_connector_test("postgres", |app_config| async move { - let config = app_config - .connections - .get(0) - .unwrap() - .config - .as_ref() - .unwrap(); + let config = &app_config.connections[0].config; let conn_config = map_connection_config(config).unwrap(); let postgres_config = PostgresConfig { name: "test".to_string(), @@ -75,13 +69,7 @@ mod tests { #[serial] async fn test_connector_continue_replication_from_lsn() { run_connector_test("postgres", |app_config| async move { - let config = app_config - .connections - .get(0) - .unwrap() - .config - .as_ref() - .unwrap(); + let config = &app_config.connections[0].config; let mut test_client = TestPostgresClient::new(config).await; let mut rng = rand::thread_rng(); diff --git a/dozer-ingestion/src/connectors/snowflake/tests.rs b/dozer-ingestion/src/connectors/snowflake/tests.rs index 79cf650bab..5d6b02e0b0 100644 --- a/dozer-ingestion/src/connectors/snowflake/tests.rs +++ b/dozer-ingestion/src/connectors/snowflake/tests.rs @@ -22,11 +22,10 @@ use crate::connectors::snowflake::stream_consumer::StreamConsumer; #[ignore] async fn test_disabled_connector_and_read_from_stream() { run_connector_test("snowflake", |config| async move { - let connection = config.connections.get(0).unwrap(); - let ConnectionConfig::Snowflake(connection) = connection.config.as_ref().unwrap() else { + let ConnectionConfig::Snowflake(connection) = &config.connections[0].config else { panic!("Snowflake config expected"); }; - let source = config.sources.get(0).unwrap().clone(); + let source = config.sources[0].clone(); let env = create_environment_v3().map_err(|e| e.unwrap()).unwrap(); let client = Client::new(connection, &env); @@ -76,8 +75,7 @@ async fn test_disabled_connector_and_read_from_stream() { #[ignore] async fn test_disabled_connector_get_schemas_test() { run_connector_test("snowflake", |config| async move { - let connection = config.connections.get(0).unwrap(); - let ConnectionConfig::Snowflake(connection) = connection.config.as_ref().unwrap() else { + let ConnectionConfig::Snowflake(connection) = &config.connections[0].config else { panic!("Snowflake config expected"); }; let connector = SnowflakeConnector::new("snowflake".to_string(), connection.clone()); @@ -112,7 +110,7 @@ async fn test_disabled_connector_get_schemas_test() { .unwrap(); let schemas = connector.get_schemas(&table_infos).await.unwrap(); - let source_schema = schemas.get(0).unwrap().as_ref().unwrap(); + let source_schema = schemas[0].as_ref().unwrap(); for field in &source_schema.schema.fields { let expected_type = match field.name.as_str() { @@ -140,9 +138,8 @@ async fn test_disabled_connector_get_schemas_test() { #[tokio::test] #[ignore] async fn test_disabled_connector_missing_table_validator() { - run_connector_test("snowflake", |config| async move { - let connection = config.connections.get(0).unwrap(); - let connector = get_connector(connection.clone()).unwrap(); + run_connector_test("snowflake", |mut config| async move { + let connector = get_connector(config.connections.remove(0)).unwrap(); let not_existing_table = "not_existing_table".to_string(); let result = connector @@ -151,7 +148,7 @@ async fn test_disabled_connector_missing_table_validator() { assert!(matches!(result.unwrap_err(), TableNotFound(_))); - let existing_table = &config.sources.get(0).unwrap().table_name; + let existing_table = &config.sources[0].table_name; let table_infos = connector .list_columns(vec![TableIdentifier::from_table_name( existing_table.clone(), @@ -160,7 +157,7 @@ async fn test_disabled_connector_missing_table_validator() { .unwrap(); let result = connector.get_schemas(&table_infos).await.unwrap(); - assert!(result.get(0).unwrap().is_ok()); + assert!(result[0].is_ok()); }) .await } @@ -168,10 +165,9 @@ async fn test_disabled_connector_missing_table_validator() { #[tokio::test] #[ignore] async fn test_disabled_connector_is_stream_created() { - run_connector_test("snowflake", |config| async move { - let connection = config.connections.get(0).unwrap(); - let snowflake_config = match connection.config.as_ref().unwrap() { - ConnectionConfig::Snowflake(snowflake_config) => snowflake_config.clone(), + run_connector_test("snowflake", |mut config| async move { + let snowflake_config = match config.connections.remove(0).config { + ConnectionConfig::Snowflake(snowflake_config) => snowflake_config, _ => { panic!("Snowflake config expected"); } diff --git a/dozer-ingestion/src/errors.rs b/dozer-ingestion/src/errors.rs index 32b0425704..08497a2b13 100644 --- a/dozer-ingestion/src/errors.rs +++ b/dozer-ingestion/src/errors.rs @@ -39,9 +39,6 @@ use crate::connectors::mongodb::MongodbConnectorError; #[derive(Error, Debug)] pub enum ConnectorError { - #[error("Missing `config` for connector {0}")] - MissingConfiguration(String), - #[error("Failed to map configuration: {0}")] WrongConnectionConfiguration(DeserializationError), @@ -51,8 +48,8 @@ pub enum ConnectorError { #[error("Failed to map configuration: {0}")] UnableToInferSchema(DataFusionError), - #[error("Unsupported grpc adapter: {0} {1}")] - UnsupportedGrpcAdapter(String, String), + #[error("Unsupported grpc adapter: {0} {1:?}")] + UnsupportedGrpcAdapter(String, Option), #[error("Arrow error: {0}")] Arrow(#[from] ArrowError), @@ -154,9 +151,6 @@ pub enum ConfigurationError { } #[derive(Error, Debug)] pub enum NestedDozerConnectorError { - #[error("Missing `grpc` field in `config`")] - MissingGrpcConfig, - #[error("Failed to connect to upstream dozer at {0}: {1:?}")] ConnectionError(String, #[source] dozer_types::tonic::transport::Error), diff --git a/dozer-ingestion/src/test_util.rs b/dozer-ingestion/src/test_util.rs index c22c324527..d454371bfe 100644 --- a/dozer-ingestion/src/test_util.rs +++ b/dozer-ingestion/src/test_util.rs @@ -31,7 +31,7 @@ pub async fn run_connector_test< let dozer_config = dozer_types::serde_yaml::from_str::(&dozer_config).unwrap(); let connection = dozer_config.connections.get(0).unwrap(); - if let Some(ConnectionConfig::Postgres(connection_config)) = connection.config.clone() { + if let ConnectionConfig::Postgres(connection_config) = connection.config.clone() { let mut config = tokio_postgres::Config::new(); let replenished_config = connection_config.replenish().unwrap(); config diff --git a/dozer-ingestion/src/tests/cases/postgres/dozer-config.yaml b/dozer-ingestion/src/tests/cases/postgres/dozer-config.yaml index 5ed73f09e5..404c0e313c 100644 --- a/dozer-ingestion/src/tests/cases/postgres/dozer-config.yaml +++ b/dozer-ingestion/src/tests/cases/postgres/dozer-config.yaml @@ -46,8 +46,7 @@ endpoints: index: primary_key: - id - - id: null - name: stocks_meta + - name: stocks_meta path: /stocks-meta # Direct from source table_name: stocks_meta diff --git a/dozer-ingestion/tests/test_suite/connectors/dozer.rs b/dozer-ingestion/tests/test_suite/connectors/dozer.rs index e514f3a737..9b667dccfa 100644 --- a/dozer-ingestion/tests/test_suite/connectors/dozer.rs +++ b/dozer-ingestion/tests/test_suite/connectors/dozer.rs @@ -15,12 +15,10 @@ use dozer_types::grpc_types::types::Record; use dozer_types::ingestion_types::GrpcConfig; use dozer_types::log::info; use dozer_types::models::api_endpoint::ApiEndpoint; -use dozer_types::models::config::{default_cache_dir, default_home_dir}; use dozer_types::models::source::Source; use dozer_types::types::{Field, FieldDefinition, FieldType}; use dozer_types::{ ingestion_types::{NestedDozerConfig, NestedDozerLogOptions}, - models::api_config::AppGrpcOptions, serde_json, }; @@ -190,23 +188,19 @@ async fn create_nested_dozer_server( std::fs::write(&dozer_config_path, DOZER_CONFIG).expect("Failed to write dozer config"); let grpc_config = GrpcConfig { - host: "0.0.0.0".to_owned(), - port: 8085, - schemas: Some(dozer_types::ingestion_types::GrpcConfigSchemas::Inline( - schema_string, - )), - adapter: "default".to_owned(), + host: Some("0.0.0.0".to_owned()), + port: Some(8085), + schemas: dozer_types::ingestion_types::GrpcConfigSchemas::Inline(schema_string), + adapter: Some("default".to_owned()), }; let config = dozer_types::models::config::Config { version: 1, app_name: "nested-dozer-connector-test".to_owned(), - home_dir: default_home_dir(), - cache_dir: default_cache_dir(), + home_dir: None, + cache_dir: None, connections: vec![dozer_types::models::connection::Connection { - config: Some(dozer_types::models::connection::ConnectionConfig::Grpc( - grpc_config, - )), + config: dozer_types::models::connection::ConnectionConfig::Grpc(grpc_config), name: "ingest".to_owned(), }], sources: vec![Source { @@ -215,16 +209,16 @@ async fn create_nested_dozer_server( columns: vec![], connection: "ingest".to_owned(), schema: None, - refresh_config: None, + refresh_config: Default::default(), }], endpoints: vec![ApiEndpoint { name: table_name.to_owned(), path: "/test".to_owned(), table_name: table_name.clone(), - index: None, - conflict_resolution: None, + index: Default::default(), + conflict_resolution: Default::default(), version: None, - log_reader_options: None, + log_reader_options: Default::default(), }], ..Default::default() }; @@ -239,15 +233,12 @@ async fn create_nested_dozer_server( let client = try_connect_ingest("http://localhost:8085".to_owned()).await; let connector = NestedDozerConnector::new(NestedDozerConfig { - grpc: Some(AppGrpcOptions { - port: 50053, - host: "localhost".to_owned(), - }), - log_options: Some(NestedDozerLogOptions { - batch_size: 1, - timeout_in_millis: 3000, - buffer_size: 1, - }), + url: "http://localhost:50053".to_owned(), + log_options: NestedDozerLogOptions { + batch_size: Some(1), + timeout_in_millis: Some(3000), + buffer_size: Some(1), + }, }); let test = DozerConnectorTest { diff --git a/dozer-ingestion/tests/test_suite/connectors/object_store/local_storage.rs b/dozer-ingestion/tests/test_suite/connectors/object_store/local_storage.rs index cc1f24a698..4689590aa0 100644 --- a/dozer-ingestion/tests/test_suite/connectors/object_store/local_storage.rs +++ b/dozer-ingestion/tests/test_suite/connectors/object_store/local_storage.rs @@ -93,8 +93,7 @@ fn create_connector( config: Some(TableConfig::Parquet(ParquetConfig { path: table_name.to_string(), extension: ".parquet".to_string(), - marker_file: false, - marker_extension: String::new(), + marker_extension: None, })), name: table_name, }], diff --git a/dozer-log/src/replication/persist.rs b/dozer-log/src/replication/persist.rs index 75b5b00d71..177d4391d8 100644 --- a/dozer-log/src/replication/persist.rs +++ b/dozer-log/src/replication/persist.rs @@ -12,7 +12,7 @@ pub async fn create_data_storage( data_dir: String, ) -> Result<(Box, String), storage::Error> { match storage_config { - DataStorage::Local(()) => Ok(( + DataStorage::Local => Ok(( Box::new(LocalStorage::new(data_dir).await?), String::default(), )), diff --git a/dozer-sql/expression/src/builder.rs b/dozer-sql/expression/src/builder.rs index d6c3c4c062..c00d6d4e15 100644 --- a/dozer-sql/expression/src/builder.rs +++ b/dozer-sql/expression/src/builder.rs @@ -471,7 +471,7 @@ impl ExpressionBuilder { let udf_type = udfs.iter().find(|udf| udf.name == function_name); if let Some(udf_type) = udf_type { return match &udf_type.config { - Some(UdfType::Onnx(config)) => { + UdfType::Onnx(config) => { #[cfg(feature = "onnx")] { self.parse_onnx_udf( @@ -489,7 +489,6 @@ impl ExpressionBuilder { Err(Error::OnnxNotEnabled) } } - None => Err(Error::UdfConfigMissing(function_name.clone())), }; } diff --git a/dozer-sql/expression/src/error.rs b/dozer-sql/expression/src/error.rs index 0576d32776..786fe30005 100644 --- a/dozer-sql/expression/src/error.rs +++ b/dozer-sql/expression/src/error.rs @@ -18,8 +18,6 @@ pub enum Error { UnsupportedFunctionArg(FunctionArg), #[error("Invalid ident: {}", .0.iter().map(|ident| ident.value.as_str()).collect::>().join("."))] InvalidIdent(Vec), - #[error("Udf is defined but missing with config: {0}")] - UdfConfigMissing(String), #[error("Unknown function: {0}")] UnknownFunction(String), #[error("Missing leading field in interval")] diff --git a/dozer-sql/src/builder.rs b/dozer-sql/src/builder.rs index f6fe18806a..55915805ec 100644 --- a/dozer-sql/src/builder.rs +++ b/dozer-sql/src/builder.rs @@ -299,7 +299,8 @@ fn select_to_pipeline( pipeline .flags() .enable_probabilistic_optimizations - .in_aggregations, + .in_aggregations + .unwrap_or(false), query_ctx.udfs.clone(), ); @@ -498,7 +499,11 @@ fn set_to_pipeline( let set_proc_fac = SetProcessorFactory::new( gen_set_name.clone(), set_quantifier, - pipeline.flags().enable_probabilistic_optimizations.in_sets, + pipeline + .flags() + .enable_probabilistic_optimizations + .in_sets + .unwrap_or(false), ); pipeline.add_processor(Box::new(set_proc_fac), &gen_set_name, vec![]); diff --git a/dozer-sql/src/pipeline_builder/join_builder.rs b/dozer-sql/src/pipeline_builder/join_builder.rs index 46d97868da..f743c23754 100644 --- a/dozer-sql/src/pipeline_builder/join_builder.rs +++ b/dozer-sql/src/pipeline_builder/join_builder.rs @@ -55,7 +55,11 @@ pub(crate) fn insert_join_to_pipeline( left_name_or_alias.clone(), right_name_or_alias, join.join_operator.clone(), - pipeline.flags().enable_probabilistic_optimizations.in_joins, + pipeline + .flags() + .enable_probabilistic_optimizations + .in_joins + .unwrap_or(false), ); let mut pipeline_entry_points = vec![]; diff --git a/dozer-tests/src/dozer_test_client.rs b/dozer-tests/src/dozer_test_client.rs index 37becb63af..65ec9c63a3 100644 --- a/dozer-tests/src/dozer_test_client.rs +++ b/dozer-tests/src/dozer_test_client.rs @@ -36,13 +36,10 @@ async fn main() { // Modify dozer config to route to the dozer API host. let mut dozer_config = case.dozer_config; - let mut api = dozer_config.api.unwrap_or_default(); - let mut rest = api.rest.unwrap_or_default(); - rest.host = args.dozer_api_host.clone(); - api.rest = Some(rest); - let mut grpc = api.grpc.unwrap_or_default(); - grpc.host = args.dozer_api_host; - api.grpc = Some(grpc); - dozer_config.api = Some(api); + let api = &mut dozer_config.api; + let rest = &mut api.rest; + rest.host = Some(args.dozer_api_host.clone()); + let grpc = &mut api.grpc; + grpc.host = Some(args.dozer_api_host); run_test_client(dozer_config, &expectations).await; } diff --git a/dozer-tests/src/e2e_tests/checker/client.rs b/dozer-tests/src/e2e_tests/checker/client.rs index b80bb8e120..546daf1461 100644 --- a/dozer-tests/src/e2e_tests/checker/client.rs +++ b/dozer-tests/src/e2e_tests/checker/client.rs @@ -14,8 +14,9 @@ use dozer_types::{ }, }, models::{ - api_config::{default_api_grpc, default_api_rest}, + api_config::{default_grpc_port, default_host, default_rest_port}, config::Config, + flags::default_dynamic, }, types::{FieldDefinition, FieldType, DATE_FORMAT}, }; @@ -32,17 +33,21 @@ pub struct Client { impl Client { pub async fn new(config: Config) -> Self { - let api = config.api.as_ref(); + let api = &config.api; - let rest = api - .and_then(|api| api.rest.clone()) - .unwrap_or_else(default_api_rest); - let rest_endpoint = format!("http://{}:{}", rest.host, rest.port); + let rest = &api.rest; + let rest_endpoint = format!( + "http://{}:{}", + rest.host.clone().unwrap_or_else(default_host), + rest.port.unwrap_or_else(default_rest_port) + ); - let grpc = api - .and_then(|api| api.grpc.clone()) - .unwrap_or_else(default_api_grpc); - let grpc_endpoint_string = format!("http://{}:{}", grpc.host, grpc.port); + let grpc = &api.grpc; + let grpc_endpoint_string = format!( + "http://{}:{}", + grpc.host.clone().unwrap_or_else(default_host), + grpc.port.unwrap_or_else(default_grpc_port) + ); let grpc_endpoint = Endpoint::from_shared(grpc_endpoint_string.clone()) .unwrap_or_else(|e| panic!("Invalid grpc endpoint {grpc_endpoint_string}: {e}")); @@ -104,7 +109,7 @@ impl Client { } // gRPC health. - let services = if self.config.flags.clone().unwrap_or_default().dynamic { + let services = if self.config.flags.dynamic.unwrap_or_else(default_dynamic) { vec!["common", "typed", ""] } else { vec!["common", ""] diff --git a/dozer-tests/src/e2e_tests/runner/local.rs b/dozer-tests/src/e2e_tests/runner/local.rs index 2c4dcae099..99510d68a4 100644 --- a/dozer-tests/src/e2e_tests/runner/local.rs +++ b/dozer-tests/src/e2e_tests/runner/local.rs @@ -1,6 +1,6 @@ use std::{path::Path, process::Command}; -use dozer_types::constants::LOCK_FILE; +use dozer_types::{constants::LOCK_FILE, models::config::default_home_dir}; use dozer_utils::{ process::{run_command, run_docker_compose}, Cleanup, @@ -49,7 +49,12 @@ impl Runner { } // Start dozer. - cleanups.push(Cleanup::RemoveDirectory(case.dozer_config.home_dir.clone())); + cleanups.push(Cleanup::RemoveDirectory( + case.dozer_config + .home_dir + .clone() + .unwrap_or_else(default_home_dir), + )); cleanups.push(Cleanup::RemoveFile(LOCK_FILE.to_owned())); cleanups.extend(spawn_dozer(&self.dozer_bin, &case.dozer_config_path)); @@ -70,7 +75,12 @@ impl Runner { &docker_compose.connections_healthy_service_name, )); } - cleanups.push(Cleanup::RemoveDirectory(case.dozer_config.home_dir.clone())); + cleanups.push(Cleanup::RemoveDirectory( + case.dozer_config + .home_dir + .clone() + .unwrap_or_else(default_home_dir), + )); cleanups.push(Cleanup::RemoveFile(LOCK_FILE.to_owned())); (Command::new(&self.dozer_bin), cleanups) diff --git a/dozer-tests/src/e2e_tests/runner/running_env.rs b/dozer-tests/src/e2e_tests/runner/running_env.rs index eb832dd6c8..14df9f0fbf 100644 --- a/dozer-tests/src/e2e_tests/runner/running_env.rs +++ b/dozer-tests/src/e2e_tests/runner/running_env.rs @@ -331,10 +331,7 @@ fn write_dozer_config_for_running_in_docker_compose( }; for connection in &mut config.connections { - let config = connection - .config - .as_mut() - .expect("Connection should always have config"); + let config = &mut connection.config; match config { ConnectionConfig::Postgres(postgres) => { diff --git a/dozer-tests/src/init.rs b/dozer-tests/src/init.rs index d9c141352d..48d3fccb2c 100644 --- a/dozer-tests/src/init.rs +++ b/dozer-tests/src/init.rs @@ -20,7 +20,7 @@ fn download(folder_name: &str) { static INIT: Once = Once::new(); pub fn init() { INIT.call_once(|| { - dozer_tracing::init_telemetry(None, None); + dozer_tracing::init_telemetry(None, &Default::default()); download("actor"); dozer_cli::set_panic_hook(); diff --git a/dozer-tests/src/tests/e2e/mod.rs b/dozer-tests/src/tests/e2e/mod.rs index 2f996642de..7c9a5ed8c2 100644 --- a/dozer-tests/src/tests/e2e/mod.rs +++ b/dozer-tests/src/tests/e2e/mod.rs @@ -10,7 +10,12 @@ use dozer_types::{ common::common_grpc_service_client::CommonGrpcServiceClient, ingest::ingest_service_client::IngestServiceClient, }, - models::{api_config::default_api_grpc, config::Config, connection::ConnectionConfig}, + ingestion_types::{default_ingest_host, default_ingest_port}, + models::{ + api_config::{default_grpc_port, default_host}, + config::Config, + connection::ConnectionConfig, + }, serde_yaml, }; use tempdir::TempDir; @@ -34,20 +39,24 @@ impl DozerE2eTest { let temp_dir = TempDir::new("tests").unwrap(); let config = serde_yaml::from_str::(config_str).unwrap(); - let api_grpc = config - .api - .as_ref() - .and_then(|api| api.grpc.clone()) - .unwrap_or_else(default_api_grpc); - let common_service_url = format!("http://{}:{}", api_grpc.host, api_grpc.port); + let api_grpc = &config.api.grpc; + let common_service_url = format!( + "http://{}:{}", + api_grpc.host.clone().unwrap_or_else(default_host), + api_grpc.port.unwrap_or_else(default_grpc_port), + ); let mut ingest_service_url = None; for connection in &config.connections { - if let Some(ConnectionConfig::Grpc(config)) = &connection.config { + if let ConnectionConfig::Grpc(config) = &connection.config { if ingest_service_url.is_some() { panic!("Found more than one ingest service"); } - ingest_service_url = Some(format!("http://{}:{}", config.host, config.port)); + ingest_service_url = Some(format!( + "http://{}:{}", + config.host.clone().unwrap_or_else(default_ingest_host), + config.port.unwrap_or_else(default_ingest_port), + )); } } diff --git a/dozer-tracing/src/exporter.rs b/dozer-tracing/src/exporter.rs index f53c398fa6..d769192cbb 100644 --- a/dozer-tracing/src/exporter.rs +++ b/dozer-tracing/src/exporter.rs @@ -13,7 +13,7 @@ use crate::helper::{self, events_schema, spans_schema}; use dozer_types::arrow_types::from_arrow::serialize_record_batch; use dozer_types::arrow_types::to_arrow::map_record_to_arrow; use dozer_types::grpc_types::ingest::IngestArrowRequest; -use dozer_types::models::telemetry::DozerTelemetryConfig; +use dozer_types::models::telemetry::{default_ingest_address, DozerTelemetryConfig}; use dozer_types::tonic; use opentelemetry::sdk::export::trace::SpanData; use std::sync::atomic::Ordering; @@ -40,7 +40,11 @@ impl SpanExporter for DozerExporter { batch: Vec, ) -> futures_util::future::BoxFuture<'static, opentelemetry::sdk::export::trace::ExportResult> { - let endpoint = self.config.endpoint.clone(); + let endpoint = self + .config + .endpoint + .clone() + .unwrap_or_else(default_ingest_address); let seq_no = self.seq_no.clone(); Box::pin(async move { let res = ingest_span(batch, endpoint, seq_no).await; diff --git a/dozer-tracing/src/telemetry.rs b/dozer-tracing/src/telemetry.rs index 37b95fcef1..2b3deeb139 100644 --- a/dozer-tracing/src/telemetry.rs +++ b/dozer-tracing/src/telemetry.rs @@ -2,7 +2,7 @@ use std::time::Duration; use dozer_types::log::{debug, error}; use dozer_types::models::telemetry::{ - DozerTelemetryConfig, TelemetryConfig, TelemetryTraceConfig, XRayConfig, + default_sample_ratio, DozerTelemetryConfig, TelemetryConfig, TelemetryTraceConfig, XRayConfig, }; use dozer_types::tracing::{self, Metadata, Subscriber}; use metrics_exporter_prometheus::PrometheusBuilder; @@ -20,7 +20,7 @@ use tracing_subscriber::{filter, fmt, EnvFilter, Layer}; use crate::exporter::DozerExporter; // Init telemetry by setting a global handler -pub fn init_telemetry(app_name: Option<&str>, telemetry_config: Option) { +pub fn init_telemetry(app_name: Option<&str>, telemetry_config: &TelemetryConfig) { // log errors from open telemetry opentelemetry::global::set_error_handler(|e| { error!("OpenTelemetry error: {}", e); @@ -29,15 +29,13 @@ pub fn init_telemetry(app_name: Option<&str>, telemetry_config: Option( app_name: Option<&str>, - telemetry_config: Option, + telemetry_config: &TelemetryConfig, closure: impl FnOnce() -> T, ) -> T { - let subscriber = create_subscriber(app_name, telemetry_config.as_ref(), false); + let subscriber = create_subscriber(app_name, telemetry_config, false); dozer_types::tracing::subscriber::with_default(subscriber, closure) } fn create_subscriber( app_name: Option<&str>, - telemetry_config: Option<&TelemetryConfig>, + telemetry_config: &TelemetryConfig, init_console_subscriber: bool, ) -> impl Subscriber { let app_name = app_name.unwrap_or("dozer"); @@ -78,26 +76,23 @@ fn create_subscriber( #[cfg(not(feature = "tokio-console"))] let _ = init_console_subscriber; - let layers = telemetry_config.map_or((None, None), |c| { - let trace_filter = EnvFilter::try_from_env("DOZER_TRACE_FILTER") - .or_else(|_| EnvFilter::try_new("dozer=trace")) - .unwrap(); - match &c.trace { - None => (None, None), - Some(TelemetryTraceConfig::Dozer(config)) => ( - Some(get_dozer_tracer(config).with_filter(trace_filter)), - None, + let trace_filter = EnvFilter::try_from_env("DOZER_TRACE_FILTER") + .unwrap_or_else(|_| EnvFilter::try_new("dozer=trace").unwrap()); + let layers = match &telemetry_config.trace { + None => (None, None), + Some(TelemetryTraceConfig::Dozer(config)) => ( + Some(get_dozer_tracer(config).with_filter(trace_filter)), + None, + ), + Some(TelemetryTraceConfig::XRay(config)) => ( + None, + Some( + get_xray_tracer(app_name, config).with_filter(filter::filter_fn( + |metadata: &Metadata| metadata.level() == &tracing::Level::ERROR, + )), ), - Some(TelemetryTraceConfig::XRay(config)) => ( - None, - Some( - get_xray_tracer(app_name, config).with_filter(filter::filter_fn( - |metadata: &Metadata| metadata.level() == &tracing::Level::ERROR, - )), - ), - ), - } - }); + ), + }; let stdout_is_tty = atty::is(atty::Stream::Stdout); let subscriber = tracing_subscriber::registry(); @@ -156,7 +151,7 @@ where + dozer_types::tracing::Subscriber, { let builder = sdk::trace::TracerProvider::builder(); - let sample_percent = config.sample_percent as f64 / 100.0; + let sample_percent = config.sample_percent.unwrap_or_else(default_sample_ratio) as f64 / 100.0; let exporter = DozerExporter::new(config.clone()); let batch_config = BatchConfig::default() .with_max_concurrent_exports(100000) diff --git a/dozer-types/src/ingestion_types.rs b/dozer-types/src/ingestion_types.rs index cdd6f08b19..fdf8251619 100644 --- a/dozer-types/src/ingestion_types.rs +++ b/dozer-types/src/ingestion_types.rs @@ -4,7 +4,7 @@ use std::fmt::Debug; use serde::{Deserialize, Serialize}; -use crate::{models::api_config::AppGrpcOptions, node::OpIdentifier, types::Operation}; +use crate::{node::OpIdentifier, types::Operation}; #[derive(Clone, Debug, PartialEq)] /// All possible kinds of `IngestionMessage`. @@ -41,44 +41,33 @@ pub struct EthFilter { #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)] pub struct GrpcConfig { - #[serde(default = "default_ingest_host")] - pub host: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub host: Option, - #[serde(default = "default_ingest_port")] - pub port: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, - pub schemas: Option, + pub schemas: GrpcConfigSchemas, - #[serde(default = "default_grpc_adapter")] - pub adapter: String, -} -impl Default for GrpcConfig { - fn default() -> Self { - Self { - host: default_ingest_host(), - port: default_ingest_port(), - schemas: None, - adapter: default_grpc_adapter(), - } - } + #[serde(skip_serializing_if = "Option::is_none")] + pub adapter: Option, } -fn default_grpc_adapter() -> String { +pub fn default_grpc_adapter() -> String { "default".to_owned() } -fn default_ingest_host() -> String { +pub fn default_ingest_host() -> String { "0.0.0.0".to_owned() } -fn default_ingest_port() -> u32 { +pub fn default_ingest_port() -> u32 { 8085 } #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)] pub enum GrpcConfigSchemas { Inline(String), - Path(String), } @@ -112,11 +101,11 @@ pub struct EthTraceConfig { pub to_block: Option, - #[serde(default = "default_batch_size")] - pub batch_size: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub batch_size: Option, } -fn default_batch_size() -> u64 { +pub fn default_batch_size() -> u64 { 3 } @@ -269,11 +258,8 @@ pub struct CsvConfig { pub extension: String, - #[serde(default = "default_false")] - pub marker_file: bool, - - #[serde(default = "default_marker")] - pub marker_extension: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub marker_extension: Option, } #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)] @@ -287,11 +273,8 @@ pub struct ParquetConfig { pub extension: String, - #[serde(default = "default_false")] - pub marker_file: bool, - - #[serde(default = "default_marker")] - pub marker_extension: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub marker_extension: Option, } #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)] @@ -378,48 +361,33 @@ pub struct MySQLConfig { pub server_id: Option, } -fn default_false() -> bool { - false -} - -fn default_marker() -> String { - String::from(".marker") -} - #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)] pub struct NestedDozerConfig { - pub grpc: Option, - - pub log_options: Option, + pub url: String, + #[serde(default)] + pub log_options: NestedDozerLogOptions, } -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)] +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema, Default)] pub struct NestedDozerLogOptions { - #[serde(default = "default_log_batch_size")] - pub batch_size: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub batch_size: Option, - #[serde(default = "default_timeout")] - pub timeout_in_millis: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout_in_millis: Option, - #[serde(default = "default_buffer_size")] - pub buffer_size: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub buffer_size: Option, } -fn default_log_batch_size() -> u32 { +pub fn default_log_batch_size() -> u32 { 30 } -fn default_timeout() -> u32 { - 1000 -} -fn default_buffer_size() -> u32 { +pub fn default_timeout() -> u32 { 1000 } -pub fn default_log_options() -> NestedDozerLogOptions { - NestedDozerLogOptions { - batch_size: default_log_batch_size(), - timeout_in_millis: default_timeout(), - buffer_size: default_buffer_size(), - } +pub fn default_buffer_size() -> u32 { + 1000 } diff --git a/dozer-types/src/models/api_config.rs b/dozer-types/src/models/api_config.rs index 6a56f48177..f4948fedd5 100644 --- a/dozer-types/src/models/api_config.rs +++ b/dozer-types/src/models/api_config.rs @@ -1,117 +1,94 @@ use crate::constants::DEFAULT_DEFAULT_MAX_NUM_RECORDS; -use super::api_security::ApiSecurity; +use super::{api_security::ApiSecurity, equal_default}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; + #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, JsonSchema, Default)] +#[serde(deny_unknown_fields)] pub struct ApiConfig { #[serde(skip_serializing_if = "Option::is_none")] /// The security configuration for the API; Default: None pub api_security: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub rest: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub rest: RestApiOptions, - #[serde(skip_serializing_if = "Option::is_none")] - pub grpc: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub grpc: GrpcApiOptions, - #[serde(skip_serializing_if = "Option::is_none")] - pub app_grpc: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub app_grpc: AppGrpcOptions, - #[serde(default = "default_default_max_num_records")] + #[serde(skip_serializing_if = "Option::is_none")] // max records to be returned from the endpoints - pub default_max_num_records: u32, + pub default_max_num_records: Option, } + #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, JsonSchema, Default)] +#[serde(deny_unknown_fields)] pub struct RestApiOptions { - #[serde(default = "default_rest_port")] - pub port: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, - #[serde(default = "default_host")] - pub host: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub host: Option, - #[serde(default = "default_cors")] - pub cors: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub cors: Option, - #[serde(default = "default_enabled")] - pub enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub enabled: Option, } + #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, JsonSchema, Default)] +#[serde(deny_unknown_fields)] pub struct GrpcApiOptions { - #[serde(default = "default_grpc_port")] - pub port: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, - #[serde(default = "default_host")] - pub host: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub host: Option, - #[serde(default = "default_cors")] - pub cors: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub cors: Option, - #[serde(default = "default_enable_web")] - pub web: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub web: Option, - #[serde(default = "default_enabled")] - pub enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub enabled: Option, } #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema, Default)] +#[serde(deny_unknown_fields)] pub struct AppGrpcOptions { - #[serde(default = "default_app_grpc_port")] - pub port: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, - #[serde(default = "default_app_grpc_host")] - pub host: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub host: Option, } -fn default_app_grpc_port() -> u32 { +pub fn default_app_grpc_port() -> u32 { 50053 } -fn default_app_grpc_host() -> String { + +pub fn default_app_grpc_host() -> String { "0.0.0.0".to_owned() } pub fn default_default_max_num_records() -> u32 { DEFAULT_DEFAULT_MAX_NUM_RECORDS as u32 } -pub fn default_app_grpc() -> AppGrpcOptions { - AppGrpcOptions { - port: default_app_grpc_port(), - host: default_app_grpc_host(), - } -} -pub fn default_api_rest() -> RestApiOptions { - RestApiOptions { - port: default_rest_port(), - host: default_host(), - cors: default_cors(), - enabled: true, - } -} -pub fn default_api_grpc() -> GrpcApiOptions { - GrpcApiOptions { - port: default_grpc_port(), - host: default_host(), - cors: default_cors(), - web: default_enable_web(), - enabled: true, - } -} -fn default_grpc_port() -> u32 { +pub fn default_grpc_port() -> u16 { 50051 } -fn default_rest_port() -> u32 { + +pub fn default_rest_port() -> u16 { 8080 } -fn default_enable_web() -> bool { - true -} -fn default_cors() -> bool { - true -} -fn default_enabled() -> bool { - true -} -fn default_host() -> String { +pub fn default_host() -> String { "0.0.0.0".to_owned() } diff --git a/dozer-types/src/models/api_endpoint.rs b/dozer-types/src/models/api_endpoint.rs index b127ab88d4..999739f7f0 100644 --- a/dozer-types/src/models/api_endpoint.rs +++ b/dozer-types/src/models/api_endpoint.rs @@ -1,99 +1,86 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] +use super::equal_default; + +#[derive(Debug, Serialize, Deserialize, JsonSchema, Default, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub struct ApiIndex { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub primary_key: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub secondary: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub secondary: SecondaryIndexConfig, } -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Default, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub struct SecondaryIndexConfig { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub skip_default: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub create: Vec, -} - -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] -pub struct CreateSecondaryIndex { - pub index: Option, + pub create: Vec, } -#[derive(Debug, Serialize, JsonSchema, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub enum SecondaryIndex { SortedInverted(SortedInverted), - FullText(FullText), } -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub struct SortedInverted { pub fields: Vec, } -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub struct FullText { pub field: String, } -#[derive(Debug, Serialize, JsonSchema, Deserialize, Eq, PartialEq, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone, Copy, Default)] +#[serde(deny_unknown_fields)] pub enum OnInsertResolutionTypes { - Nothing(()), - - Update(()), - - Panic(()), + #[default] + Nothing, + Update, + Panic, } -impl Default for OnInsertResolutionTypes { - fn default() -> Self { - OnInsertResolutionTypes::Nothing(()) - } -} - -#[derive(Debug, Serialize, JsonSchema, Deserialize, Eq, PartialEq, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone, Copy, Default)] +#[serde(deny_unknown_fields)] pub enum OnUpdateResolutionTypes { - Nothing(()), - - Upsert(()), - - Panic(()), + #[default] + Nothing, + Upsert, + Panic, } -impl Default for OnUpdateResolutionTypes { - fn default() -> Self { - OnUpdateResolutionTypes::Nothing(()) - } -} - -#[derive(Debug, Serialize, JsonSchema, Deserialize, Eq, PartialEq, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone, Copy, Default)] +#[serde(deny_unknown_fields)] pub enum OnDeleteResolutionTypes { - Nothing(()), - - Panic(()), + #[default] + Nothing, + Panic, } -impl Default for OnDeleteResolutionTypes { - fn default() -> Self { - OnDeleteResolutionTypes::Nothing(()) - } -} - -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Default, Eq, PartialEq, Clone, Copy)] +#[serde(deny_unknown_fields)] pub struct ConflictResolution { - pub on_insert: Option, - - pub on_update: Option, - - pub on_delete: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub on_insert: OnInsertResolutionTypes, + #[serde(default, skip_serializing_if = "equal_default")] + pub on_update: OnUpdateResolutionTypes, + #[serde(default, skip_serializing_if = "equal_default")] + pub on_delete: OnDeleteResolutionTypes, } -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Default, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub struct LogReaderOptions { #[serde(skip_serializing_if = "Option::is_none")] pub batch_size: Option, @@ -105,7 +92,8 @@ pub struct LogReaderOptions { pub buffer_size: Option, } -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Default, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub struct ApiEndpoint { pub name: String, @@ -115,16 +103,17 @@ pub struct ApiEndpoint { /// path of endpoint - e.g: /stocks pub path: String, - pub index: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub index: ApiIndex, - #[serde(skip_serializing_if = "Option::is_none")] - pub conflict_resolution: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub conflict_resolution: ConflictResolution, #[serde(skip_serializing_if = "Option::is_none")] pub version: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub log_reader_options: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub log_reader_options: LogReaderOptions, } pub fn default_log_reader_batch_size() -> u32 { diff --git a/dozer-types/src/models/api_security.rs b/dozer-types/src/models/api_security.rs index d0026a4924..5fffc5d5dd 100644 --- a/dozer-types/src/models/api_security.rs +++ b/dozer-types/src/models/api_security.rs @@ -1,7 +1,9 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -#[doc = r"The security model option for the API"] + +/// The security model option for the API #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)] +#[serde(deny_unknown_fields)] pub enum ApiSecurity { /// Initialize with a JWT_SECRET Jwt(String), diff --git a/dozer-types/src/models/app_config.rs b/dozer-types/src/models/app_config.rs index ce4fc992ac..dc1308b2f6 100644 --- a/dozer-types/src/models/app_config.rs +++ b/dozer-types/src/models/app_config.rs @@ -1,30 +1,30 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use super::equal_default; + #[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq, Default)] +#[serde(deny_unknown_fields)] pub struct AppConfig { /// Pipeline buffer size - #[serde(skip_serializing_if = "Option::is_none")] pub app_buffer_size: Option, /// Commit size - #[serde(skip_serializing_if = "Option::is_none")] pub commit_size: Option, /// Commit timeout - #[serde(skip_serializing_if = "Option::is_none")] pub commit_timeout: Option, + /// Maximum number of pending persisting requests. #[serde(skip_serializing_if = "Option::is_none")] pub persist_queue_capacity: Option, /// The storage to use for the log. - - #[serde(skip_serializing_if = "Option::is_none")] - pub data_storage: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub data_storage: DataStorage, /// How many errors we can tolerate before bringing down the app. #[serde(skip_serializing_if = "Option::is_none")] @@ -39,26 +39,21 @@ pub struct AppConfig { pub max_interval_before_persist_in_seconds: Option, } -#[derive(Debug, JsonSchema, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, JsonSchema, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(deny_unknown_fields)] pub enum DataStorage { - Local(()), - + #[default] + Local, S3(S3Storage), } #[derive(Debug, JsonSchema, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct S3Storage { pub region: String, - pub bucket_name: String, } -impl Default for DataStorage { - fn default() -> Self { - Self::Local(()) - } -} - pub fn default_persist_queue_capacity() -> u32 { 100 } diff --git a/dozer-types/src/models/cloud.rs b/dozer-types/src/models/cloud.rs index bbd8193df9..bc5ba0a053 100644 --- a/dozer-types/src/models/cloud.rs +++ b/dozer-types/src/models/cloud.rs @@ -1,10 +1,13 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use super::equal_default; + #[derive(Debug, JsonSchema, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(deny_unknown_fields)] pub struct Cloud { - #[serde(skip_serializing_if = "Option::is_none")] - pub update_current_version_strategy: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub update_current_version_strategy: UpdateCurrentVersionStrategy, #[serde(skip_serializing_if = "Option::is_none")] pub app_id: Option, @@ -12,25 +15,24 @@ pub struct Cloud { #[serde(skip_serializing_if = "Option::is_none")] pub profile: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub app: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub app: AppInstance, - #[serde(skip_serializing_if = "Option::is_none")] - pub api: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub api: ApiInstance, } #[derive(Debug, JsonSchema, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(deny_unknown_fields)] pub struct AppInstance { #[serde(skip_serializing_if = "Option::is_none")] pub instance_type: Option, } #[derive(Debug, JsonSchema, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(deny_unknown_fields)] pub struct ApiInstance { - #[serde( - default = "default_num_api_instances", - skip_serializing_if = "Option::is_none" - )] + #[serde(skip_serializing_if = "Option::is_none")] pub instances_count: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -41,19 +43,10 @@ pub struct ApiInstance { pub volume_size: Option, } -#[derive(Debug, JsonSchema, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, JsonSchema, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(deny_unknown_fields)] pub enum UpdateCurrentVersionStrategy { - OnCreate(()), - - Manual(()), -} - -impl Default for UpdateCurrentVersionStrategy { - fn default() -> Self { - UpdateCurrentVersionStrategy::OnCreate(()) - } -} - -fn default_num_api_instances() -> Option { - Some(2) + #[default] + OnCreate, + Manual, } diff --git a/dozer-types/src/models/config.rs b/dozer-types/src/models/config.rs index 5c10aea594..ef7a088ab4 100644 --- a/dozer-types/src/models/config.rs +++ b/dozer-types/src/models/config.rs @@ -2,7 +2,8 @@ use std::path::Path; use super::{ api_config::ApiConfig, api_endpoint::ApiEndpoint, app_config::AppConfig, cloud::Cloud, - connection::Connection, flags::Flags, source::Source, telemetry::TelemetryConfig, + connection::Connection, equal_default, flags::Flags, source::Source, + telemetry::TelemetryConfig, }; use crate::constants::DEFAULT_HOME_DIR; use crate::models::udf_config::UdfConfig; @@ -11,6 +12,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default, JsonSchema)] +#[serde(deny_unknown_fields)] /// The configuration for the app pub struct Config { pub version: u32, @@ -18,60 +20,53 @@ pub struct Config { /// name of the app pub app_name: String, - #[serde(skip_serializing_if = "String::is_empty", default = "default_home_dir")] + #[serde(skip_serializing_if = "Option::is_none")] ///directory for all process; Default: ./.dozer - pub home_dir: String, + pub home_dir: Option, - #[serde( - skip_serializing_if = "String::is_empty", - default = "default_cache_dir" - )] + #[serde(skip_serializing_if = "Option::is_none")] ///directory for cache. Default: ./.dozer/cache - pub cache_dir: String, + pub cache_dir: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] - /// connections to databases: Eg: Postgres, Snowflake, etc pub connections: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] - /// sources to ingest data related to particular connection pub sources: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] - /// api endpoints to expose pub endpoints: Vec, + #[serde(default, skip_serializing_if = "equal_default")] /// Api server config related: port, host, etc - #[serde(skip_serializing_if = "Option::is_none")] - pub api: Option, + pub api: ApiConfig, #[serde(skip_serializing_if = "Option::is_none")] /// transformations to apply to source data in SQL format as multiple queries pub sql: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "equal_default")] /// flags to enable/disable features - pub flags: Option, - - /// Cache lmdb max map size + pub flags: Flags, #[serde(skip_serializing_if = "Option::is_none")] + /// Cache lmdb max map size pub cache_max_map_size: Option, + #[serde(default, skip_serializing_if = "equal_default")] /// App runtime config: behaviour of pipeline and log - #[serde(skip_serializing_if = "Option::is_none")] - pub app: Option, + pub app: AppConfig, + #[serde(default, skip_serializing_if = "equal_default")] /// Instrument using Dozer - #[serde(skip_serializing_if = "Option::is_none")] - pub telemetry: Option, + pub telemetry: TelemetryConfig, + #[serde(default, skip_serializing_if = "equal_default")] /// Dozer Cloud specific configuration - #[serde(skip_serializing_if = "Option::is_none")] - pub cloud: Option, + pub cloud: Cloud, /// UDF specific configuration (eg. !Onnx) #[serde(default, skip_serializing_if = "Vec::is_empty")] diff --git a/dozer-types/src/models/connection.rs b/dozer-types/src/models/connection.rs index 6deff9f598..e708be4b66 100644 --- a/dozer-types/src/models/connection.rs +++ b/dozer-types/src/models/connection.rs @@ -19,16 +19,16 @@ pub trait SchemaExample { fn example() -> Self; } -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema, Default)] +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)] +#[serde(deny_unknown_fields)] pub struct Connection { - /// authentication config - depends on db_type - pub config: Option, - + pub config: ConnectionConfig, pub name: String, } /// Configuration for a Postgres connection #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema, Default)] +#[serde(deny_unknown_fields)] #[schemars(example = "Self::example")] pub struct PostgresConfig { /// The username to use for authentication @@ -192,6 +192,7 @@ fn get_sslmode(mode: String) -> Result { } #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Hash, JsonSchema)] +#[serde(deny_unknown_fields)] pub enum ConnectionConfig { /// In yaml, present as tag: `!Postgres` Postgres(PostgresConfig), diff --git a/dozer-types/src/models/flags.rs b/dozer-types/src/models/flags.rs index d5f958d6d5..85b198f6cd 100644 --- a/dozer-types/src/models/flags.rs +++ b/dozer-types/src/models/flags.rs @@ -1,59 +1,51 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize, JsonSchema, Deserialize, PartialEq, Eq, Clone)] + +use super::equal_default; + +#[derive(Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Clone, Default)] +#[serde(deny_unknown_fields)] pub struct Flags { /// dynamic grpc enabled; Default: true - #[serde(default = "default_true")] - pub dynamic: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub dynamic: Option, + /// http1 + web support for grpc. This is required for browser clients.; Default: true - #[serde(default = "default_true")] - pub grpc_web: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub grpc_web: Option, /// push events enabled.; Default: true - #[serde(default = "default_push_events")] - pub push_events: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub push_events: Option, /// require authentication to access grpc server reflection service if true.; Default: false - #[serde(default = "default_false")] - pub authenticate_server_reflection: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub authenticate_server_reflection: Option, /// probablistic optimizations reduce memory consumption at the expense of accuracy. - #[serde(skip_serializing_if = "Option::is_none")] - pub enable_probabilistic_optimizations: Option, + #[serde(default, skip_serializing_if = "equal_default")] + pub enable_probabilistic_optimizations: EnableProbabilisticOptimizations, } -impl Default for Flags { - fn default() -> Self { - Self { - dynamic: true, - grpc_web: true, - push_events: true, - authenticate_server_reflection: false, - enable_probabilistic_optimizations: Default::default(), - } - } + +pub fn default_dynamic() -> bool { + true } -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, PartialEq, Eq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Default, PartialEq, Eq, Clone)] +#[serde(deny_unknown_fields)] pub struct EnableProbabilisticOptimizations { /// enable probabilistic optimizations in set operations (UNION, EXCEPT, INTERSECT); Default: false - #[serde(default = "default_false")] - pub in_sets: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub in_sets: Option, /// enable probabilistic optimizations in JOIN operations; Default: false - #[serde(default = "default_false")] - pub in_joins: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub in_joins: Option, /// enable probabilistic optimizations in aggregations (SUM, COUNT, MIN, etc.); Default: false - #[serde(default = "default_false")] - pub in_aggregations: bool, -} - -fn default_true() -> bool { - true -} -fn default_false() -> bool { - false + #[serde(skip_serializing_if = "Option::is_none")] + pub in_aggregations: Option, } pub fn default_push_events() -> bool { diff --git a/dozer-types/src/models/mod.rs b/dozer-types/src/models/mod.rs index 413f8514fc..93f01e2081 100644 --- a/dozer-types/src/models/mod.rs +++ b/dozer-types/src/models/mod.rs @@ -11,3 +11,7 @@ pub mod source; pub mod telemetry; pub mod udf_config; pub use json_schema_helper::{get_connection_schemas, get_dozer_schema}; + +fn equal_default(t: &T) -> bool { + t == &T::default() +} diff --git a/dozer-types/src/models/source.rs b/dozer-types/src/models/source.rs index 1c56adc49d..8f0c449ec7 100644 --- a/dozer-types/src/models/source.rs +++ b/dozer-types/src/models/source.rs @@ -1,7 +1,10 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] +use super::equal_default; + +#[derive(Debug, Serialize, Deserialize, JsonSchema, Default, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub struct Source { /// name of the source - to distinguish between multiple sources; Type: String pub name: String, @@ -15,59 +18,22 @@ pub struct Source { /// reference to pre-defined connection name; Type: String pub connection: String, - /// name of schema source database; Type: String - #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + /// name of schema source database; Type: String pub schema: Option, - #[serde(default = "default_refresh_config")] - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "equal_default")] /// setting for how to refresh the data; Default: RealTime - pub refresh_config: Option, -} - -fn default_refresh_config() -> Option { - Some(RefreshConfig::default()) + pub refresh_config: RefreshConfig, } -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] -pub enum Value { - Ref(String), -} - -#[derive(Debug, Serialize, JsonSchema, Deserialize, Eq, PartialEq, Clone)] -pub enum HistoryType { - Master(MasterHistoryConfig), - Transactional(TransactionalHistoryConfig), -} -#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone)] -pub enum MasterHistoryConfig { - AppendOnly { - unique_key_field: String, - open_date_field: String, - closed_date_field: String, - }, - Overwrite, -} - -#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone)] -pub enum TransactionalHistoryConfig { - RetainPartial { - timestamp_field: String, - retention_period: u32, - }, -} -#[derive(Debug, Serialize, JsonSchema, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone, Default)] +#[serde(deny_unknown_fields)] pub enum RefreshConfig { // Hour { minute: u32 }, // Day { time: String }, // CronExpression { expression: String }, - RealTime(RealTimeConfig), -} -impl Default for RefreshConfig { - fn default() -> Self { - RefreshConfig::RealTime(RealTimeConfig {}) - } + #[default] + RealTime, } -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] -pub struct RealTimeConfig {} diff --git a/dozer-types/src/models/telemetry.rs b/dozer-types/src/models/telemetry.rs index a2926fe078..a746094d39 100644 --- a/dozer-types/src/models/telemetry.rs +++ b/dozer-types/src/models/telemetry.rs @@ -1,51 +1,48 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, PartialEq, Eq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Default, PartialEq, Eq, Clone)] +#[serde(deny_unknown_fields)] pub struct TelemetryConfig { pub trace: Option, - pub metrics: Option, } -#[derive(Debug, Serialize, JsonSchema, Deserialize, PartialEq, Eq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Clone)] pub enum TelemetryTraceConfig { Dozer(DozerTelemetryConfig), - XRay(XRayConfig), } -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, PartialEq, Eq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Default, PartialEq, Eq, Clone)] +#[serde(deny_unknown_fields)] pub struct DozerTelemetryConfig { - #[serde(default = "default_ingest_address")] - pub endpoint: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub endpoint: Option, - #[serde(default = "default_grpc_adapter")] - pub adapter: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub adapter: Option, - #[serde(default = "default_sample_ratio")] - pub sample_percent: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub sample_percent: Option, } -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, PartialEq, Eq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Clone)] +#[serde(deny_unknown_fields)] pub struct XRayConfig { pub endpoint: String, - pub timeout_in_seconds: u64, } -fn default_grpc_adapter() -> String { - "arrow".to_owned() -} - -fn default_ingest_address() -> String { +pub fn default_ingest_address() -> String { "0.0.0.0:7006".to_string() } -fn default_sample_ratio() -> u32 { +pub fn default_sample_ratio() -> u32 { 10 } -#[derive(Debug, Serialize, JsonSchema, Deserialize, PartialEq, Eq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Clone)] +#[serde(deny_unknown_fields)] pub enum TelemetryMetricsConfig { - Prometheus(()), + Prometheus, } diff --git a/dozer-types/src/models/udf_config.rs b/dozer-types/src/models/udf_config.rs index aabe7b41d9..914e6091e7 100644 --- a/dozer-types/src/models/udf_config.rs +++ b/dozer-types/src/models/udf_config.rs @@ -2,22 +2,23 @@ use schemars::JsonSchema; use crate::serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub struct UdfConfig { /// name of the model function pub name: String, - - #[serde(skip_serializing_if = "Option::is_none")] /// setting for what type of udf to use; Default: Onnx - pub config: Option, + pub config: UdfType, } -#[derive(Debug, Serialize, JsonSchema, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub enum UdfType { Onnx(OnnxConfig), } -#[derive(Debug, Serialize, JsonSchema, Default, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq, Clone)] +#[serde(deny_unknown_fields)] pub struct OnnxConfig { /// path to the model file pub path: String, diff --git a/dozer-types/src/tests/api_config_yaml_deserialize.rs b/dozer-types/src/tests/api_config_yaml_deserialize.rs index 450b21e0e4..db3df2edd2 100644 --- a/dozer-types/src/tests/api_config_yaml_deserialize.rs +++ b/dozer-types/src/tests/api_config_yaml_deserialize.rs @@ -1,7 +1,5 @@ use crate::models::{ - api_config::{ - default_api_grpc, default_api_rest, default_app_grpc, GrpcApiOptions, RestApiOptions, - }, + api_config::{GrpcApiOptions, RestApiOptions}, api_security::ApiSecurity, config::Config, }; @@ -17,19 +15,14 @@ fn override_rest_port() { home_dir: './.dozer' "#; let deserialize_result = serde_yaml::from_str::(input_config); - assert!(deserialize_result.is_ok()); let api_config = deserialize_result.unwrap().api; - assert!(api_config.is_some()); - let api_config = api_config.unwrap(); - assert!(api_config.rest.is_some()); - let default_api_rest = default_api_rest(); let expected_rest_config = RestApiOptions { - port: 9876, - host: default_api_rest.host, - cors: default_api_rest.cors, - enabled: true, + port: Some(9876), + host: None, + cors: None, + enabled: None, }; - assert_eq!(api_config.rest.unwrap(), expected_rest_config); + assert_eq!(api_config.rest, expected_rest_config); } #[test] @@ -43,19 +36,14 @@ fn override_rest_host() { home_dir: './.dozer' "#; let deserialize_result = serde_yaml::from_str::(input_config); - assert!(deserialize_result.is_ok()); let api_config = deserialize_result.unwrap().api; - assert!(api_config.is_some()); - let api_config = api_config.unwrap(); - assert!(api_config.rest.is_some()); - let default_api_rest = default_api_rest(); let expected_rest_config = RestApiOptions { - port: default_api_rest.port, - host: "localhost".to_owned(), - cors: default_api_rest.cors, - enabled: true, + port: None, + host: Some("localhost".to_owned()), + cors: None, + enabled: None, }; - assert_eq!(api_config.rest.unwrap(), expected_rest_config); + assert_eq!(api_config.rest, expected_rest_config); } #[test] @@ -69,19 +57,14 @@ fn override_rest_enabled() { home_dir: './.dozer' "#; let deserialize_result = serde_yaml::from_str::(input_config); - assert!(deserialize_result.is_ok()); let api_config = deserialize_result.unwrap().api; - assert!(api_config.is_some()); - let api_config = api_config.unwrap(); - assert!(api_config.rest.is_some()); - let default_api_rest = default_api_rest(); let expected_rest_config = RestApiOptions { - port: default_api_rest.port, - host: default_api_rest.host, - cors: default_api_rest.cors, - enabled: false, + port: None, + host: None, + cors: None, + enabled: Some(false), }; - assert_eq!(api_config.rest.unwrap(), expected_rest_config); + assert_eq!(api_config.rest, expected_rest_config); } #[test] @@ -95,19 +78,15 @@ fn override_grpc_port() { home_dir: './.dozer' "#; let deserialize_result = serde_yaml::from_str::(input_config); - assert!(deserialize_result.is_ok()); let api_config = deserialize_result.unwrap().api; - assert!(api_config.is_some()); - let api_config = api_config.unwrap(); - let default_api_grpc = default_api_grpc(); let expected_grpc_config = GrpcApiOptions { - port: 4232, - host: default_api_grpc.host, - cors: default_api_grpc.cors, - web: default_api_grpc.web, - enabled: true, + port: Some(4232), + host: None, + cors: None, + web: None, + enabled: None, }; - assert_eq!(api_config.grpc.unwrap(), expected_grpc_config); + assert_eq!(api_config.grpc, expected_grpc_config); } #[test] @@ -121,95 +100,29 @@ fn override_grpc_enabled() { home_dir: './.dozer' "#; let deserialize_result = serde_yaml::from_str::(input_config); - assert!(deserialize_result.is_ok()); let api_config = deserialize_result.unwrap().api; - assert!(api_config.is_some()); - let api_config = api_config.unwrap(); - let default_api_grpc = default_api_grpc(); let expected_grpc_config = GrpcApiOptions { - enabled: false, - port: default_api_grpc.port, - host: default_api_grpc.host, - cors: default_api_grpc.cors, - web: default_api_grpc.web, + enabled: Some(false), + port: None, + host: None, + cors: None, + web: None, }; - assert_eq!(api_config.grpc.unwrap(), expected_grpc_config); + assert_eq!(api_config.grpc, expected_grpc_config); } #[test] -fn override_grpc_and_rest_port() { +fn override_jwt() { let input_config = r#" app_name: working_app version: 1 api: - grpc: - port: 4232 - rest: - port: 3324 - home_dir: './.dozer' -"#; - let deserialize_result = serde_yaml::from_str::(input_config); - assert!(deserialize_result.is_ok()); - let api_config = deserialize_result.unwrap().api; - assert!(api_config.is_some()); - let api_config = api_config.unwrap(); - assert!(api_config.rest.is_some()); - let default_api_rest = default_api_rest(); - let default_api_grpc = default_api_grpc(); - let expected_grpc_config = GrpcApiOptions { - port: 4232, - host: default_api_grpc.host, - cors: default_api_grpc.cors, - web: default_api_grpc.web, - enabled: true, - }; - let expected_rest_config = RestApiOptions { - port: 3324, - host: default_api_rest.host, - cors: default_api_rest.cors, - enabled: true, - }; - assert_eq!(api_config.rest.unwrap(), expected_rest_config); - assert_eq!(api_config.grpc.unwrap(), expected_grpc_config); -} - -#[test] -fn override_grpc_and_rest_port_jwt() { - let input_config = r#" - app_name: working_app - version: 1 - api: - grpc: - port: 4232 - rest: - port: 3324 api_security: !Jwt Vv44T1GugX home_dir: './.dozer' "#; let deserialize_result = serde_yaml::from_str::(input_config); - assert!(deserialize_result.is_ok()); let api_config = deserialize_result.unwrap().api; - assert!(api_config.is_some()); - let api_config = api_config.unwrap(); - assert!(api_config.rest.is_some()); - let default_api_rest = default_api_rest(); - let default_api_grpc = default_api_grpc(); - let expected_grpc_config = GrpcApiOptions { - port: 4232, - host: default_api_grpc.host, - cors: default_api_grpc.cors, - web: default_api_grpc.web, - enabled: true, - }; - let expected_rest_config = RestApiOptions { - port: 3324, - host: default_api_rest.host, - cors: default_api_rest.cors, - enabled: true, - }; - assert_eq!(api_config.rest.unwrap(), expected_rest_config); - assert_eq!(api_config.grpc.unwrap(), expected_grpc_config); let api_security = api_config.api_security; assert!(api_security.is_some()); let api_security = api_security.unwrap(); @@ -218,7 +131,7 @@ fn override_grpc_and_rest_port_jwt() { } #[test] -fn override_grpc_and_rest_port_jwt_pipeline_home_dir() { +fn override_pipeline_port() { let input_config = r#" app_name: working_app version: 1 @@ -230,43 +143,14 @@ fn override_grpc_and_rest_port_jwt_pipeline_home_dir() { api_security: !Jwt Vv44T1GugX app_grpc: - home_dir: './pipeline_folder' port: 3993 home_dir: './.dozer' "#; let deserialize_result = serde_yaml::from_str::(input_config); - assert!(deserialize_result.is_ok()); let api_config = deserialize_result.unwrap().api; - assert!(api_config.is_some()); - let api_config = api_config.unwrap(); - assert!(api_config.rest.is_some()); - let default_api_rest = default_api_rest(); - let default_api_grpc = default_api_grpc(); - let expected_grpc_config = GrpcApiOptions { - port: 4232, - host: default_api_grpc.host, - cors: default_api_grpc.cors, - web: default_api_grpc.web, - enabled: true, - }; - let expected_rest_config = RestApiOptions { - port: 3324, - host: default_api_rest.host, - cors: default_api_rest.cors, - enabled: true, - }; - assert_eq!(api_config.rest.unwrap(), expected_rest_config); - assert_eq!(api_config.grpc.unwrap(), expected_grpc_config); - let api_security = api_config.api_security; - assert!(api_security.is_some()); - let api_security = api_security.unwrap(); - let expected_api_security = ApiSecurity::Jwt("Vv44T1GugX".to_owned()); - assert_eq!(api_security, expected_api_security); - - let app_grpc = api_config.app_grpc.unwrap(); - let default_app_grpc = default_app_grpc(); - assert_eq!(app_grpc.port, 3993); - assert_eq!(app_grpc.host, default_app_grpc.host); + let app_grpc = api_config.app_grpc; + assert_eq!(app_grpc.port, Some(3993)); + assert_eq!(app_grpc.host, None); } diff --git a/dozer-types/src/tests/dozer_yaml_deserialize.rs b/dozer-types/src/tests/dozer_yaml_deserialize.rs index bc0f456978..98627f5164 100644 --- a/dozer-types/src/tests/dozer_yaml_deserialize.rs +++ b/dozer-types/src/tests/dozer_yaml_deserialize.rs @@ -49,13 +49,12 @@ fn error_missing_field_general() { version: 1 home_dir: './.dozer' connections: - - authentication: !Postgres + - config: !Postgres user: postgres password: postgres host: localhost port: 5432 database: users - db_type: Postgres name: users sources: - table_name: users @@ -86,13 +85,12 @@ fn error_missing_field_in_source() { version: 1 home_dir: './.dozer' connections: - - authentication: !Postgres + - config: !Postgres user: postgres password: postgres host: localhost port: 5432 database: users - db_type: Postgres name: users sources: - table_name: users @@ -118,13 +116,12 @@ fn error_missing_field_connection_ref_in_source() { version: 1 home_dir: './.dozer' connections: - - authentication: !Postgres + - config: !Postgres user: postgres password: postgres host: localhost port: 5432 database: users - db_type: Postgres name: users sources: - name: users diff --git a/dozer-types/src/tests/flags_config_yaml_deserialize.rs b/dozer-types/src/tests/flags_config_yaml_deserialize.rs index 83926a2937..571f485b37 100644 --- a/dozer-types/src/tests/flags_config_yaml_deserialize.rs +++ b/dozer-types/src/tests/flags_config_yaml_deserialize.rs @@ -1,7 +1,4 @@ -use crate::models::{ - config::Config, - flags::{EnableProbabilisticOptimizations, Flags}, -}; +use crate::models::{config::Config, flags::Flags}; #[test] fn test_partial_flag_config_input() { @@ -14,11 +11,10 @@ fn test_partial_flag_config_input() { "#; let deserializer_result = serde_yaml::from_str::(input_config_with_flag).unwrap(); let default_flags = Flags::default(); - assert!(deserializer_result.flags.is_some()); - let flags_deserialize = deserializer_result.flags.unwrap(); - assert!(flags_deserialize.dynamic); - assert!(!flags_deserialize.grpc_web); - assert!(!flags_deserialize.push_events); + let flags_deserialize = deserializer_result.flags; + assert_eq!(flags_deserialize.dynamic, None); + assert_eq!(flags_deserialize.grpc_web, Some(false)); + assert_eq!(flags_deserialize.push_events, Some(false)); assert_eq!( flags_deserialize.authenticate_server_reflection, default_flags.authenticate_server_reflection, @@ -36,26 +32,3 @@ fn test_storage_params_config() { let deserializer_result = serde_yaml::from_str::(input_config_without_flag).unwrap(); assert_eq!(deserializer_result.cache_max_map_size, Some(1073741824)); } - -#[test] -fn test_flags_default() { - assert_eq!( - Flags::default(), - Flags { - dynamic: true, - grpc_web: true, - push_events: true, - authenticate_server_reflection: false, - enable_probabilistic_optimizations: None, - } - ); - - assert_eq!( - EnableProbabilisticOptimizations::default(), - EnableProbabilisticOptimizations { - in_sets: false, - in_joins: false, - in_aggregations: false - } - ) -} diff --git a/dozer-types/src/tests/secondary_index_yaml_deserialize.rs b/dozer-types/src/tests/secondary_index_yaml_deserialize.rs index b70eacc4d2..714266d80f 100644 --- a/dozer-types/src/tests/secondary_index_yaml_deserialize.rs +++ b/dozer-types/src/tests/secondary_index_yaml_deserialize.rs @@ -1,6 +1,4 @@ -use crate::models::api_endpoint::{ - CreateSecondaryIndex, FullText, SecondaryIndex, SecondaryIndexConfig, SortedInverted, -}; +use crate::models::api_endpoint::{FullText, SecondaryIndex, SecondaryIndexConfig, SortedInverted}; #[test] fn standard() { @@ -8,11 +6,11 @@ fn standard() { - field1 - field2 create: - - index: !SortedInverted + - !SortedInverted fields: - field1 - field2 - - index: !FullText + - !FullText field: field3 "#; let config: SecondaryIndexConfig = serde_yaml::from_str(secondary).unwrap(); @@ -20,16 +18,12 @@ create: assert_eq!( config.create, vec![ - CreateSecondaryIndex { - index: Some(SecondaryIndex::SortedInverted(SortedInverted { - fields: vec!["field1".to_string(), "field2".to_string()] - })) - }, - CreateSecondaryIndex { - index: Some(SecondaryIndex::FullText(FullText { - field: "field3".to_string() - })) - } + SecondaryIndex::SortedInverted(SortedInverted { + fields: vec!["field1".to_string(), "field2".to_string()] + }), + SecondaryIndex::FullText(FullText { + field: "field3".to_string() + }), ] ); } diff --git a/dozer-types/src/tests/udf_yaml_deserialize.rs b/dozer-types/src/tests/udf_yaml_deserialize.rs index eb03916572..6cec087a5b 100644 --- a/dozer-types/src/tests/udf_yaml_deserialize.rs +++ b/dozer-types/src/tests/udf_yaml_deserialize.rs @@ -9,9 +9,9 @@ fn standard() { "#; let deserializer_result = serde_yaml::from_str::(udf_config).unwrap(); let udf_conf = UdfConfig { - config: Some(UdfType::Onnx(OnnxConfig { + config: UdfType::Onnx(OnnxConfig { path: "./models/model_file".to_string(), - })), + }), name: "is_fraudulent".to_string(), }; let expected = udf_conf; diff --git a/json_schemas/connections.json b/json_schemas/connections.json index 8e981e8640..627d09080d 100644 --- a/json_schemas/connections.json +++ b/json_schemas/connections.json @@ -69,7 +69,8 @@ "null" ] } - } + }, + "additionalProperties": false } }, { @@ -209,8 +210,10 @@ ], "properties": { "batch_size": { - "default": 3, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint64", "minimum": 0.0 }, @@ -241,30 +244,32 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "GrpcConfig", "type": "object", + "required": [ + "schemas" + ], "properties": { "adapter": { - "default": "default", - "type": "string" + "type": [ + "string", + "null" + ] }, "host": { - "default": "0.0.0.0", - "type": "string" + "type": [ + "string", + "null" + ] }, "port": { - "default": 8085, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint32", "minimum": 0.0 }, "schemas": { - "anyOf": [ - { - "$ref": "#/definitions/GrpcConfigSchemas" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/GrpcConfigSchemas" } }, "definitions": { @@ -410,12 +415,10 @@ "type": "string" }, "marker_extension": { - "default": ".marker", - "type": "string" - }, - "marker_file": { - "default": false, - "type": "boolean" + "type": [ + "string", + "null" + ] }, "path": { "type": "string" @@ -444,12 +447,10 @@ "type": "string" }, "marker_extension": { - "default": ".marker", - "type": "string" - }, - "marker_file": { - "default": false, - "type": "boolean" + "type": [ + "string", + "null" + ] }, "path": { "type": "string" @@ -582,12 +583,10 @@ "type": "string" }, "marker_extension": { - "default": ".marker", - "type": "string" - }, - "marker_file": { - "default": false, - "type": "boolean" + "type": [ + "string", + "null" + ] }, "path": { "type": "string" @@ -627,12 +626,10 @@ "type": "string" }, "marker_extension": { - "default": ".marker", - "type": "string" - }, - "marker_file": { - "default": false, - "type": "boolean" + "type": [ + "string", + "null" + ] }, "path": { "type": "string" @@ -785,62 +782,47 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "NestedDozerConfig", "type": "object", + "required": [ + "url" + ], "properties": { - "grpc": { - "anyOf": [ - { - "$ref": "#/definitions/AppGrpcOptions" - }, - { - "type": "null" - } - ] - }, "log_options": { - "anyOf": [ + "default": {}, + "allOf": [ { "$ref": "#/definitions/NestedDozerLogOptions" - }, - { - "type": "null" } ] + }, + "url": { + "type": "string" } }, "definitions": { - "AppGrpcOptions": { - "type": "object", - "properties": { - "host": { - "default": "0.0.0.0", - "type": "string" - }, - "port": { - "default": 50053, - "type": "integer", - "format": "uint32", - "minimum": 0.0 - } - } - }, "NestedDozerLogOptions": { "type": "object", "properties": { "batch_size": { - "default": 30, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint32", "minimum": 0.0 }, "buffer_size": { - "default": 1000, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint32", "minimum": 0.0 }, "timeout_in_millis": { - "default": 1000, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint32", "minimum": 0.0 } diff --git a/json_schemas/dozer.json b/json_schemas/dozer.json index c64ad02024..d28aa9caaa 100644 --- a/json_schemas/dozer.json +++ b/json_schemas/dozer.json @@ -10,23 +10,17 @@ "properties": { "api": { "description": "Api server config related: port, host, etc", - "anyOf": [ + "allOf": [ { "$ref": "#/definitions/ApiConfig" - }, - { - "type": "null" } ] }, "app": { "description": "App runtime config: behaviour of pipeline and log", - "anyOf": [ + "allOf": [ { "$ref": "#/definitions/AppConfig" - }, - { - "type": "null" } ] }, @@ -36,8 +30,10 @@ }, "cache_dir": { "description": "directory for cache. Default: ./.dozer/cache", - "default": "./.dozer/cache", - "type": "string" + "type": [ + "string", + "null" + ] }, "cache_max_map_size": { "description": "Cache lmdb max map size", @@ -50,12 +46,9 @@ }, "cloud": { "description": "Dozer Cloud specific configuration", - "anyOf": [ + "allOf": [ { "$ref": "#/definitions/Cloud" - }, - { - "type": "null" } ] }, @@ -75,19 +68,18 @@ }, "flags": { "description": "flags to enable/disable features", - "anyOf": [ + "allOf": [ { "$ref": "#/definitions/Flags" - }, - { - "type": "null" } ] }, "home_dir": { "description": "directory for all process; Default: ./.dozer", - "default": "./.dozer", - "type": "string" + "type": [ + "string", + "null" + ] }, "sources": { "description": "sources to ingest data related to particular connection", @@ -105,12 +97,9 @@ }, "telemetry": { "description": "Instrument using Dozer", - "anyOf": [ + "allOf": [ { "$ref": "#/definitions/TelemetryConfig" - }, - { - "type": "null" } ] }, @@ -127,6 +116,7 @@ "minimum": 0.0 } }, + "additionalProperties": false, "definitions": { "ApiConfig": { "type": "object", @@ -143,42 +133,24 @@ ] }, "app_grpc": { - "anyOf": [ - { - "$ref": "#/definitions/AppGrpcOptions" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/AppGrpcOptions" }, "default_max_num_records": { - "default": 50, - "type": "integer", - "format": "uint32", + "type": [ + "integer", + "null" + ], + "format": "uint", "minimum": 0.0 }, "grpc": { - "anyOf": [ - { - "$ref": "#/definitions/GrpcApiOptions" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/GrpcApiOptions" }, "rest": { - "anyOf": [ - { - "$ref": "#/definitions/RestApiOptions" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/RestApiOptions" } - } + }, + "additionalProperties": false }, "ApiEndpoint": { "type": "object", @@ -189,34 +161,13 @@ ], "properties": { "conflict_resolution": { - "anyOf": [ - { - "$ref": "#/definitions/ConflictResolution" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/ConflictResolution" }, "index": { - "anyOf": [ - { - "$ref": "#/definitions/ApiIndex" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/ApiIndex" }, "log_reader_options": { - "anyOf": [ - { - "$ref": "#/definitions/LogReaderOptions" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/LogReaderOptions" }, "name": { "type": "string" @@ -237,7 +188,8 @@ "format": "uint32", "minimum": 0.0 } - } + }, + "additionalProperties": false }, "ApiIndex": { "type": "object", @@ -249,16 +201,10 @@ } }, "secondary": { - "anyOf": [ - { - "$ref": "#/definitions/SecondaryIndexConfig" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/SecondaryIndexConfig" } - } + }, + "additionalProperties": false }, "ApiInstance": { "type": "object", @@ -270,7 +216,6 @@ ] }, "instances_count": { - "default": 2, "type": [ "integer", "null" @@ -287,7 +232,8 @@ "format": "uint32", "minimum": 0.0 } - } + }, + "additionalProperties": false }, "ApiSecurity": { "description": "The security model option for the API", @@ -339,12 +285,9 @@ }, "data_storage": { "description": "The storage to use for the log.", - "anyOf": [ + "allOf": [ { "$ref": "#/definitions/DataStorage" - }, - { - "type": "null" } ] }, @@ -376,6 +319,7 @@ "minimum": 0.0 }, "persist_queue_capacity": { + "description": "Maximum number of pending persisting requests.", "type": [ "integer", "null" @@ -383,22 +327,28 @@ "format": "uint32", "minimum": 0.0 } - } + }, + "additionalProperties": false }, "AppGrpcOptions": { "type": "object", "properties": { "host": { - "default": "0.0.0.0", - "type": "string" + "type": [ + "string", + "null" + ] }, "port": { - "default": 50053, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint32", "minimum": 0.0 } - } + }, + "additionalProperties": false }, "AppInstance": { "type": "object", @@ -409,30 +359,17 @@ "null" ] } - } + }, + "additionalProperties": false }, "Cloud": { "type": "object", "properties": { "api": { - "anyOf": [ - { - "$ref": "#/definitions/ApiInstance" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/ApiInstance" }, "app": { - "anyOf": [ - { - "$ref": "#/definitions/AppInstance" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/AppInstance" }, "app_id": { "type": [ @@ -447,73 +384,41 @@ ] }, "update_current_version_strategy": { - "anyOf": [ - { - "$ref": "#/definitions/UpdateCurrentVersionStrategy" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/UpdateCurrentVersionStrategy" } - } + }, + "additionalProperties": false }, "ConflictResolution": { "type": "object", "properties": { "on_delete": { - "anyOf": [ - { - "$ref": "#/definitions/OnDeleteResolutionTypes" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/OnDeleteResolutionTypes" }, "on_insert": { - "anyOf": [ - { - "$ref": "#/definitions/OnInsertResolutionTypes" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/OnInsertResolutionTypes" }, "on_update": { - "anyOf": [ - { - "$ref": "#/definitions/OnUpdateResolutionTypes" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/OnUpdateResolutionTypes" } - } + }, + "additionalProperties": false }, "Connection": { "type": "object", "required": [ + "config", "name" ], "properties": { "config": { - "description": "authentication config - depends on db_type", - "anyOf": [ - { - "$ref": "#/definitions/ConnectionConfig" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/ConnectionConfig" }, "name": { "type": "string" } - } + }, + "additionalProperties": false }, "ConnectionConfig": { "oneOf": [ @@ -662,21 +567,6 @@ } ] }, - "CreateSecondaryIndex": { - "type": "object", - "properties": { - "index": { - "anyOf": [ - { - "$ref": "#/definitions/SecondaryIndex" - }, - { - "type": "null" - } - ] - } - } - }, "CsvConfig": { "type": "object", "required": [ @@ -688,12 +578,10 @@ "type": "string" }, "marker_extension": { - "default": ".marker", - "type": "string" - }, - "marker_file": { - "default": false, - "type": "boolean" + "type": [ + "string", + "null" + ] }, "path": { "type": "string" @@ -703,16 +591,10 @@ "DataStorage": { "oneOf": [ { - "type": "object", - "required": [ + "type": "string", + "enum": [ "Local" - ], - "properties": { - "Local": { - "type": "null" - } - }, - "additionalProperties": false + ] }, { "type": "object", @@ -772,40 +654,54 @@ "type": "object", "properties": { "adapter": { - "default": "arrow", - "type": "string" + "type": [ + "string", + "null" + ] }, "endpoint": { - "default": "0.0.0.0:7006", - "type": "string" + "type": [ + "string", + "null" + ] }, "sample_percent": { - "default": 10, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint32", "minimum": 0.0 } - } + }, + "additionalProperties": false }, "EnableProbabilisticOptimizations": { "type": "object", "properties": { "in_aggregations": { "description": "enable probabilistic optimizations in aggregations (SUM, COUNT, MIN, etc.); Default: false", - "default": false, - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "in_joins": { "description": "enable probabilistic optimizations in JOIN operations; Default: false", - "default": false, - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "in_sets": { "description": "enable probabilistic optimizations in set operations (UNION, EXCEPT, INTERSECT); Default: false", - "default": false, - "type": "boolean" + "type": [ + "boolean", + "null" + ] } - } + }, + "additionalProperties": false }, "EthConfig": { "type": "object", @@ -940,8 +836,10 @@ ], "properties": { "batch_size": { - "default": 3, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint64", "minimum": 0.0 }, @@ -968,36 +866,42 @@ "properties": { "authenticate_server_reflection": { "description": "require authentication to access grpc server reflection service if true.; Default: false", - "default": false, - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "dynamic": { "description": "dynamic grpc enabled; Default: true", - "default": true, - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "enable_probabilistic_optimizations": { "description": "probablistic optimizations reduce memory consumption at the expense of accuracy.", - "anyOf": [ + "allOf": [ { "$ref": "#/definitions/EnableProbabilisticOptimizations" - }, - { - "type": "null" } ] }, "grpc_web": { "description": "http1 + web support for grpc. This is required for browser clients.; Default: true", - "default": true, - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "push_events": { "description": "push events enabled.; Default: true", - "default": true, - "type": "boolean" + "type": [ + "boolean", + "null" + ] } - } + }, + "additionalProperties": false }, "FullText": { "type": "object", @@ -1008,61 +912,75 @@ "field": { "type": "string" } - } + }, + "additionalProperties": false }, "GrpcApiOptions": { "type": "object", "properties": { "cors": { - "default": true, - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "enabled": { - "default": true, - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "host": { - "default": "0.0.0.0", - "type": "string" + "type": [ + "string", + "null" + ] }, "port": { - "default": 50051, - "type": "integer", - "format": "uint32", + "type": [ + "integer", + "null" + ], + "format": "uint16", "minimum": 0.0 }, "web": { - "default": true, - "type": "boolean" + "type": [ + "boolean", + "null" + ] } - } + }, + "additionalProperties": false }, "GrpcConfig": { "type": "object", + "required": [ + "schemas" + ], "properties": { "adapter": { - "default": "default", - "type": "string" + "type": [ + "string", + "null" + ] }, "host": { - "default": "0.0.0.0", - "type": "string" + "type": [ + "string", + "null" + ] }, "port": { - "default": 8085, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint32", "minimum": 0.0 }, "schemas": { - "anyOf": [ - { - "$ref": "#/definitions/GrpcConfigSchemas" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/GrpcConfigSchemas" } } }, @@ -1173,7 +1091,8 @@ "format": "uint32", "minimum": 0.0 } - } + }, + "additionalProperties": false }, "MongodbConfig": { "type": "object", @@ -1207,26 +1126,20 @@ }, "NestedDozerConfig": { "type": "object", + "required": [ + "url" + ], "properties": { - "grpc": { - "anyOf": [ - { - "$ref": "#/definitions/AppGrpcOptions" - }, - { - "type": "null" - } - ] - }, "log_options": { - "anyOf": [ + "default": {}, + "allOf": [ { "$ref": "#/definitions/NestedDozerLogOptions" - }, - { - "type": "null" } ] + }, + "url": { + "type": "string" } } }, @@ -1234,131 +1147,52 @@ "type": "object", "properties": { "batch_size": { - "default": 30, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint32", "minimum": 0.0 }, "buffer_size": { - "default": 1000, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint32", "minimum": 0.0 }, "timeout_in_millis": { - "default": 1000, - "type": "integer", + "type": [ + "integer", + "null" + ], "format": "uint32", "minimum": 0.0 } } }, "OnDeleteResolutionTypes": { - "oneOf": [ - { - "type": "object", - "required": [ - "Nothing" - ], - "properties": { - "Nothing": { - "type": "null" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "Panic" - ], - "properties": { - "Panic": { - "type": "null" - } - }, - "additionalProperties": false - } + "type": "string", + "enum": [ + "Nothing", + "Panic" ] }, "OnInsertResolutionTypes": { - "oneOf": [ - { - "type": "object", - "required": [ - "Nothing" - ], - "properties": { - "Nothing": { - "type": "null" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "Update" - ], - "properties": { - "Update": { - "type": "null" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "Panic" - ], - "properties": { - "Panic": { - "type": "null" - } - }, - "additionalProperties": false - } + "type": "string", + "enum": [ + "Nothing", + "Update", + "Panic" ] }, "OnUpdateResolutionTypes": { - "oneOf": [ - { - "type": "object", - "required": [ - "Nothing" - ], - "properties": { - "Nothing": { - "type": "null" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "Upsert" - ], - "properties": { - "Upsert": { - "type": "null" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "Panic" - ], - "properties": { - "Panic": { - "type": "null" - } - }, - "additionalProperties": false - } + "type": "string", + "enum": [ + "Nothing", + "Upsert", + "Panic" ] }, "OnnxConfig": { @@ -1371,7 +1205,8 @@ "description": "path to the model file", "type": "string" } - } + }, + "additionalProperties": false }, "ParquetConfig": { "type": "object", @@ -1384,12 +1219,10 @@ "type": "string" }, "marker_extension": { - "default": ".marker", - "type": "string" - }, - "marker_file": { - "default": false, - "type": "boolean" + "type": [ + "string", + "null" + ] }, "path": { "type": "string" @@ -1462,49 +1295,46 @@ "null" ] } - } - }, - "RealTimeConfig": { - "type": "object" + }, + "additionalProperties": false }, "RefreshConfig": { - "oneOf": [ - { - "type": "object", - "required": [ - "RealTime" - ], - "properties": { - "RealTime": { - "$ref": "#/definitions/RealTimeConfig" - } - }, - "additionalProperties": false - } + "type": "string", + "enum": [ + "RealTime" ] }, "RestApiOptions": { "type": "object", "properties": { "cors": { - "default": true, - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "enabled": { - "default": true, - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "host": { - "default": "0.0.0.0", - "type": "string" + "type": [ + "string", + "null" + ] }, "port": { - "default": 8080, - "type": "integer", - "format": "uint32", + "type": [ + "integer", + "null" + ], + "format": "uint16", "minimum": 0.0 } - } + }, + "additionalProperties": false }, "S3Details": { "type": "object", @@ -1566,7 +1396,8 @@ "region": { "type": "string" } - } + }, + "additionalProperties": false }, "SecondaryIndex": { "oneOf": [ @@ -1602,7 +1433,7 @@ "create": { "type": "array", "items": { - "$ref": "#/definitions/CreateSecondaryIndex" + "$ref": "#/definitions/SecondaryIndex" } }, "skip_default": { @@ -1611,7 +1442,8 @@ "type": "string" } } - } + }, + "additionalProperties": false }, "SnowflakeConfig": { "type": "object", @@ -1670,7 +1502,8 @@ "type": "string" } } - } + }, + "additionalProperties": false }, "Source": { "type": "object", @@ -1697,21 +1530,14 @@ }, "refresh_config": { "description": "setting for how to refresh the data; Default: RealTime", - "default": { - "RealTime": {} - }, - "anyOf": [ + "allOf": [ { "$ref": "#/definitions/RefreshConfig" - }, - { - "type": "null" } ] }, "schema": { "description": "name of schema source database; Type: String", - "default": null, "type": [ "string", "null" @@ -1721,7 +1547,8 @@ "description": "name of the table in source database; Type: String", "type": "string" } - } + }, + "additionalProperties": false }, "Table": { "type": "object", @@ -1807,22 +1634,13 @@ } ] } - } + }, + "additionalProperties": false }, "TelemetryMetricsConfig": { - "oneOf": [ - { - "type": "object", - "required": [ - "Prometheus" - ], - "properties": { - "Prometheus": { - "type": "null" - } - }, - "additionalProperties": false - } + "type": "string", + "enum": [ + "Prometheus" ] }, "TelemetryTraceConfig": { @@ -1856,17 +1674,15 @@ "UdfConfig": { "type": "object", "required": [ + "config", "name" ], "properties": { "config": { "description": "setting for what type of udf to use; Default: Onnx", - "anyOf": [ + "allOf": [ { "$ref": "#/definitions/UdfType" - }, - { - "type": "null" } ] }, @@ -1874,7 +1690,8 @@ "description": "name of the model function", "type": "string" } - } + }, + "additionalProperties": false }, "UdfType": { "oneOf": [ @@ -1893,31 +1710,10 @@ ] }, "UpdateCurrentVersionStrategy": { - "oneOf": [ - { - "type": "object", - "required": [ - "OnCreate" - ], - "properties": { - "OnCreate": { - "type": "null" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "Manual" - ], - "properties": { - "Manual": { - "type": "null" - } - }, - "additionalProperties": false - } + "type": "string", + "enum": [ + "OnCreate", + "Manual" ] }, "XRayConfig": { @@ -1935,7 +1731,8 @@ "format": "uint64", "minimum": 0.0 } - } + }, + "additionalProperties": false } } } \ No newline at end of file