Skip to content

Commit

Permalink
chore: remove redudant code
Browse files Browse the repository at this point in the history
  • Loading branch information
@v3g42
v3g42 committed Sep 21, 2023
1 parent e4fd621 commit 3dd9d32
Show file tree
Hide file tree
Showing 20 changed files with 2,178 additions and 546 deletions.
14 changes: 9 additions & 5 deletions dozer-cli/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,20 @@ use std::io::Write;
use std::path::Path;

fn main() {
let schema_path = Path::new("../json_schemas");
// Define the path to the file we want to create or overwrite
let out_path = Path::new("../json_schemas").join("connections.json");

// Open the file in write-only mode (this will create the file if it doesn't exist)
let mut file = File::create(&out_path).expect("Failed to create connections.json");
let connection_path = schema_path.join("connections.json");
let dozer_path = schema_path.join("dozer.json");

let mut file = File::create(connection_path).expect("Failed to create connections.json");
let schemas = dozer_types::models::get_connection_schemas().unwrap();
write!(file, "{}", schemas).expect("Unable to write file");

let mut dozer_schema_file = File::create(dozer_path).expect("Failed to create dozer.json");
let schema = dozer_types::models::get_dozer_schema().unwrap();
write!(dozer_schema_file, "{}", schema).expect("Unable to write file");

// Print a message to indicate the file has been written
println!("cargo:rerun-if-changed=build.rs");
println!("Written to {:?}", out_path.display());
println!("Written to {:?}", schema_path.display());
}
4 changes: 1 addition & 3 deletions dozer-cli/src/simple/cloud_orchestrator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,9 +193,7 @@ impl CloudOrchestrator for SimpleOrchestrator {
let mut table = table!();

for app in response.apps {
if let Some(app_data) = app.app {
table.add_row(row![app.app_id, app_data.convert_to_table()]);
}
table.add_row(row![app.app_id, app.app_name]);
}

table.printstd();
Expand Down
58 changes: 0 additions & 58 deletions dozer-types/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,64 +49,6 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Cloud Service & Types
tonic_build::configure()
.protoc_arg("--experimental_allow_proto3_optional")
.extern_path(
".dozer.cloud.Endpoint",
"crate::models::api_endpoint::ApiEndpoint",
)
.extern_path(".dozer.cloud.Source", "crate::models::source::Source")
.extern_path(".dozer.cloud.AppConfig", "crate::models::config::Config")
.extern_path(
".dozer.cloud.ConnectionConfig",
"crate::models::connection::ConnectionConfig",
)
.extern_path(
".dozer.cloud.Connection",
"crate::models::connection::Connection",
)
.extern_path(
".dozer.cloud.EthContract",
"crate::ingestion_types::EthContract",
)
.extern_path(
".dozer.cloud.EthereumFilter",
"crate::ingestion_types::EthereumFilter",
)
.extern_path(
".dozer.cloud.DeltaLakeConfig",
"crate::ingestion_types::DeltaLakeConfig",
)
.extern_path(
".dozer.cloud.LocalStorage",
"crate::ingestion_types::LocalStorage",
)
.extern_path(
".dozer.cloud.S3Storage",
"crate::ingestion_types::S3Storage",
)
.extern_path(
".dozer.cloud.KafkaConfig",
"crate::ingestion_types::KafkaConfig",
)
.extern_path(
".dozer.cloud.SnowflakeConfig",
"crate::ingestion_types::SnowflakeConfig",
)
.extern_path(
".dozer.cloud::grpc_config::Schemas",
"crate::ingestion_types::GrpcConfigSchemas",
)
.extern_path(
".dozer.cloud.GrpcConfig",
"crate::ingestion_types::GrpcConfig",
)
.extern_path(
".dozer.cloud.EthereumConfig",
"crate::ingestion_types::EthConfig",
)
.extern_path(
".dozer.cloud.PostgresConfig",
"crate::models::connection::PostgresConfig",
)
.file_descriptor_set_path(out_dir.join("cloud.bin"))
.compile(&["protos/cloud.proto"], &["protos"])
.unwrap();
Expand Down
28 changes: 17 additions & 11 deletions dozer-types/protos/cloud.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import "google/protobuf/timestamp.proto";
import "google/protobuf/empty.proto";

service DozerCloud {

rpc validate_connection(ConnectionRequest)
returns (ValidateConnectionResponse);
rpc create_connection(ConnectionRequest) returns (ConnectionResponse);
Expand All @@ -23,11 +23,11 @@ service DozerCloud {
rpc deploy_application(DeployAppRequest) returns (DeployAppResponse);
rpc update_application(UpdateAppRequest) returns (AppResponse);
rpc delete_application(DeleteAppRequest) returns (DeleteAppResponse);
rpc get_application(GetAppRequest) returns (AppResponse);
rpc get_application(GetAppRequest) returns (AppResponse);
rpc stop_dozer(StopRequest) returns (StopResponse);
rpc get_status(GetStatusRequest) returns (GetStatusResponse);
rpc list_deployments(ListDeploymentRequest) returns (ListDeploymentResponse);
rpc list_versions(ListVersionsRequest) returns (ListVersionsResponse);
rpc list_versions(ListVersionsRequest) returns (ListVersionsResponse);
rpc upsert_version(UpsertVersionRequest) returns (UpsertVersionResponse);
rpc set_current_version(SetCurrentVersionRequest) returns (SetCurrentVersionResponse);
rpc list_files(ListFilesRequest) returns (ListFilesResponse);
Expand Down Expand Up @@ -92,7 +92,7 @@ message DeployAppResponse {

message AppResponse {
string app_id = 1;
AppConfig app = 2;
string app_name = 2;
repeated DeploymentInfo deployments = 4;
google.protobuf.Timestamp created_at = 5;
google.protobuf.Timestamp updated_at = 6;
Expand Down Expand Up @@ -151,7 +151,7 @@ message DeploymentResource {
// api, app
string typ = 2;
string created_at = 3;

optional int32 desired = 4;

optional int32 available = 5;
Expand Down Expand Up @@ -224,17 +224,22 @@ message SetCurrentVersionRequest {

message SetCurrentVersionResponse {}

message ConnectionRequest { Connection connection = 1; }
message ConnectionRequest {
string yaml_content = 2;
}
message GetConnectionRequest {
string connection_id = 1;
}
message ValidateConnectionResponse { bool success = 1; }
message ConnectionResponse {
string id = 1;
Connection connection = 2;
string name = 2;
string yaml_content = 3;
}
message Connection {
string name = 1;
string yaml_content = 3;
}

message GetTablesRequest { string connection_id = 2; }

message GetTablesResponse {
Expand All @@ -251,8 +256,9 @@ message GetAllConnectionResponse {
Pagination pagination = 2;
}
message UpdateConnectionRequest {
Connection connection = 1;
string connection_id = 3;
string id = 1;
string name = 2;
string yaml_content = 3;
}
message ListFilesRequest {
string app_id = 1;
Expand Down Expand Up @@ -345,4 +351,4 @@ message ListSecretsRequest {

message ListSecretsResponse {
repeated string secrets = 1;
}
}
185 changes: 0 additions & 185 deletions dozer-types/protos/cloud_types.proto
View file
Original file line number Diff line number Diff line change
@@ -1,142 +1,6 @@
syntax = "proto3";
package dozer.cloud;

message AppConfig {
string app_name = 2;
string home_dir = 3;
string cache_dir = 4;
repeated Connection connections = 5;
repeated Source sources = 6;
repeated Endpoint endpoints = 7;
ApiConfig api = 8;
optional string sql = 9;
Flags flags = 10;
optional uint64 cache_max_map_size = 11;
}
message Flags {
bool dynamic = 1;
bool grpc_web = 2;
bool push_events = 3;
bool authenticate_server_reflection = 4;
EnableProbabilisticOptimizations enable_probabilistic_optimizations = 5;
}

message EnableProbabilisticOptimizations {
bool in_sets = 1;
bool in_joins = 2;
bool in_aggregations = 3;
}

message Connection {
oneof config {
PostgresConfig Postgres = 1;
EthereumConfig Ethereum = 2;
GrpcConfig Grpc = 3;
SnowflakeConfig Snowflake = 4;
KafkaConfig Kafka = 5;
S3Storage S3Storage = 6;
LocalStorage LocalStorage = 7;
DeltaLakeConfig DeltaLake = 8;
}
string name = 9;
}

message ConnectionConfig {
oneof config {
PostgresConfig Postgres = 1;
EthereumConfig Ethereum = 2;
GrpcConfig Grpc = 3;
SnowflakeConfig Snowflake = 4;
KafkaConfig Kafka = 5;
S3Storage S3Storage = 6;
LocalStorage LocalStorage = 7;
DeltaLakeConfig DeltaLake = 8;
}
}
message DeltaLakeConfig {
repeated Table tables = 1;
}
message S3Storage {
S3Details details = 1;
repeated Table tables = 2;
}

message S3Details {
string access_key_id = 1;
string secret_access_key = 2;
string region = 3;
string bucket_name = 4;
}

message LocalStorage {
LocalDetails details = 1;
repeated Table tables = 2;
}

message LocalDetails {
string path = 1;
}

message Table {
string name = 1;
string prefix = 2;
string file_type = 3;
string extension = 4;
}

message SnowflakeConfig {
string server = 1;
string port = 2;
string user = 3;
string password = 4;
string database = 5;
string schema = 6;
string warehouse = 7;
optional string driver = 8;
}
message PostgresConfig {
string user = 1;
string password = 2;
string host = 3;
uint32 port = 4;
string database = 5;
}

message GrpcConfig {
string host = 1;
uint32 port = 2;
oneof schemas {
string Inline = 3;
string Path = 4;
}
string adapter = 5;
}

message KafkaConfig {
string broker = 1;
string topic = 2;
optional string schema_registry_url = 3;
}
message EventsConfig { string database = 1; }

message EthereumConfig {
EthereumFilter filter = 1;
string wss_url = 2;
string name = 3;
repeated EthContract contracts = 4;
}

message EthereumFilter {
optional uint64 from_block = 1;
repeated string addresses = 2;
repeated string topics = 3;
}
message EthContract {
string name = 1;
string address = 2;
string abi = 3;
}

message TableInfo {
string table_name = 1;
repeated ColumnInfo columns = 2;
Expand All @@ -146,55 +10,6 @@ message ColumnInfo {
bool is_nullable = 2;
}

message Endpoint {
string name = 1;
string table_name = 2;
string path = 3;
ApiIndex index = 4;
}

message ApiIndex { repeated string primary_key = 1; }

message Source {
string name = 1;
string table_name = 2;
repeated string columns = 3;
string connection = 4;
optional string schema = 5;
RefreshConfig refresh_config = 7;
}

message ApiConfig {
oneof ApiSecurity { string Jwt = 1; }
RestApiOptions rest = 2;
GrpcApiOptions grpc = 3;
GrpcApiOptions app_grpc = 4;
}

message RestApiOptions {
uint32 port = 1;
string url = 2;
bool cors = 3;
}
message GrpcApiOptions {
uint32 port = 1;
string url = 2;
bool cors = 3;
bool web = 4;
}
message RefreshConfig {
oneof config {
RefreshConfigHour hour = 1;
RefreshConfigDay day = 2;
RefreshConfigCronExpression cron_expression = 3;
RefreshConfigRealTime realtime = 4;
}
}
message RefreshConfigHour { uint32 minute = 1; }
message RefreshConfigDay { string time = 1; }
message RefreshConfigCronExpression { string expression = 1; }
message RefreshConfigRealTime {}

message File {
string name = 1;
string content = 2;
Expand Down
Loading

0 comments on commit 3dd9d32

Please sign in to comment.