feat: implment connections json schema (#2069)

* chore: create json schema from connection types * chore: remove redudant code * chore: fix tests * chore: fix grpc tests
getdozer · Sep 22, 2023 · 0970f91 · 0970f91
1 parent 556ce6b
commit 0970f91
Show file tree

Hide file tree

Showing 25 changed files with 3,181 additions and 542 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/dozer-cli/Cargo.toml b/dozer-cli/Cargo.toml
@@ -49,6 +49,9 @@ tempfile = "3.8"
 actix-files = "0.6.2"
 prometheus-parse = "0.2.4"
 
+[build-dependencies]
+dozer-types = { path = "../dozer-types" }
+
 [[bin]]
 edition = "2021"
 name = "dozer"

diff --git a/dozer-cli/build.rs b/dozer-cli/build.rs
@@ -0,0 +1,22 @@
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+
+fn main() {
+    let schema_path = Path::new("../json_schemas");
+    // Define the path to the file we want to create or overwrite
+    let connection_path = schema_path.join("connections.json");
+    let dozer_path = schema_path.join("dozer.json");
+
+    let mut file = File::create(connection_path).expect("Failed to create connections.json");
+    let schemas = dozer_types::models::get_connection_schemas().unwrap();
+    write!(file, "{}", schemas).expect("Unable to write file");
+
+    let mut dozer_schema_file = File::create(dozer_path).expect("Failed to create dozer.json");
+    let schema = dozer_types::models::get_dozer_schema().unwrap();
+    write!(dozer_schema_file, "{}", schema).expect("Unable to write file");
+
+    // Print a message to indicate the file has been written
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("Written to {:?}", schema_path.display());
+}
diff --git a/dozer-cli/src/simple/cloud_orchestrator.rs b/dozer-cli/src/simple/cloud_orchestrator.rs
@@ -193,9 +193,7 @@ impl CloudOrchestrator for SimpleOrchestrator {
             let mut table = table!();
 
             for app in response.apps {
-                if let Some(app_data) = app.app {
-                    table.add_row(row![app.app_id, app_data.convert_to_table()]);
-                }
+                table.add_row(row![app.app_id, app.app_name]);
             }
 
             table.printstd();

diff --git a/dozer-types/Cargo.toml b/dozer-types/Cargo.toml
@@ -37,6 +37,7 @@ tokio-postgres = { version = "0.7.7", features = [
 ] }
 serde_bytes = "0.11.12"
 arbitrary = { version = "1", features = ["derive"], optional = true }
+schemars = "0.8.15"
 
 [build-dependencies]
 tonic-build = "0.10.0"

diff --git a/dozer-types/build.rs b/dozer-types/build.rs
@@ -49,64 +49,6 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Cloud Service & Types
     tonic_build::configure()
         .protoc_arg("--experimental_allow_proto3_optional")
-        .extern_path(
-            ".dozer.cloud.Endpoint",
-            "crate::models::api_endpoint::ApiEndpoint",
-        )
-        .extern_path(".dozer.cloud.Source", "crate::models::source::Source")
-        .extern_path(".dozer.cloud.AppConfig", "crate::models::config::Config")
-        .extern_path(
-            ".dozer.cloud.ConnectionConfig",
-            "crate::models::connection::ConnectionConfig",
-        )
-        .extern_path(
-            ".dozer.cloud.Connection",
-            "crate::models::connection::Connection",
-        )
-        .extern_path(
-            ".dozer.cloud.EthContract",
-            "crate::ingestion_types::EthContract",
-        )
-        .extern_path(
-            ".dozer.cloud.EthereumFilter",
-            "crate::ingestion_types::EthereumFilter",
-        )
-        .extern_path(
-            ".dozer.cloud.DeltaLakeConfig",
-            "crate::ingestion_types::DeltaLakeConfig",
-        )
-        .extern_path(
-            ".dozer.cloud.LocalStorage",
-            "crate::ingestion_types::LocalStorage",
-        )
-        .extern_path(
-            ".dozer.cloud.S3Storage",
-            "crate::ingestion_types::S3Storage",
-        )
-        .extern_path(
-            ".dozer.cloud.KafkaConfig",
-            "crate::ingestion_types::KafkaConfig",
-        )
-        .extern_path(
-            ".dozer.cloud.SnowflakeConfig",
-            "crate::ingestion_types::SnowflakeConfig",
-        )
-        .extern_path(
-            ".dozer.cloud::grpc_config::Schemas",
-            "crate::ingestion_types::GrpcConfigSchemas",
-        )
-        .extern_path(
-            ".dozer.cloud.GrpcConfig",
-            "crate::ingestion_types::GrpcConfig",
-        )
-        .extern_path(
-            ".dozer.cloud.EthereumConfig",
-            "crate::ingestion_types::EthConfig",
-        )
-        .extern_path(
-            ".dozer.cloud.PostgresConfig",
-            "crate::models::connection::PostgresConfig",
-        )
         .file_descriptor_set_path(out_dir.join("cloud.bin"))
         .compile(&["protos/cloud.proto"], &["protos"])
         .unwrap();

diff --git a/dozer-types/protos/cloud.proto b/dozer-types/protos/cloud.proto
@@ -6,7 +6,7 @@ import "google/protobuf/timestamp.proto";
 import "google/protobuf/empty.proto";
 
 service DozerCloud {
-  
+
   rpc validate_connection(ConnectionRequest)
       returns (ValidateConnectionResponse);
   rpc create_connection(ConnectionRequest) returns (ConnectionResponse);
@@ -23,11 +23,11 @@ service DozerCloud {
   rpc deploy_application(DeployAppRequest) returns (DeployAppResponse);
   rpc update_application(UpdateAppRequest) returns (AppResponse);
   rpc delete_application(DeleteAppRequest) returns (DeleteAppResponse);
-  rpc get_application(GetAppRequest) returns (AppResponse);  
+  rpc get_application(GetAppRequest) returns (AppResponse);
   rpc stop_dozer(StopRequest) returns (StopResponse);
   rpc get_status(GetStatusRequest) returns (GetStatusResponse);
   rpc list_deployments(ListDeploymentRequest) returns (ListDeploymentResponse);
-  rpc list_versions(ListVersionsRequest) returns (ListVersionsResponse);  
+  rpc list_versions(ListVersionsRequest) returns (ListVersionsResponse);
   rpc upsert_version(UpsertVersionRequest) returns (UpsertVersionResponse);
   rpc set_current_version(SetCurrentVersionRequest) returns (SetCurrentVersionResponse);
   rpc list_files(ListFilesRequest) returns (ListFilesResponse);
@@ -92,7 +92,7 @@ message DeployAppResponse {
 
 message AppResponse {
   string app_id = 1;
-  AppConfig app = 2;
+  string app_name = 2;
   repeated DeploymentInfo deployments = 4;
   google.protobuf.Timestamp created_at = 5;
   google.protobuf.Timestamp updated_at = 6;
@@ -151,7 +151,7 @@ message DeploymentResource {
   // api, app
   string typ = 2;
   string created_at = 3;
-  
+
   optional int32 desired = 4;
 
   optional int32 available = 5;
@@ -224,17 +224,22 @@ message SetCurrentVersionRequest {
 
 message SetCurrentVersionResponse {}
 
-message ConnectionRequest { Connection connection = 1; }
+message ConnectionRequest {
+  string yaml_content = 2;
+}
 message GetConnectionRequest {
   string connection_id = 1;
 }
 message ValidateConnectionResponse { bool success = 1; }
 message ConnectionResponse {
   string id = 1;
-  Connection connection = 2;
+  string name = 2;
+  string yaml_content = 3;
+}
+message Connection {
+  string name = 1;
   string yaml_content = 3;
 }
-
 message GetTablesRequest { string connection_id = 2; }
 
 message GetTablesResponse {
@@ -251,8 +256,9 @@ message GetAllConnectionResponse {
   Pagination pagination = 2;
 }
 message UpdateConnectionRequest {
-  Connection connection = 1;
-  string connection_id = 3;
+  string id = 1;
+  string name = 2;
+  string yaml_content = 3;
 }
 message ListFilesRequest {
   string app_id = 1;
@@ -345,4 +351,4 @@ message ListSecretsRequest {
 
 message ListSecretsResponse {
   repeated string secrets = 1;
-}
+}