Skip to content

Commit

Permalink
feat: update proto for internal ingest
Browse files Browse the repository at this point in the history
  • Loading branch information
Mahanmmi committed Apr 15, 2024
1 parent aa580ba commit 6c96639
Show file tree
Hide file tree
Showing 12 changed files with 1,279 additions and 996 deletions.
6 changes: 5 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/kaytu-io/kaytu-util
go 1.21

require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0
github.com/aws/aws-sdk-go-v2 v1.24.1
Expand All @@ -12,8 +13,10 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.26.7
github.com/confluentinc/confluent-kafka-go/v2 v2.3.0
github.com/elastic/go-elasticsearch/v7 v7.17.10
github.com/envoyproxy/go-control-plane v0.11.1
github.com/fluxcd/helm-controller/api v0.37.4
github.com/globocom/echo-prometheus v0.1.2
github.com/gogo/googleapis v1.4.1
github.com/golang/protobuf v1.5.3
github.com/google/uuid v1.6.0
github.com/hashicorp/go-hclog v1.6.2
Expand Down Expand Up @@ -59,7 +62,6 @@ require (
cloud.google.com/go/iam v1.1.5 // indirect
cloud.google.com/go/storage v1.36.0 // indirect
dario.cat/mergo v1.0.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
Expand All @@ -86,6 +88,7 @@ require (
github.com/btubbs/datetime v0.1.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 // indirect
github.com/containerd/continuity v0.4.3 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect
Expand All @@ -101,6 +104,7 @@ require (
github.com/eko/gocache/store/bigcache/v4 v4.2.1 // indirect
github.com/eko/gocache/store/ristretto/v4 v4.2.1 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect
github.com/evanphx/json-patch/v5 v5.8.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/fatih/structs v1.1.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM=
github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
Expand Down Expand Up @@ -435,6 +437,8 @@ github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDs
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0=
github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0=
github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
Expand Down
44 changes: 44 additions & 0 deletions pkg/grpc/auth.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package grpc

import (
"context"
envoyAuth "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3"
"github.com/gogo/googleapis/google/rpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"net/http"
)

func CheckGRPCAuth(ctx context.Context, authClient envoyAuth.AuthorizationClient) error {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return status.Errorf(codes.Unauthenticated, "missing metadata")
}

mdHeaders := make(map[string]string)
for k, v := range md {
if len(v) > 0 {
mdHeaders[k] = v[0]
}
}

result, err := authClient.Check(ctx, &envoyAuth.CheckRequest{
Attributes: &envoyAuth.AttributeContext{
Request: &envoyAuth.AttributeContext_Request{
Http: &envoyAuth.AttributeContext_HttpRequest{
Headers: mdHeaders,
},
},
},
})
if err != nil {
return status.Errorf(codes.Unauthenticated, "authentication failed: %v", err)
}

if result.GetStatus() == nil || result.GetStatus().GetCode() != int32(rpc.OK) {
return status.Errorf(codes.Unauthenticated, http.StatusText(http.StatusUnauthorized))
}

return nil
}
65 changes: 5 additions & 60 deletions proto/describe.proto
Original file line number Diff line number Diff line change
Expand Up @@ -2,66 +2,16 @@ syntax = "proto3";

package kaytu.describe.v1;

option go_package="github.com/kaytu-io/kaytu-engine/pkg/describe/proto/src/golang";
option go_package="github.com/kaytu-io/kaytu-util/proto/src/golang";

message DescribeJob {
uint32 job_id = 1;
uint32 schedule_job_id = 2;
uint32 parent_job_id = 3;
string resource_type = 4;
string source_id = 5;
string account_id = 6;
int64 described_at = 7;
string source_type = 8;
string config_reg = 9;
string trigger_type = 10;
uint32 retry_counter = 11;
}

message AWSResources {
repeated AWSResource resources = 1;
string kafka_topic = 2;
}

message AWSResource {
string arn = 1;
string id = 2;
string name = 3;
string account = 4;
string region = 5;
string partition = 6;
string type = 7;
string description_json = 8;
DescribeJob job = 9;
string unique_id = 10;
map<string,string> metadata = 11;
map<string,string> tags = 12;
}

message AzureResources {
repeated AzureResource resources = 1;
string kafka_topic = 2;
}
message AzureResource {
string id = 1;
string name = 2;
string type = 3;
string resource_group = 4;
string location = 5;
string subscription_id = 6;
string description_json = 7;
DescribeJob job = 8;
string unique_id = 9;
map<string,string> metadata = 10;
map<string,string> tags = 11;
}
import "entity.proto";

message DeliverResultRequest {
uint32 job_id = 1;
uint32 parent_job_id = 2;
string status = 3;
string error = 4;
DescribeJob describe_job = 5;
kaytu.entity.v1.DescribeJob describe_job = 5;
repeated string described_resource_ids = 6;
string errorCode = 7;
}
Expand All @@ -70,12 +20,7 @@ message SetInProgressRequest {
uint32 job_id = 1;
}

message ResponseOK {
}

service DescribeService {
rpc DeliverResult(DeliverResultRequest) returns (ResponseOK) {}
rpc SetInProgress(SetInProgressRequest) returns (ResponseOK) {}
rpc DeliverAWSResources(AWSResources) returns (ResponseOK) {}
rpc DeliverAzureResources(AzureResources) returns (ResponseOK) {}
rpc DeliverResult(DeliverResultRequest) returns (kaytu.entity.v1.ResponseOK) {}
rpc SetInProgress(SetInProgressRequest) returns (kaytu.entity.v1.ResponseOK) {}
}
22 changes: 22 additions & 0 deletions proto/entity.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
syntax = "proto3";

package kaytu.entity.v1;

option go_package="github.com/kaytu-io/kaytu-util/proto/src/golang";

message DescribeJob {
uint32 job_id = 1;
uint32 schedule_job_id = 2;
uint32 parent_job_id = 3;
string resource_type = 4;
string source_id = 5;
string account_id = 6;
int64 described_at = 7;
string source_type = 8;
string config_reg = 9;
string trigger_type = 10;
uint32 retry_counter = 11;
}

message ResponseOK {
}
48 changes: 48 additions & 0 deletions proto/es-sink.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
syntax = "proto3";

package kaytu.es_sink.v1;

option go_package="github.com/kaytu-io/kaytu-util/proto/src/golang";

import "entity.proto";

message AWSResources {
repeated AWSResource resources = 1;
}

message AWSResource {
string arn = 1;
string id = 2;
string name = 3;
string account = 4;
string region = 5;
string partition = 6;
string type = 7;
string description_json = 8;
kaytu.entity.v1.DescribeJob job = 9;
string unique_id = 10;
map<string,string> metadata = 11;
map<string,string> tags = 12;
}

message AzureResources {
repeated AzureResource resources = 1;
}
message AzureResource {
string id = 1;
string name = 2;
string type = 3;
string resource_group = 4;
string location = 5;
string subscription_id = 6;
string description_json = 7;
kaytu.entity.v1.DescribeJob job = 8;
string unique_id = 9;
map<string,string> metadata = 10;
map<string,string> tags = 11;
}

service DescribeService {
rpc DeliverAWSResources(AWSResources) returns (kaytu.entity.v1.ResponseOK) {}
rpc DeliverAzureResources(AzureResources) returns (kaytu.entity.v1.ResponseOK) {}
}
Loading

0 comments on commit 6c96639

Please sign in to comment.