From 3cdeb22edad71256609fce8f27c8759323710d30 Mon Sep 17 00:00:00 2001 From: artaasadi Date: Mon, 18 Nov 2024 18:16:31 +0100 Subject: [PATCH 1/6] fix: make steampipe plugin optional --- pkg/describer/worker.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/describer/worker.go b/pkg/describer/worker.go index cbe90565..9482604e 100755 --- a/pkg/describer/worker.go +++ b/pkg/describer/worker.go @@ -124,9 +124,13 @@ func doDescribe( return fmt.Errorf("unmarshal description: %v", err.Error()) } - tags, _, err := steampipe.ExtractTagsAndNames(logger, plg, job.ResourceType, resource) - if err != nil { - logger.Error("failed to build tags for service", zap.Error(err), zap.String("resourceType", job.ResourceType), zap.Any("resource", resource)) + tags := make(map[string]string) + + if plg != nil { + tags, _, err = steampipe.ExtractTagsAndNames(logger, plg, job.ResourceType, resource) + if err != nil { + logger.Error("failed to build tags for service", zap.Error(err), zap.String("resourceType", job.ResourceType), zap.Any("resource", resource)) + } } var description any From 91e99f95e12be1bf87b6a71a752e5316c600ad36 Mon Sep 17 00:00:00 2001 From: artaasadi Date: Wed, 20 Nov 2024 10:37:04 +0100 Subject: [PATCH 2/6] fix: add command for testing --- command/LICENSE | 0 command/cmd/describer.go | 159 +++++++++++++++++++++++++++++++++++++++ command/cmd/root.go | 34 +++++++++ command/main.go | 10 +++ 4 files changed, 203 insertions(+) create mode 100644 command/LICENSE create mode 100644 command/cmd/describer.go create mode 100644 command/cmd/root.go create mode 100644 command/main.go diff --git a/command/LICENSE b/command/LICENSE new file mode 100644 index 00000000..e69de29b diff --git a/command/cmd/describer.go b/command/cmd/describer.go new file mode 100644 index 00000000..ad3e5578 --- /dev/null +++ b/command/cmd/describer.go @@ -0,0 +1,159 @@ +/* +Copyright © 2023 NAME HERE +*/ +package cmd + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "github.com/opengovern/og-describer-template/pkg/describer" + model "github.com/opengovern/og-describer-template/pkg/sdk/models" + "github.com/opengovern/og-describer-template/provider" + "github.com/opengovern/og-describer-template/provider/configs" + "github.com/opengovern/og-describer-template/steampipe" + "github.com/opengovern/og-util/pkg/describe" + "github.com/opengovern/og-util/pkg/es" + "github.com/spf13/cobra" + "go.uber.org/zap" + "golang.org/x/net/context" + "strconv" + "strings" + "time" +) + +var ( + resourceType string +) + +// describerCmd represents the describer command +var describerCmd = &cobra.Command{ + Use: "describer", + Short: "A brief description of your command", + RunE: func(cmd *cobra.Command, args []string) error { + + job := describe.DescribeJob{ + JobID: uint(uuid.New().ID()), + ResourceType: resourceType, + IntegrationID: "", + ProviderID: "", + DescribedAt: time.Now().UnixMilli(), + IntegrationType: configs.IntegrationTypeLower, + CipherText: "", + IntegrationLabels: nil, + IntegrationAnnotations: nil, + } + + ctx := context.Background() + logger, _ := zap.NewProduction() + + // TODO: Set the credentials + creds := configs.IntegrationCredentials{} + + additionalParameters, err := provider.GetAdditionalParameters(job) + if err != nil { + return err + } + plg := steampipe.Plugin() + + f := func(resource model.Resource) error { + if resource.Description == nil { + return nil + } + descriptionJSON, err := json.Marshal(resource.Description) + if err != nil { + return fmt.Errorf("failed to marshal description: %w", err) + } + descriptionJSON, err = trimJsonFromEmptyObjects(descriptionJSON) + if err != nil { + return fmt.Errorf("failed to trim json: %w", err) + } + + metadata, err := provider.GetResourceMetadata(job, resource) + if err != nil { + return fmt.Errorf("failed to get resource metadata") + } + err = provider.AdjustResource(job, &resource) + if err != nil { + return fmt.Errorf("failed to get resource metadata") + } + + desc := resource.Description + err = json.Unmarshal(descriptionJSON, &desc) + if err != nil { + return fmt.Errorf("unmarshal description: %v", err.Error()) + } + + if plg != nil { + _, _, err = steampipe.ExtractTagsAndNames(logger, plg, job.ResourceType, resource) + if err != nil { + logger.Error("failed to build tags for service", zap.Error(err), zap.String("resourceType", job.ResourceType), zap.Any("resource", resource)) + } + } + + var description any + err = json.Unmarshal([]byte(descriptionJSON), &description) + if err != nil { + logger.Error("failed to parse resource description json", zap.Error(err)) + return fmt.Errorf("failed to parse resource description json") + } + + fmt.Println(es.Resource{ + PlatformID: fmt.Sprintf("%s:::%s:::%s", job.IntegrationID, job.ResourceType, resource.UniqueID()), + ResourceID: resource.UniqueID(), + ResourceName: resource.Name, + Description: description, + IntegrationType: configs.IntegrationName, + ResourceType: strings.ToLower(job.ResourceType), + IntegrationID: job.IntegrationID, + Metadata: metadata, + DescribedAt: job.DescribedAt, + DescribedBy: strconv.FormatUint(uint64(job.JobID), 10), + }) + return nil + } + clientStream := (*model.StreamSender)(&f) + + err = describer.GetResources( + ctx, + logger, + job.ResourceType, + job.TriggerType, + creds, + additionalParameters, + clientStream, + ) + if err != nil { + return err + } + return nil + }, +} + +func init() { + describerCmd.Flags().StringVar(&resourceType, "resourceType", "", "Resource type") +} + +func trimJsonFromEmptyObjects(input []byte) ([]byte, error) { + unknownData := map[string]any{} + err := json.Unmarshal(input, &unknownData) + if err != nil { + return nil, err + } + trimEmptyMaps(unknownData) + return json.Marshal(unknownData) +} + +func trimEmptyMaps(input map[string]any) { + for key, value := range input { + switch value.(type) { + case map[string]any: + if len(value.(map[string]any)) != 0 { + trimEmptyMaps(value.(map[string]any)) + } + if len(value.(map[string]any)) == 0 { + delete(input, key) + } + } + } +} diff --git a/command/cmd/root.go b/command/cmd/root.go new file mode 100644 index 00000000..48d08ce2 --- /dev/null +++ b/command/cmd/root.go @@ -0,0 +1,34 @@ +/* +Copyright © 2023 NAME HERE +*/ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "run", + Short: "OpenGovernance aws describer manual", + RunE: func(cmd *cobra.Command, args []string) error { + var items []string + items = append(items, "describer") + + return describerCmd.Help() + + }, +} + +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + rootCmd.AddCommand(describerCmd) +} diff --git a/command/main.go b/command/main.go new file mode 100644 index 00000000..e2f92487 --- /dev/null +++ b/command/main.go @@ -0,0 +1,10 @@ +/* +Copyright © 2023 NAME HERE +*/ +package main + +import "github.com/opengovern/og-aws-describer/command/cmd" + +func main() { + cmd.Execute() +} From 50e399240563bc32f251794bd81e8ecae6109082 Mon Sep 17 00:00:00 2001 From: artaasadi Date: Wed, 20 Nov 2024 10:37:28 +0100 Subject: [PATCH 3/6] fix: add command for testing --- command/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/main.go b/command/main.go index e2f92487..771a4018 100644 --- a/command/main.go +++ b/command/main.go @@ -3,7 +3,7 @@ Copyright © 2023 NAME HERE */ package main -import "github.com/opengovern/og-aws-describer/command/cmd" +import "github.com/opengovern/og-describer-template/command/cmd" func main() { cmd.Execute() From 4893358d265473b98dcde88c333579efc6f538c9 Mon Sep 17 00:00:00 2001 From: artaasadi Date: Wed, 20 Nov 2024 10:54:52 +0100 Subject: [PATCH 4/6] fix: output test data to file --- command/cmd/describer.go | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/command/cmd/describer.go b/command/cmd/describer.go index ad3e5578..455354c3 100644 --- a/command/cmd/describer.go +++ b/command/cmd/describer.go @@ -1,6 +1,3 @@ -/* -Copyright © 2023 NAME HERE -*/ package cmd import ( @@ -17,6 +14,7 @@ import ( "github.com/spf13/cobra" "go.uber.org/zap" "golang.org/x/net/context" + "os" "strconv" "strings" "time" @@ -24,6 +22,7 @@ import ( var ( resourceType string + outputFile string ) // describerCmd represents the describer command @@ -31,6 +30,12 @@ var describerCmd = &cobra.Command{ Use: "describer", Short: "A brief description of your command", RunE: func(cmd *cobra.Command, args []string) error { + // Open the output file + file, err := os.Create(outputFile) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer file.Close() // Ensure the file is closed at the end job := describe.DescribeJob{ JobID: uint(uuid.New().ID()), @@ -75,7 +80,7 @@ var describerCmd = &cobra.Command{ } err = provider.AdjustResource(job, &resource) if err != nil { - return fmt.Errorf("failed to get resource metadata") + return fmt.Errorf("failed to adjust resource metadata") } desc := resource.Description @@ -98,7 +103,7 @@ var describerCmd = &cobra.Command{ return fmt.Errorf("failed to parse resource description json") } - fmt.Println(es.Resource{ + res := es.Resource{ PlatformID: fmt.Sprintf("%s:::%s:::%s", job.IntegrationID, job.ResourceType, resource.UniqueID()), ResourceID: resource.UniqueID(), ResourceName: resource.Name, @@ -109,7 +114,22 @@ var describerCmd = &cobra.Command{ Metadata: metadata, DescribedAt: job.DescribedAt, DescribedBy: strconv.FormatUint(uint64(job.JobID), 10), - }) + } + + // Write the resource JSON to the file + resJSON, err := json.Marshal(res) + if err != nil { + return fmt.Errorf("failed to marshal resource JSON: %w", err) + } + _, err = file.Write(resJSON) + if err != nil { + return fmt.Errorf("failed to write to file: %w", err) + } + _, err = file.Write([]byte(",\n")) // Add a newline for readability + if err != nil { + return fmt.Errorf("failed to write newline to file: %w", err) + } + return nil } clientStream := (*model.StreamSender)(&f) @@ -132,6 +152,7 @@ var describerCmd = &cobra.Command{ func init() { describerCmd.Flags().StringVar(&resourceType, "resourceType", "", "Resource type") + describerCmd.Flags().StringVar(&outputFile, "outputFile", "output.json", "File to write JSON outputs") } func trimJsonFromEmptyObjects(input []byte) ([]byte, error) { From a4cddcf0978bd02c09fe7fe21ef55106032befd8 Mon Sep 17 00:00:00 2001 From: artaasadi Date: Wed, 20 Nov 2024 14:50:24 +0100 Subject: [PATCH 5/6] fix: add getDescribers --- pkg/describer/resources.go | 27 +++++++++++++++++++++++++++ pkg/sdk/models/resource_type.go | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/pkg/describer/resources.go b/pkg/describer/resources.go index 7418f05e..fa52865e 100755 --- a/pkg/describer/resources.go +++ b/pkg/describer/resources.go @@ -69,3 +69,30 @@ func describe(ctx context.Context, logger *zap.Logger, accountCfg configs.Integr return resourceTypeObject.ListDescriber(ctx, accountCfg, triggerType, additionalParameters, stream) } + +func GetSingleResource( + ctx context.Context, + logger *zap.Logger, + resourceType string, + triggerType enums.DescribeTriggerType, + cfg configs.IntegrationCredentials, + additionalParameters map[string]string, + resourceId string, + stream *model.StreamSender, +) error { + _, err := describeSingle(ctx, logger, cfg, resourceType, resourceId, triggerType, additionalParameters, stream) + if err != nil { + return err + } + return nil +} + +func describeSingle(ctx context.Context, logger *zap.Logger, accountCfg configs.IntegrationCredentials, resourceType string, resourceID string, triggerType enums.DescribeTriggerType, additionalParameters map[string]string, stream *model.StreamSender) (*model.Resource, error) { + resourceTypeObject, ok := provider.ResourceTypes[resourceType] + if !ok { + return nil, fmt.Errorf("unsupported resource type: %s", resourceType) + } + ctx = describer.WithLogger(ctx, logger) + + return resourceTypeObject.GetDescriber(ctx, accountCfg, triggerType, resourceID, additionalParameters) +} diff --git a/pkg/sdk/models/resource_type.go b/pkg/sdk/models/resource_type.go index 9a50252f..adb80425 100644 --- a/pkg/sdk/models/resource_type.go +++ b/pkg/sdk/models/resource_type.go @@ -9,7 +9,7 @@ import ( // any types are used to load your provider configuration. type ResourceDescriber func(context.Context, configs.IntegrationCredentials, enums.DescribeTriggerType, map[string]string, *StreamSender) ([]Resource, error) -type SingleResourceDescriber func(context.Context, configs.IntegrationCredentials, enums.DescribeTriggerType, map[string]string) (*Resource, error) +type SingleResourceDescriber func(context.Context, configs.IntegrationCredentials, enums.DescribeTriggerType, string, map[string]string) (*Resource, error) type ResourceType struct { IntegrationType integration.Type From bd36020030a880ff7783a2e92efed2edffc27195 Mon Sep 17 00:00:00 2001 From: artaasadi Date: Wed, 20 Nov 2024 14:53:16 +0100 Subject: [PATCH 6/6] fix: add getDescribers teste --- command/cmd/getDescriber.go | 157 ++++++++++++++++++++++++++++++++++++ command/cmd/root.go | 1 + 2 files changed, 158 insertions(+) create mode 100644 command/cmd/getDescriber.go diff --git a/command/cmd/getDescriber.go b/command/cmd/getDescriber.go new file mode 100644 index 00000000..9595cdc1 --- /dev/null +++ b/command/cmd/getDescriber.go @@ -0,0 +1,157 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "github.com/opengovern/og-describer-template/pkg/describer" + model "github.com/opengovern/og-describer-template/pkg/sdk/models" + "github.com/opengovern/og-describer-template/provider" + "github.com/opengovern/og-describer-template/provider/configs" + "github.com/opengovern/og-describer-template/steampipe" + "github.com/opengovern/og-util/pkg/describe" + "github.com/opengovern/og-util/pkg/es" + "github.com/spf13/cobra" + "go.uber.org/zap" + "golang.org/x/net/context" + "os" + "strconv" + "strings" + "time" +) + +var ( + resourceID string +) + +// getDescriberCmd represents the describer command +var getDescriberCmd = &cobra.Command{ + Use: "getDescriber", + Short: "A brief description of your command", + RunE: func(cmd *cobra.Command, args []string) error { + // Open the output file + file, err := os.Create(outputFile) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer file.Close() // Ensure the file is closed at the end + + job := describe.DescribeJob{ + JobID: uint(uuid.New().ID()), + ResourceType: resourceType, + IntegrationID: "", + ProviderID: "", + DescribedAt: time.Now().UnixMilli(), + IntegrationType: configs.IntegrationTypeLower, + CipherText: "", + IntegrationLabels: nil, + IntegrationAnnotations: nil, + } + + ctx := context.Background() + logger, _ := zap.NewProduction() + + // TODO: Set the credentials + creds := configs.IntegrationCredentials{} + + additionalParameters, err := provider.GetAdditionalParameters(job) + if err != nil { + return err + } + plg := steampipe.Plugin() + + f := func(resource model.Resource) error { + if resource.Description == nil { + return nil + } + descriptionJSON, err := json.Marshal(resource.Description) + if err != nil { + return fmt.Errorf("failed to marshal description: %w", err) + } + descriptionJSON, err = trimJsonFromEmptyObjects(descriptionJSON) + if err != nil { + return fmt.Errorf("failed to trim json: %w", err) + } + + metadata, err := provider.GetResourceMetadata(job, resource) + if err != nil { + return fmt.Errorf("failed to get resource metadata") + } + err = provider.AdjustResource(job, &resource) + if err != nil { + return fmt.Errorf("failed to adjust resource metadata") + } + + desc := resource.Description + err = json.Unmarshal(descriptionJSON, &desc) + if err != nil { + return fmt.Errorf("unmarshal description: %v", err.Error()) + } + + if plg != nil { + _, _, err = steampipe.ExtractTagsAndNames(logger, plg, job.ResourceType, resource) + if err != nil { + logger.Error("failed to build tags for service", zap.Error(err), zap.String("resourceType", job.ResourceType), zap.Any("resource", resource)) + } + } + + var description any + err = json.Unmarshal([]byte(descriptionJSON), &description) + if err != nil { + logger.Error("failed to parse resource description json", zap.Error(err)) + return fmt.Errorf("failed to parse resource description json") + } + + res := es.Resource{ + PlatformID: fmt.Sprintf("%s:::%s:::%s", job.IntegrationID, job.ResourceType, resource.UniqueID()), + ResourceID: resource.UniqueID(), + ResourceName: resource.Name, + Description: description, + IntegrationType: configs.IntegrationName, + ResourceType: strings.ToLower(job.ResourceType), + IntegrationID: job.IntegrationID, + Metadata: metadata, + DescribedAt: job.DescribedAt, + DescribedBy: strconv.FormatUint(uint64(job.JobID), 10), + } + + // Write the resource JSON to the file + resJSON, err := json.Marshal(res) + if err != nil { + return fmt.Errorf("failed to marshal resource JSON: %w", err) + } + _, err = file.Write(resJSON) + if err != nil { + return fmt.Errorf("failed to write to file: %w", err) + } + _, err = file.Write([]byte(",\n")) // Add a newline for readability + if err != nil { + return fmt.Errorf("failed to write newline to file: %w", err) + } + + return nil + } + clientStream := (*model.StreamSender)(&f) + + err = describer.GetSingleResource( + ctx, + logger, + job.ResourceType, + job.TriggerType, + creds, + additionalParameters, + resourceID, + clientStream, + ) + if err != nil { + return err + } + return nil + }, +} + +func init() { + getDescriberCmd.Flags().StringVar(&resourceType, "resourceType", "", "Resource type") + getDescriberCmd.Flags().StringVar(&resourceID, "resourceID", "", "Resource ID") + getDescriberCmd.Flags().StringVar(&outputFile, "outputFile", "output.json", "File to write JSON outputs") +} diff --git a/command/cmd/root.go b/command/cmd/root.go index 48d08ce2..5ca36e29 100644 --- a/command/cmd/root.go +++ b/command/cmd/root.go @@ -31,4 +31,5 @@ func Execute() { func init() { rootCmd.AddCommand(describerCmd) + rootCmd.AddCommand(getDescriberCmd) }