diff --git a/command/LICENSE b/command/LICENSE new file mode 100644 index 00000000..e69de29b diff --git a/command/cmd/describer.go b/command/cmd/describer.go new file mode 100644 index 00000000..455354c3 --- /dev/null +++ b/command/cmd/describer.go @@ -0,0 +1,180 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "github.com/opengovern/og-describer-template/pkg/describer" + model "github.com/opengovern/og-describer-template/pkg/sdk/models" + "github.com/opengovern/og-describer-template/provider" + "github.com/opengovern/og-describer-template/provider/configs" + "github.com/opengovern/og-describer-template/steampipe" + "github.com/opengovern/og-util/pkg/describe" + "github.com/opengovern/og-util/pkg/es" + "github.com/spf13/cobra" + "go.uber.org/zap" + "golang.org/x/net/context" + "os" + "strconv" + "strings" + "time" +) + +var ( + resourceType string + outputFile string +) + +// describerCmd represents the describer command +var describerCmd = &cobra.Command{ + Use: "describer", + Short: "A brief description of your command", + RunE: func(cmd *cobra.Command, args []string) error { + // Open the output file + file, err := os.Create(outputFile) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer file.Close() // Ensure the file is closed at the end + + job := describe.DescribeJob{ + JobID: uint(uuid.New().ID()), + ResourceType: resourceType, + IntegrationID: "", + ProviderID: "", + DescribedAt: time.Now().UnixMilli(), + IntegrationType: configs.IntegrationTypeLower, + CipherText: "", + IntegrationLabels: nil, + IntegrationAnnotations: nil, + } + + ctx := context.Background() + logger, _ := zap.NewProduction() + + // TODO: Set the credentials + creds := configs.IntegrationCredentials{} + + additionalParameters, err := provider.GetAdditionalParameters(job) + if err != nil { + return err + } + plg := steampipe.Plugin() + + f := func(resource model.Resource) error { + if resource.Description == nil { + return nil + } + descriptionJSON, err := json.Marshal(resource.Description) + if err != nil { + return fmt.Errorf("failed to marshal description: %w", err) + } + descriptionJSON, err = trimJsonFromEmptyObjects(descriptionJSON) + if err != nil { + return fmt.Errorf("failed to trim json: %w", err) + } + + metadata, err := provider.GetResourceMetadata(job, resource) + if err != nil { + return fmt.Errorf("failed to get resource metadata") + } + err = provider.AdjustResource(job, &resource) + if err != nil { + return fmt.Errorf("failed to adjust resource metadata") + } + + desc := resource.Description + err = json.Unmarshal(descriptionJSON, &desc) + if err != nil { + return fmt.Errorf("unmarshal description: %v", err.Error()) + } + + if plg != nil { + _, _, err = steampipe.ExtractTagsAndNames(logger, plg, job.ResourceType, resource) + if err != nil { + logger.Error("failed to build tags for service", zap.Error(err), zap.String("resourceType", job.ResourceType), zap.Any("resource", resource)) + } + } + + var description any + err = json.Unmarshal([]byte(descriptionJSON), &description) + if err != nil { + logger.Error("failed to parse resource description json", zap.Error(err)) + return fmt.Errorf("failed to parse resource description json") + } + + res := es.Resource{ + PlatformID: fmt.Sprintf("%s:::%s:::%s", job.IntegrationID, job.ResourceType, resource.UniqueID()), + ResourceID: resource.UniqueID(), + ResourceName: resource.Name, + Description: description, + IntegrationType: configs.IntegrationName, + ResourceType: strings.ToLower(job.ResourceType), + IntegrationID: job.IntegrationID, + Metadata: metadata, + DescribedAt: job.DescribedAt, + DescribedBy: strconv.FormatUint(uint64(job.JobID), 10), + } + + // Write the resource JSON to the file + resJSON, err := json.Marshal(res) + if err != nil { + return fmt.Errorf("failed to marshal resource JSON: %w", err) + } + _, err = file.Write(resJSON) + if err != nil { + return fmt.Errorf("failed to write to file: %w", err) + } + _, err = file.Write([]byte(",\n")) // Add a newline for readability + if err != nil { + return fmt.Errorf("failed to write newline to file: %w", err) + } + + return nil + } + clientStream := (*model.StreamSender)(&f) + + err = describer.GetResources( + ctx, + logger, + job.ResourceType, + job.TriggerType, + creds, + additionalParameters, + clientStream, + ) + if err != nil { + return err + } + return nil + }, +} + +func init() { + describerCmd.Flags().StringVar(&resourceType, "resourceType", "", "Resource type") + describerCmd.Flags().StringVar(&outputFile, "outputFile", "output.json", "File to write JSON outputs") +} + +func trimJsonFromEmptyObjects(input []byte) ([]byte, error) { + unknownData := map[string]any{} + err := json.Unmarshal(input, &unknownData) + if err != nil { + return nil, err + } + trimEmptyMaps(unknownData) + return json.Marshal(unknownData) +} + +func trimEmptyMaps(input map[string]any) { + for key, value := range input { + switch value.(type) { + case map[string]any: + if len(value.(map[string]any)) != 0 { + trimEmptyMaps(value.(map[string]any)) + } + if len(value.(map[string]any)) == 0 { + delete(input, key) + } + } + } +} diff --git a/command/cmd/getDescriber.go b/command/cmd/getDescriber.go new file mode 100644 index 00000000..9595cdc1 --- /dev/null +++ b/command/cmd/getDescriber.go @@ -0,0 +1,157 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "github.com/opengovern/og-describer-template/pkg/describer" + model "github.com/opengovern/og-describer-template/pkg/sdk/models" + "github.com/opengovern/og-describer-template/provider" + "github.com/opengovern/og-describer-template/provider/configs" + "github.com/opengovern/og-describer-template/steampipe" + "github.com/opengovern/og-util/pkg/describe" + "github.com/opengovern/og-util/pkg/es" + "github.com/spf13/cobra" + "go.uber.org/zap" + "golang.org/x/net/context" + "os" + "strconv" + "strings" + "time" +) + +var ( + resourceID string +) + +// getDescriberCmd represents the describer command +var getDescriberCmd = &cobra.Command{ + Use: "getDescriber", + Short: "A brief description of your command", + RunE: func(cmd *cobra.Command, args []string) error { + // Open the output file + file, err := os.Create(outputFile) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer file.Close() // Ensure the file is closed at the end + + job := describe.DescribeJob{ + JobID: uint(uuid.New().ID()), + ResourceType: resourceType, + IntegrationID: "", + ProviderID: "", + DescribedAt: time.Now().UnixMilli(), + IntegrationType: configs.IntegrationTypeLower, + CipherText: "", + IntegrationLabels: nil, + IntegrationAnnotations: nil, + } + + ctx := context.Background() + logger, _ := zap.NewProduction() + + // TODO: Set the credentials + creds := configs.IntegrationCredentials{} + + additionalParameters, err := provider.GetAdditionalParameters(job) + if err != nil { + return err + } + plg := steampipe.Plugin() + + f := func(resource model.Resource) error { + if resource.Description == nil { + return nil + } + descriptionJSON, err := json.Marshal(resource.Description) + if err != nil { + return fmt.Errorf("failed to marshal description: %w", err) + } + descriptionJSON, err = trimJsonFromEmptyObjects(descriptionJSON) + if err != nil { + return fmt.Errorf("failed to trim json: %w", err) + } + + metadata, err := provider.GetResourceMetadata(job, resource) + if err != nil { + return fmt.Errorf("failed to get resource metadata") + } + err = provider.AdjustResource(job, &resource) + if err != nil { + return fmt.Errorf("failed to adjust resource metadata") + } + + desc := resource.Description + err = json.Unmarshal(descriptionJSON, &desc) + if err != nil { + return fmt.Errorf("unmarshal description: %v", err.Error()) + } + + if plg != nil { + _, _, err = steampipe.ExtractTagsAndNames(logger, plg, job.ResourceType, resource) + if err != nil { + logger.Error("failed to build tags for service", zap.Error(err), zap.String("resourceType", job.ResourceType), zap.Any("resource", resource)) + } + } + + var description any + err = json.Unmarshal([]byte(descriptionJSON), &description) + if err != nil { + logger.Error("failed to parse resource description json", zap.Error(err)) + return fmt.Errorf("failed to parse resource description json") + } + + res := es.Resource{ + PlatformID: fmt.Sprintf("%s:::%s:::%s", job.IntegrationID, job.ResourceType, resource.UniqueID()), + ResourceID: resource.UniqueID(), + ResourceName: resource.Name, + Description: description, + IntegrationType: configs.IntegrationName, + ResourceType: strings.ToLower(job.ResourceType), + IntegrationID: job.IntegrationID, + Metadata: metadata, + DescribedAt: job.DescribedAt, + DescribedBy: strconv.FormatUint(uint64(job.JobID), 10), + } + + // Write the resource JSON to the file + resJSON, err := json.Marshal(res) + if err != nil { + return fmt.Errorf("failed to marshal resource JSON: %w", err) + } + _, err = file.Write(resJSON) + if err != nil { + return fmt.Errorf("failed to write to file: %w", err) + } + _, err = file.Write([]byte(",\n")) // Add a newline for readability + if err != nil { + return fmt.Errorf("failed to write newline to file: %w", err) + } + + return nil + } + clientStream := (*model.StreamSender)(&f) + + err = describer.GetSingleResource( + ctx, + logger, + job.ResourceType, + job.TriggerType, + creds, + additionalParameters, + resourceID, + clientStream, + ) + if err != nil { + return err + } + return nil + }, +} + +func init() { + getDescriberCmd.Flags().StringVar(&resourceType, "resourceType", "", "Resource type") + getDescriberCmd.Flags().StringVar(&resourceID, "resourceID", "", "Resource ID") + getDescriberCmd.Flags().StringVar(&outputFile, "outputFile", "output.json", "File to write JSON outputs") +} diff --git a/command/cmd/root.go b/command/cmd/root.go new file mode 100644 index 00000000..5ca36e29 --- /dev/null +++ b/command/cmd/root.go @@ -0,0 +1,35 @@ +/* +Copyright © 2023 NAME HERE +*/ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "run", + Short: "OpenGovernance aws describer manual", + RunE: func(cmd *cobra.Command, args []string) error { + var items []string + items = append(items, "describer") + + return describerCmd.Help() + + }, +} + +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + rootCmd.AddCommand(describerCmd) + rootCmd.AddCommand(getDescriberCmd) +} diff --git a/command/main.go b/command/main.go new file mode 100644 index 00000000..771a4018 --- /dev/null +++ b/command/main.go @@ -0,0 +1,10 @@ +/* +Copyright © 2023 NAME HERE +*/ +package main + +import "github.com/opengovern/og-describer-template/command/cmd" + +func main() { + cmd.Execute() +} diff --git a/pkg/describer/resources.go b/pkg/describer/resources.go index 9e3ad611..8d0f4e33 100755 --- a/pkg/describer/resources.go +++ b/pkg/describer/resources.go @@ -69,3 +69,30 @@ func describe(ctx context.Context, logger *zap.Logger, accountCfg configs.Integr return resourceTypeObject.ListDescriber(ctx, accountCfg, triggerType, additionalParameters, stream) } + +func GetSingleResource( + ctx context.Context, + logger *zap.Logger, + resourceType string, + triggerType enums.DescribeTriggerType, + cfg configs.IntegrationCredentials, + additionalParameters map[string]string, + resourceId string, + stream *model.StreamSender, +) error { + _, err := describeSingle(ctx, logger, cfg, resourceType, resourceId, triggerType, additionalParameters, stream) + if err != nil { + return err + } + return nil +} + +func describeSingle(ctx context.Context, logger *zap.Logger, accountCfg configs.IntegrationCredentials, resourceType string, resourceID string, triggerType enums.DescribeTriggerType, additionalParameters map[string]string, stream *model.StreamSender) (*model.Resource, error) { + resourceTypeObject, ok := provider.ResourceTypes[resourceType] + if !ok { + return nil, fmt.Errorf("unsupported resource type: %s", resourceType) + } + ctx = describer.WithLogger(ctx, logger) + + return resourceTypeObject.GetDescriber(ctx, accountCfg, triggerType, resourceID, additionalParameters) +} diff --git a/pkg/describer/worker.go b/pkg/describer/worker.go index 05956434..c372d1c4 100755 --- a/pkg/describer/worker.go +++ b/pkg/describer/worker.go @@ -124,9 +124,13 @@ func doDescribe( return fmt.Errorf("unmarshal description: %v", err.Error()) } - tags, _, err := steampipe.ExtractTagsAndNames(logger, plg, job.ResourceType, resource) - if err != nil { - logger.Error("failed to build tags for service", zap.Error(err), zap.String("resourceType", job.ResourceType), zap.Any("resource", resource)) + tags := make(map[string]string) + + if plg != nil { + tags, _, err = steampipe.ExtractTagsAndNames(logger, plg, job.ResourceType, resource) + if err != nil { + logger.Error("failed to build tags for service", zap.Error(err), zap.String("resourceType", job.ResourceType), zap.Any("resource", resource)) + } } var description any diff --git a/pkg/sdk/models/resource_type.go b/pkg/sdk/models/resource_type.go index 6bd89d08..e334bb0c 100644 --- a/pkg/sdk/models/resource_type.go +++ b/pkg/sdk/models/resource_type.go @@ -9,7 +9,7 @@ import ( // any types are used to load your provider configuration. type ResourceDescriber func(context.Context, configs.IntegrationCredentials, enums.DescribeTriggerType, map[string]string, *StreamSender) ([]Resource, error) -type SingleResourceDescriber func(context.Context, configs.IntegrationCredentials, enums.DescribeTriggerType, map[string]string) (*Resource, error) +type SingleResourceDescriber func(context.Context, configs.IntegrationCredentials, enums.DescribeTriggerType, string, map[string]string) (*Resource, error) type ResourceType struct { IntegrationType integration.Type