diff --git a/go.mod b/go.mod index 711a4dbfd..bcddc870e 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/lib/pq v1.10.9 github.com/nats-io/nats.go v1.36.0 github.com/open-policy-agent/opa v0.69.0 - github.com/opengovern/og-util v1.1.1 + github.com/opengovern/og-util v1.1.2 github.com/opengovern/plugin-aws v0.7.3 github.com/opengovern/plugin-gcp v0.0.0-20241014134959-2c0f222fc07b github.com/opengovern/plugin-kubernetes-internal v0.18.12 diff --git a/go.sum b/go.sum index b995f24c1..c29237b34 100644 --- a/go.sum +++ b/go.sum @@ -987,10 +987,8 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.2.0 h1:qke7ZVCmJcKrJVY2iHJVC+0kql9uYdkusOPsQOOeBw4= github.com/opencontainers/runc v1.2.0/go.mod h1:/PXzF0h531HTMsYQnmxXkBD7YaGShm/2zcRB79dksUc= -github.com/opengovern/og-util v1.1.0 h1:bV2XKX8aIpJGC2CR4kh9CU/rJWPQed76gtVXM2zBDXM= -github.com/opengovern/og-util v1.1.0/go.mod h1:dyn8rhmxq59o1jnbgGfmcUvW7iB/eN6OxoTUUx6jEHA= -github.com/opengovern/og-util v1.1.1 h1:SXnPCNuhR9WxxupEa1hCtIOIh2Sy+MF80qVdpoxc58k= -github.com/opengovern/og-util v1.1.1/go.mod h1:dyn8rhmxq59o1jnbgGfmcUvW7iB/eN6OxoTUUx6jEHA= +github.com/opengovern/og-util v1.1.2 h1:o3KE8UBgcNZqHevZfiNC4Ly4Bo9r/T9IKUipinAyB80= +github.com/opengovern/og-util v1.1.2/go.mod h1:dyn8rhmxq59o1jnbgGfmcUvW7iB/eN6OxoTUUx6jEHA= github.com/opengovern/plugin-aws v0.7.3 h1:76hZOjulNlgn4uaq5lq1/pmGmgJqvX1ZQbgqcQn03gI= github.com/opengovern/plugin-aws v0.7.3/go.mod h1:zfTMswfCyXZ0gD6SDCsmKg55LseXzeFzOH4jXn2QJVo= github.com/opengovern/plugin-gcp v0.0.0-20241014134959-2c0f222fc07b h1:4xP98kDpOXUu6RcFJyZN63OeA2I26MLS+dEB9JWYQpY= diff --git a/pkg/analytics/command.go b/pkg/analytics/command.go index 47f569dba..12d41b145 100644 --- a/pkg/analytics/command.go +++ b/pkg/analytics/command.go @@ -158,7 +158,7 @@ func (w *Worker) Run(ctx context.Context) error { return err } - if err := steampipe.PopulateOpenGovernancePluginSteampipeConfig(w.config.ElasticSearch, w.config.Steampipe, w.config.PennywiseBaseURL); err != nil { + if err := steampipe.PopulateOpenGovernancePluginSteampipeConfig(w.config.ElasticSearch, w.config.Steampipe); err != nil { w.logger.Error("failed to populate steampipe config for opengovernance plugin", zap.Error(err)) return err } diff --git a/pkg/analytics/config/type.go b/pkg/analytics/config/type.go index 6d86029de..ee4fd402c 100644 --- a/pkg/analytics/config/type.go +++ b/pkg/analytics/config/type.go @@ -3,15 +3,14 @@ package config import "github.com/opengovern/og-util/pkg/config" type WorkerConfig struct { - NATS config.NATS - PostgreSQL config.Postgres - ElasticSearch config.ElasticSearch - Steampipe config.Postgres - Integration config.OpenGovernanceService - Scheduler config.OpenGovernanceService - Inventory config.OpenGovernanceService - EsSink config.OpenGovernanceService - PennywiseBaseURL string `yaml:"pennywise_base_url"` + NATS config.NATS + PostgreSQL config.Postgres + ElasticSearch config.ElasticSearch + Steampipe config.Postgres + Integration config.OpenGovernanceService + Scheduler config.OpenGovernanceService + Inventory config.OpenGovernanceService + EsSink config.OpenGovernanceService DoTelemetry bool `yaml:"do_telemetry"` TelemetryWorkspaceID string `yaml:"telemetry_workspace_id"` diff --git a/pkg/compliance/runner/service.go b/pkg/compliance/runner/service.go index 4369ad8e5..8a868843b 100644 --- a/pkg/compliance/runner/service.go +++ b/pkg/compliance/runner/service.go @@ -32,7 +32,6 @@ type Config struct { Metadata config.OpenGovernanceService EsSink config.OpenGovernanceService Steampipe config.Postgres - PennywiseBaseURL string `yaml:"pennywise_base_url"` PrometheusPushAddress string } @@ -68,7 +67,7 @@ func NewWorker( if err != nil { return nil, err } - if err := steampipe.PopulateOpenGovernancePluginSteampipeConfig(config.ElasticSearch, config.Steampipe, config.PennywiseBaseURL); err != nil { + if err := steampipe.PopulateOpenGovernancePluginSteampipeConfig(config.ElasticSearch, config.Steampipe); err != nil { return nil, err } diff --git a/pkg/inventory/query-runner/service.go b/pkg/inventory/query-runner/service.go index ca176428f..27862ce64 100644 --- a/pkg/inventory/query-runner/service.go +++ b/pkg/inventory/query-runner/service.go @@ -31,7 +31,6 @@ type Config struct { Metadata config.OpenGovernanceService EsSink config.OpenGovernanceService Steampipe config.Postgres - PennywiseBaseURL string `yaml:"pennywise_base_url"` PrometheusPushAddress string } @@ -64,7 +63,7 @@ func NewWorker( return nil, err } - if err := steampipe.PopulateOpenGovernancePluginSteampipeConfig(config.ElasticSearch, config.Steampipe, config.PennywiseBaseURL); err != nil { + if err := steampipe.PopulateOpenGovernancePluginSteampipeConfig(config.ElasticSearch, config.Steampipe); err != nil { return nil, err } diff --git a/pkg/steampipe-plugin-opengovernance/opengovernance-client/cost_estimate.go b/pkg/steampipe-plugin-opengovernance/opengovernance-client/cost_estimate.go index ea57492e4..7d45c8d8a 100644 --- a/pkg/steampipe-plugin-opengovernance/opengovernance-client/cost_estimate.go +++ b/pkg/steampipe-plugin-opengovernance/opengovernance-client/cost_estimate.go @@ -1,23 +1,5 @@ package opengovernance_client -import ( - "context" - "encoding/json" - "fmt" - "github.com/kaytu-io/pennywise/pkg/cost" - "github.com/kaytu-io/pennywise/pkg/schema" - "github.com/opengovern/og-util/pkg/httpclient" - essdk "github.com/opengovern/og-util/pkg/opengovernance-es-sdk" - steampipesdk "github.com/opengovern/og-util/pkg/steampipe" - "github.com/opengovern/opengovernance/pkg/steampipe-plugin-opengovernance/opengovernance-sdk/config" - integration_type "github.com/opengovern/opengovernance/services/integration/integration-type" - "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" - "github.com/turbot/steampipe-plugin-sdk/v5/plugin" - "net/http" - "runtime" - "time" -) - type ResourceCostEstimate struct { ResourceID string `json:"resource_id"` ResourceType string `json:"resource_type"` @@ -154,332 +136,3 @@ func ResourceTypeConversion(resourceType string) string { } return resourceType } - -func GetValues(resource Resource, resourceType string) (map[string]interface{}, error) { - //switch strings.ToLower(resourceType) { - //// AWS - //case "aws::elasticloadbalancing::loadbalancer": - // return getAwsLoadBalancerValues(resource) - //case "aws::elasticloadbalancingv2::loadbalancer": - // return getAwsLoadBalancer2Values(resource) - //case "aws::ec2::instance": - // return getAwsEc2InstanceValues(resource) - //case "aws::autoscaling::autoscalinggroup": - // return nil, nil - //case "aws::rds::dbinstance": - // return getAwsRdsDbInstanceValues(resource) - //case "aws::ec2::volume": - // return getAwsEbsVolumeValues(resource) - //case "aws::ec2::volumegp3": - // return getAwsEbsVolumeGp3Values(resource) - //case "aws::ec2::volumesnapshot": - // return getAwsEbsSnapshotValues(resource) - //case "aws::efs::filesystem": - // return getAwsEfsFileSystemValues(resource) - //case "aws::elasticache::cluster": - // return getAwsElastiCacheClusterValues(resource) - //case "aws::elasticache::replicationgroup": - // return getAwsElastiCacheReplicationGroupValues(resource) - //case "aws::ec2::eip": - // return getAwsEc2EipValues(resource) - //case "aws::eks::cluster": - // return getAwsEksClusterValues(resource) - //case "aws::eks::nodegroup": - // return getAwsEksNodeGroupValues(resource) - //case "aws::fsx::filesystem": - // return getAwsFSXFileSystemValues(resource) - //case "aws::ec2::natgateway": - // return getAwsNatGatewayValues(resource) - //case "aws::ec2::host": - // return getAwsEc2HostValues(resource) - //case "aws::lambda::function": - // return getAwsLambdaFunctionValues(resource) - //case "aws::elasticsearch::domain": - // return getAwsEsDomainValues(resource) - //case "aws::opensearch::domain": - // return getAwsOpenSearchDomainValues(resource) - //case "aws::dynamodb::table": - // return getAwsDynamoDbTableValues(resource) - // - //// Azure - //case "microsoft.compute/virtualmachines": - // return nil, nil - //case "microsoft.compute/disks": - // return getAzureComputeDiskValues(resource) - //case "microsoft.compute/images": - // return nil, nil - //case "microsoft.compute/snapshots": - // return getAzureComputeSnapshotValues(resource) - //case "microsoft.compute/virtualmachinescalesets": - // return nil, nil - //case "microsoft.network/loadbalancers": - // return getAzureLoadBalancerValues(resource) - //case "microsoft.network/loadbalancers/loadbalancingeules": - // return nil, nil - //case "microsoft.network/loadbalancers/outboundrules": - // return nil, nil - //case "microsoft.network/applicationgateways": - // return getAzureApplicationGatewayValues(resource) - //case "microsoft.network/natgateways": - // return nil, nil - //case "microsoft.network/publicipaddresses": - // return nil, nil - //case "microsoft.network/publicipprefixes": - // return nil, nil - //case "microsoft.containerregistry/registries": - // return nil, nil - //case "microsoft.network/privateendpoints": - // return nil, nil - //case "microsoft.storage/queues": - // return nil, nil - //case "microsoft.storage/fileshares": - // return nil, nil - //case "microsoft.storage/storageaccounts": - // return nil, nil - //case "microsoft.network/virtualnetworkgateways": - // return nil, nil - //case "microsoft.keyvault/vaults/keys": - // return nil, nil - //case "microsoft.keyvault/managedhsms": - // return nil, nil - //case "microsoft.cdn/profiles/endpoints": - // return nil, nil - //case "microsoft.network/dnszones": - // return nil, nil - //case "microsoft.network/privatednszones": - // return nil, nil - //case "microsoft.documentdb/sqldatabases": - // return nil, nil - //case "microsoft.documentdb/mongodatabases": - // return nil, nil - //case "microsoft.documentdb/mongocollection": - // return nil, nil - //case "microsoft.dbformariadb/servers": - // return nil, nil - //case "microsoft.sql/servers/databases": - // return nil, nil - //case "microsoft.sql/managedInstances": - // return nil, nil - //case "microsoft.dbformysql/servers": - // return nil, nil - //case "microsoft.dbforpostgresql/servers": - // return nil, nil - //case "microsoft.dbforpostgresql/flexibleservers": - // return nil, nil - //case "microsoft.dbformysql/flexibleservers": - // return nil, nil - //case "microsoft.containerservice/managedclusters": - // return nil, nil - //case "microsoft.web/hostingenvironments": - // return nil, nil - //case "microsoft.web/plan": - // return nil, nil - //case "microsoft.apimanagement/service": - // return nil, nil - //case "microsoft.web/sites": - // return nil, nil - //case "microsoft.search/searchservices": - // return nil, nil - //case "microsoft.automation/automationaccounts": - // return nil, nil - //} - return map[string]interface{}{}, nil -} - -type LookupQueryResponse struct { - Hits struct { - Hits []struct { - ID string `json:"_id"` - Score float64 `json:"_score"` - Index string `json:"_index"` - Type string `json:"_type"` - Version int64 `json:"_version,omitempty"` - Source LookupResource `json:"_source"` - Sort []any `json:"sort"` - } - } -} - -func FetchLookupByResourceIDType(client Client, ctx context.Context, d *plugin.QueryData) (*LookupQueryResponse, error) { - filters := essdk.BuildFilter(ctx, d.QueryContext, map[string]string{ - "resource_id": "resource_id", - "resource_type": "resource_type", - }, "", nil, nil, nil) - out, err := json.Marshal(filters) - if err != nil { - return nil, err - } - - var filterMap []map[string]any - err = json.Unmarshal(out, &filterMap) - if err != nil { - return nil, err - } - - request := make(map[string]any) - request["query"] = map[string]any{ - "bool": map[string]any{ - "filter": filterMap, - }, - } - - b, err := json.Marshal(request) - if err != nil { - return nil, err - } - - plugin.Logger(ctx).Error("ListResourceCostEstimate Query", "query=", string(b), "index=", InventorySummaryIndex) - - var response LookupQueryResponse - err = client.ES.Search(ctx, InventorySummaryIndex, string(b), &response) - if err != nil { - return nil, err - } - - return &response, nil -} - -func ListResourceCostEstimate(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (any, error) { - plugin.Logger(ctx).Warn("ListResourceCostEstimate", d) - runtime.GC() - // create service - cfg := config.GetConfig(d.Connection) - - plugin.Logger(ctx).Trace("ListResourceCostEstimate 2", cfg) - ke, err := config.NewClientCached(cfg, d.ConnectionCache, ctx) - if err != nil { - return nil, err - } - k := Client{ES: ke} - - plugin.Logger(ctx).Trace("ListResourceCostEstimate 3", k) - sc, err := steampipesdk.NewSelfClientCached(ctx, d.ConnectionCache) - if err != nil { - plugin.Logger(ctx).Error("ListResourceCostEstimate NewSelfClientCached", "error", err) - return nil, err - } - plugin.Logger(ctx).Trace("ListResourceCostEstimate 4", sc) - encodedResourceCollectionFilters, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyResourceCollectionFilters) - if err != nil { - plugin.Logger(ctx).Error("ListResourceCostEstimate GetConfigTableValueOrNil for resource_collection_filters", "error", err) - return nil, err - } - plugin.Logger(ctx).Trace("ListResourceCostEstimate 5", encodedResourceCollectionFilters) - clientType, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyClientType) - if err != nil { - plugin.Logger(ctx).Error("ListResourceCostEstimate GetConfigTableValueOrNil for client_type", "error", err) - return nil, err - } - - plugin.Logger(ctx).Trace("Columns", d.EqualsQuals) - var indexes []struct { - index string - resourceType string - } - for column, q := range d.EqualsQuals { - if column == "resource_type" { - if s, ok := q.GetValue().(*proto.QualValue_StringValue); ok && s != nil { - indexes = []struct { - index string - resourceType string - }{{index: ResourceTypeToESIndex(s.StringValue), resourceType: s.StringValue}} - } else if l := q.GetListValue(); l != nil { - for _, v := range l.GetValues() { - if v == nil { - continue - } - indexes = append(indexes, struct { - index string - resourceType string - }{index: v.GetStringValue(), resourceType: v.GetStringValue()}) - } - } - } - } - - req := schema.Submission{ - ID: "submittion-1", - CreatedAt: time.Now(), - Resources: []schema.ResourceDef{}, - } - - var resources []Resource - - for _, index := range indexes { - paginator, err := k.NewResourcePaginator(essdk.BuildFilterWithDefaultFieldName(ctx, d.QueryContext, resourceMapping, - "", nil, encodedResourceCollectionFilters, clientType, true), d.QueryContext.Limit, index.index) - if err != nil { - plugin.Logger(ctx).Error("ListResourceCostEstimate NewResourcePaginator", "error", err) - return nil, err - } - - for paginator.HasNext() { - page, err := paginator.NextPage(ctx) - if err != nil { - plugin.Logger(ctx).Error("ListResourceCostEstimate NextPage", "error", err) - return nil, err - } - plugin.Logger(ctx).Trace("ListResourceCostEstimate", "next page") - - for _, hit := range page { - resources = append(resources, hit) - - var provider schema.ProviderName - if hit.IntegrationType == integration_type.IntegrationTypeAWSAccount.String() { - provider = schema.AWSProvider - } else if hit.IntegrationType == integration_type.IntegrationTypeAzureSubscription.String() { - provider = schema.AzureProvider - } - values, err := GetValues(hit, index.resourceType) - if err != nil { - plugin.Logger(ctx).Error("GetValues ", "error", err) - return nil, err - } - req.Resources = append(req.Resources, schema.ResourceDef{ - Address: hit.ResourceID, - Type: ResourceTypeConversion(hit.ResourceType), - Name: hit.Metadata.Name, - RegionCode: hit.Metadata.Region, - ProviderName: provider, - Values: values, - }) - } - } - err = paginator.Close(ctx) - if err != nil { - return nil, err - } - } - - reqBody, err := json.Marshal(req) - if err != nil { - return nil, err - } - plugin.Logger(ctx).Warn("ListResourceCostEstimate: Pennywise") - - var response cost.State - statusCode, err := httpclient.DoRequest(ctx, "GET", *cfg.PennywiseBaseURL+"/api/v1/cost/submission", nil, reqBody, &response) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, fmt.Errorf("failed to get pennywise cost, status code = %d", statusCode) - } - - for _, hit := range resources { - resourceCost, err := response.Cost() - if err != nil { - return nil, err - } - - d.StreamListItem(ctx, ResourceCostEstimate{ - ResourceID: hit.ResourceID, - ResourceType: hit.ResourceType, - Cost: resourceCost.Decimal.InexactFloat64(), - }) - } - - plugin.Logger(ctx).Warn("ListResourceCostEstimate: Done", fmt.Sprintf("%v", response.Resources)) - return nil, nil -} diff --git a/pkg/steampipe-plugin-opengovernance/opengovernance-sdk/config/config.go b/pkg/steampipe-plugin-opengovernance/opengovernance-sdk/config/config.go index 6167a3849..9eaa3f65a 100644 --- a/pkg/steampipe-plugin-opengovernance/opengovernance-sdk/config/config.go +++ b/pkg/steampipe-plugin-opengovernance/opengovernance-sdk/config/config.go @@ -22,7 +22,6 @@ type ClientConfig struct { PgDatabase *string `cty:"pg_database"` PgSslMode *string `cty:"pg_ssl_mode"` - PennywiseBaseURL *string `cty:"pennywise_baseurl"` ComplianceServiceBaseURL *string `cty:"compliance_service_baseurl"` } @@ -64,9 +63,6 @@ func Schema() map[string]*schema.Attribute { "pg_ssl_mode": { Type: schema.TypeString, }, - "pennywise_baseurl": { - Type: schema.TypeString, - }, "compliance_service_baseurl": { Type: schema.TypeString, Required: false, diff --git a/pkg/steampipe-plugin-opengovernance/opengovernance/plugin.go b/pkg/steampipe-plugin-opengovernance/opengovernance/plugin.go index 5c1f879fb..a9dbe546c 100644 --- a/pkg/steampipe-plugin-opengovernance/opengovernance/plugin.go +++ b/pkg/steampipe-plugin-opengovernance/opengovernance/plugin.go @@ -20,7 +20,6 @@ func Plugin(ctx context.Context) *plugin.Plugin { "platform_resources": tablePlatformResources(ctx), "platform_lookup": tablePlatformLookup(ctx), "platform_cost": tablePlatformCost(ctx), - "pennywise_cost_estimate": tablePlatformCostEstimate(ctx), "platform_integrations": tablePlatformConnections(ctx), "platform_metrics": tablePlatformMetrics(ctx), "platform_api_benchmark_summary": tablePlatformApiBenchmarkSummary(ctx), diff --git a/pkg/steampipe-plugin-opengovernance/opengovernance/table_opengovernance_cost_estimate.go b/pkg/steampipe-plugin-opengovernance/opengovernance/table_opengovernance_cost_estimate.go deleted file mode 100644 index e0df0dec4..000000000 --- a/pkg/steampipe-plugin-opengovernance/opengovernance/table_opengovernance_cost_estimate.go +++ /dev/null @@ -1,38 +0,0 @@ -package opengovernance - -import ( - "context" - og_client "github.com/opengovern/opengovernance/pkg/steampipe-plugin-opengovernance/opengovernance-client" - "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" - "github.com/turbot/steampipe-plugin-sdk/v5/plugin" -) - -func tablePlatformCostEstimate(_ context.Context) *plugin.Table { - return &plugin.Table{ - Name: "pennywise_cost_estimate", - Description: "Pennywise Resource Cost Estimate", - Cache: &plugin.TableCacheOptions{ - Enabled: false, - }, - List: &plugin.ListConfig{ - Hydrate: og_client.ListResourceCostEstimate, - KeyColumns: []*plugin.KeyColumn{ - { - Name: "resource_id", - Operators: []string{"="}, - Require: "required", - }, - { - Name: "resource_type", - Operators: []string{"="}, - Require: "required", - }, - }, - }, - Columns: []*plugin.Column{ - {Name: "resource_id", Type: proto.ColumnType_STRING}, - {Name: "resource_type", Type: proto.ColumnType_STRING}, - {Name: "cost", Type: proto.ColumnType_DOUBLE}, - }, - } -}