From 4b7ba4aa49a8a9f1852e2c1d98aa356dc4308c89 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 9 Oct 2024 15:41:30 +0530 Subject: [PATCH] chore: change ci.type to non pointer & make external id lookup a string --- api/cache.go | 17 ++-- api/v1/types.go | 14 ++-- api/v1/zz_generated.deepcopy.go | 18 ++-- ...configs.flanksource.com_scrapeconfigs.yaml | 3 + config/schemas/scrape_config.schema.json | 3 + db/config.go | 2 +- db/models/config_change.go | 2 +- db/models/config_item.go | 12 +-- db/update.go | 36 ++++---- fixtures/kubernetes.yaml | 7 ++ scrapers/aws/aws.go | 82 +++++++++---------- scrapers/azure/azure.go | 8 +- scrapers/kubernetes/kubernetes.go | 10 +-- scrapers/runscrapers_test.go | 4 +- scrapers/terraform/terraform.go | 4 +- 15 files changed, 114 insertions(+), 108 deletions(-) diff --git a/api/cache.go b/api/cache.go index c10fb9ce..f3d370df 100644 --- a/api/cache.go +++ b/api/cache.go @@ -82,21 +82,16 @@ func (t *TempCache) Insert(item models.ConfigItem) { } scraperID := lo.FromPtr(item.ScraperID).String() - for _, id := range item.ExternalID { - if item.Type != nil { - t.aliases[strings.ToLower(*item.Type)+strings.ToLower(id)+scraperID] = item.ID - } else { - t.aliases[strings.ToLower(id)+scraperID] = item.ID - } + for _, extID := range item.ExternalID { + key := v1.ExternalID{ConfigType: item.Type, ExternalID: extID, ScraperID: scraperID}.Key() + t.aliases[key] = item.ID + + // Remove from nonFound cache + delete(t.notFound, key) } t.items[strings.ToLower(item.ID)] = item delete(t.notFound, strings.ToLower(item.ID)) - delete(t.notFound, v1.ExternalID{ - ConfigType: lo.FromPtr(item.Type), - ExternalID: []string(item.ExternalID), - ScraperID: lo.FromPtr(item.ScraperID).String(), - }.Key()) } func (t *TempCache) Get(ctx ScrapeContext, id string) (*models.ConfigItem, error) { diff --git a/api/v1/types.go b/api/v1/types.go index 2d338e28..b74c7a20 100644 --- a/api/v1/types.go +++ b/api/v1/types.go @@ -91,7 +91,7 @@ func (c ScraperSpec) IsDebug() bool { type ExternalID struct { ConfigType string - ExternalID []string + ExternalID string // Scraper id of the config // If left empty, the scraper id is the requester's scraper id. @@ -106,8 +106,8 @@ func (e ExternalID) GetKubernetesUID() string { strings.HasPrefix(configType, "argo::") || strings.HasPrefix(configType, "flux::") { - if uuid.Validate(e.ExternalID[0]) == nil { - return e.ExternalID[0] + if uuid.Validate(e.ExternalID) == nil { + return e.ExternalID } } @@ -115,7 +115,7 @@ func (e ExternalID) GetKubernetesUID() string { } func (e ExternalID) Find(db *gorm.DB) *gorm.DB { - query := db.Limit(1).Order("updated_at DESC").Where("deleted_at IS NULL").Where("external_id @> ?", pq.StringArray(e.ExternalID)) + query := db.Limit(1).Order("updated_at DESC").Where("deleted_at IS NULL").Where("external_id @> ?", pq.StringArray([]string{e.ExternalID})) if e.ConfigType != "" { query = query.Where("type = ?", e.ConfigType) } @@ -126,14 +126,14 @@ func (e ExternalID) Find(db *gorm.DB) *gorm.DB { } func (e ExternalID) Key() string { - return fmt.Sprintf("%s%s%s", strings.ToLower(e.ConfigType), strings.ToLower(strings.Join(e.ExternalID, ",")), e.ScraperID) + return strings.ToLower(fmt.Sprintf("%s%s%s", e.ConfigType, e.ExternalID, e.ScraperID)) } func (e ExternalID) String() string { if e.ScraperID != "" { - return fmt.Sprintf("scraper_id=%s type=%s externalids=%s", e.ScraperID, e.ConfigType, strings.Join(e.ExternalID, ",")) + return fmt.Sprintf("scraper_id=%s type=%s externalids=%s", e.ScraperID, e.ConfigType, e.ExternalID) } - return fmt.Sprintf("type=%s externalids=%s", e.ConfigType, strings.Join(e.ExternalID, ",")) + return fmt.Sprintf("type=%s externalids=%s", e.ConfigType, e.ExternalID) } func (e ExternalID) IsEmpty() bool { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index a4d2d34a..4fc097f8 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -407,6 +407,11 @@ func (in *ConfigTypeScrapeSummary) DeepCopyInto(out *ConfigTypeScrapeSummary) { *out = new(ChangeSummary) (*in).DeepCopyInto(*out) } + if in.Warnings != nil { + in, out := &in.Warnings, &out.Warnings + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigTypeScrapeSummary. @@ -494,11 +499,6 @@ func (in *DirectedRelationship) DeepCopy() *DirectedRelationship { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExternalID) DeepCopyInto(out *ExternalID) { *out = *in - if in.ExternalID != nil { - in, out := &in.ExternalID, &out.ExternalID - *out = make([]string, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalID. @@ -1108,8 +1108,8 @@ func (in *RelationshipConfig) DeepCopy() *RelationshipConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RelationshipResult) DeepCopyInto(out *RelationshipResult) { *out = *in - in.ConfigExternalID.DeepCopyInto(&out.ConfigExternalID) - in.RelatedExternalID.DeepCopyInto(&out.RelatedExternalID) + out.ConfigExternalID = in.ConfigExternalID + out.RelatedExternalID = in.RelatedExternalID } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RelationshipResult. @@ -1127,9 +1127,7 @@ func (in RelationshipResults) DeepCopyInto(out *RelationshipResults) { { in := &in *out = make(RelationshipResults, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + copy(*out, *in) } } diff --git a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml index bb2242fb..564d6fe5 100644 --- a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml +++ b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml @@ -5669,6 +5669,9 @@ spec: type: object endpoint: type: string + skipTLSVerify: + description: Skip TLS verify + type: boolean type: object local: type: string diff --git a/config/schemas/scrape_config.schema.json b/config/schemas/scrape_config.schema.json index f8d3c03f..32a4479b 100644 --- a/config/schemas/scrape_config.schema.json +++ b/config/schemas/scrape_config.schema.json @@ -766,6 +766,9 @@ "credentials": { "$ref": "#/$defs/EnvVar" }, + "skipTLSVerify": { + "type": "boolean" + }, "bucket": { "type": "string" } diff --git a/db/config.go b/db/config.go index c4ac9012..fca35854 100644 --- a/db/config.go +++ b/db/config.go @@ -134,7 +134,7 @@ func NewConfigItemFromResult(ctx api.ScrapeContext, result v1.ScrapeResult) (*mo ExternalID: append([]string{result.ID}, result.Aliases...), ID: utils.Deref(result.ConfigID), ConfigClass: result.ConfigClass, - Type: &result.Type, + Type: result.Type, Name: &result.Name, Source: &result.Source, Labels: &result.Labels, diff --git a/db/models/config_change.go b/db/models/config_change.go index ebcca914..9b937832 100644 --- a/db/models/config_change.go +++ b/db/models/config_change.go @@ -40,7 +40,7 @@ type ConfigChange struct { func (c ConfigChange) GetExternalID() v1.ExternalID { return v1.ExternalID{ - ExternalID: []string{c.ExternalID}, + ExternalID: c.ExternalID, ConfigType: c.ConfigType, } } diff --git a/db/models/config_item.go b/db/models/config_item.go index 4c5a1e12..2e776e2a 100644 --- a/db/models/config_item.go +++ b/db/models/config_item.go @@ -20,7 +20,7 @@ type ConfigItem struct { ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"` ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" ` ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" ` - Type *string `gorm:"column:type;default:null" json:"type,omitempty" ` + Type string `gorm:"column:type" json:"type,omitempty" ` Status *string `gorm:"column:status;default:null" json:"status,omitempty" ` Ready bool `json:"ready,omitempty" ` Health *models.Health `json:"health,omitempty"` @@ -50,21 +50,21 @@ type ConfigItem struct { func (ci ConfigItem) Label() string { if len(ci.ExternalID) == 0 { - return fmt.Sprintf("%s/%s id=%s ", lo.FromPtr(ci.Type), lo.FromPtr(ci.Name), ci.ID) + return fmt.Sprintf("%s/%s id=%s ", ci.Type, lo.FromPtr(ci.Name), ci.ID) } if ci.ID == ci.ExternalID[0] { - return fmt.Sprintf("%s/%s id=%s", lo.FromPtr(ci.Type), lo.FromPtr(ci.Name), ci.ID) + return fmt.Sprintf("%s/%s id=%s", ci.Type, lo.FromPtr(ci.Name), ci.ID) } - return fmt.Sprintf("%s/%s id=%s external=%s", lo.FromPtr(ci.Type), lo.FromPtr(ci.Name), ci.ID, ci.ExternalID[0]) + return fmt.Sprintf("%s/%s id=%s external=%s", ci.Type, lo.FromPtr(ci.Name), ci.ID, ci.ExternalID[0]) } func (ci ConfigItem) String() string { if len(ci.ExternalID) == 0 { - return fmt.Sprintf("id=%s type=%s name=%s ", ci.ID, lo.FromPtr(ci.Type), lo.FromPtr(ci.Name)) + return fmt.Sprintf("id=%s type=%s name=%s ", ci.ID, ci.Type, lo.FromPtr(ci.Name)) } - return fmt.Sprintf("id=%s type=%s name=%s external_id=%s", ci.ID, lo.FromPtr(ci.Type), lo.FromPtr(ci.Name), ci.ExternalID[0]) + return fmt.Sprintf("id=%s type=%s name=%s external_id=%s", ci.ID, ci.Type, lo.FromPtr(ci.Name), ci.ExternalID[0]) } func (ci ConfigItem) ConfigJSONStringMap() (map[string]interface{}, error) { diff --git a/db/update.go b/db/update.go index 547c01cd..40e26daa 100644 --- a/db/update.go +++ b/db/update.go @@ -127,9 +127,9 @@ func updateCI(ctx api.ScrapeContext, summary *v1.ScrapeSummary, result v1.Scrape ctx.Errorf("[%s] failed to check for changes: %v", ci, err) } else if changeResult != nil { if ctx.Logger.V(5).Enabled() { - ctx.Logger.V(5).Infof("[%s/%s] detected changes %v", *ci.Type, ci.ExternalID[0], lo.FromPtr(changeResult.Diff)) + ctx.Logger.V(5).Infof("[%s/%s] detected changes %v", ci.Type, ci.ExternalID[0], lo.FromPtr(changeResult.Diff)) } else { - ctx.Logger.V(3).Infof("[%s/%s] detected changes", *ci.Type, ci.ExternalID[0]) + ctx.Logger.V(3).Infof("[%s/%s] detected changes", ci.Type, ci.ExternalID[0]) } result.Changes = []v1.ChangeResult{*changeResult} @@ -195,7 +195,7 @@ func updateCI(ctx api.ScrapeContext, summary *v1.ScrapeSummary, result v1.Scrape if !stringEqual(ci.Source, existing.Source) { updates["source"] = ci.Source } - if !stringEqual(ci.Type, existing.Type) { + if ci.Type != existing.Type { updates["type"] = ci.Type } if !stringEqual(ci.Status, existing.Status) { @@ -247,7 +247,7 @@ func updateCI(ctx api.ScrapeContext, summary *v1.ScrapeSummary, result v1.Scrape // same config items if lo.FromPtr(existing.ScraperID) != lo.FromPtr(ci.ScraperID) { updates["scraper_id"] = ci.ScraperID - summary.AddWarning(*ci.Type, fmt.Sprintf("updated scraper_id of config[%s] from %s to %s", ci, existing.ScraperID, ci.ScraperID)) + summary.AddWarning(ci.Type, fmt.Sprintf("updated scraper_id of config[%s] from %s to %s", ci, existing.ScraperID, ci.ScraperID)) } if ci.Properties != nil && len(*ci.Properties) > 0 && (existing.Properties == nil || !mapEqual(ci.Properties.AsMap(), existing.Properties.AsMap())) { @@ -399,7 +399,7 @@ var orphanCache = cache.New(60*time.Minute, 10*time.Minute) func upsertAnalysis(ctx api.ScrapeContext, result *v1.ScrapeResult) error { analysis := result.AnalysisResult.ToConfigAnalysis() - ciID, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: analysis.ConfigType, ExternalID: []string{analysis.ExternalID}}) + ciID, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: analysis.ConfigType, ExternalID: analysis.ExternalID}) if err != nil { return err } else if ciID == nil { @@ -443,7 +443,7 @@ func SaveResults(ctx api.ScrapeContext, results []v1.ScrapeResult) (v1.ScrapeSum func syncCRDChanges(ctx api.ScrapeContext, configs []*models.ConfigItem) error { for _, config := range configs { - if !strings.HasPrefix(*config.Type, api.MissionControlConfigTypePrefix) { + if !strings.HasPrefix(config.Type, api.MissionControlConfigTypePrefix) { continue } @@ -468,7 +468,7 @@ func syncCRDChanges(ctx api.ScrapeContext, configs []*models.ConfigItem) error { return err } - switch strings.TrimPrefix(*config.Type, api.MissionControlConfigTypePrefix) { + switch strings.TrimPrefix(config.Type, api.MissionControlConfigTypePrefix) { case "ScrapeConfig": scrapeConfig := dutyModels.ConfigScraper{ Name: fmt.Sprintf("%s/%s", namespace, *config.Name), @@ -576,7 +576,7 @@ func saveResults(ctx api.ScrapeContext, results []v1.ScrapeResult) (v1.ScrapeSum return summary, fmt.Errorf("failed to create config items: %w", dutydb.ErrorDetails(err)) } for _, config := range newConfigs { - summary.AddInserted(*config.Type) + summary.AddInserted(config.Type) } // nonUpdatedConfigs are existing configs that were not updated in this scrape. @@ -591,9 +591,9 @@ func saveResults(ctx api.ScrapeContext, results []v1.ScrapeResult) (v1.ScrapeSum } if updated { - summary.AddUpdated(*updateArg.Existing.Type) + summary.AddUpdated(updateArg.Existing.Type) } else { - summary.AddUnchanged(*updateArg.Existing.Type) + summary.AddUnchanged(updateArg.Existing.Type) nonUpdatedConfigs = append(nonUpdatedConfigs, updateArg.Existing.ID) } @@ -790,7 +790,7 @@ func generateConfigChange(ctx api.ScrapeContext, newConf, prev models.ConfigItem } return &v1.ChangeResult{ - ConfigType: lo.FromPtr(newConf.Type), + ConfigType: newConf.Type, ChangeType: "diff", ExternalID: newConf.ExternalID[0], Diff: &diff, @@ -811,7 +811,7 @@ func relationshipSelectorToResults(ctx dutyContext.Context, inputs []v1.ScrapeRe for _, id := range linkedConfigIDs { rel := v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{input.ID}, ConfigType: input.Type}, + ConfigExternalID: v1.ExternalID{ExternalID: input.ID, ConfigType: input.Type}, RelatedConfigID: id.String(), } @@ -954,7 +954,7 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime return nil, nil, nil, nil, allChangeSummary, fmt.Errorf("config item %s has no external id", ci) } - parentExternalKey := configExternalKey{externalID: ci.ExternalID[0], parentType: lo.FromPtr(ci.Type)} + parentExternalKey := configExternalKey{externalID: ci.ExternalID[0], parentType: ci.Type} parentTypeToConfigMap[parentExternalKey] = ci.ID existing := &models.ConfigItem{} @@ -963,7 +963,7 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime return nil, nil, nil, nil, allChangeSummary, fmt.Errorf("unable to lookup existing config(%s): %w", ci, err) } } else { - if existing, err = ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: *ci.Type, ExternalID: []string{ci.ExternalID[0]}}); err != nil { + if existing, err = ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: ci.Type, ExternalID: ci.ExternalID[0]}); err != nil { return nil, nil, nil, nil, allChangeSummary, fmt.Errorf("unable to lookup external id(%s): %w", ci, err) } } @@ -993,8 +993,8 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime } else { if !changeSummary.IsEmpty() { var configType string - if ci != nil && ci.Type != nil { - configType = lo.FromPtr(ci.Type) + if ci != nil { + configType = ci.Type } else if len(result.Changes) > 0 { configType = result.Changes[0].ConfigType } @@ -1067,7 +1067,7 @@ func setConfigProbableParents(ctx api.ScrapeContext, parentTypeToConfigMap map[c continue } - if foundParent, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: parent.Type, ExternalID: []string{parent.ExternalID}}); err != nil { + if foundParent, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: parent.Type, ExternalID: parent.ExternalID}); err != nil { return err } else if foundParent != nil { // Ignore self parent reference @@ -1095,7 +1095,7 @@ func setConfigChildren(ctx api.ScrapeContext, allConfigs models.ConfigItems) err continue } - found, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: child.Type, ExternalID: []string{child.ExternalID}}) + found, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: child.Type, ExternalID: child.ExternalID}) if err != nil { return err } diff --git a/fixtures/kubernetes.yaml b/fixtures/kubernetes.yaml index 4535dec7..a49af8dd 100644 --- a/fixtures/kubernetes.yaml +++ b/fixtures/kubernetes.yaml @@ -44,6 +44,9 @@ spec: - selector: 'config_type == "Kubernetes::Certificate"' jsonpath: .spec.commonName value: md5sum + - selector: config_class == 'Connection' + jsonpath: "$..['password','bearer','clientSecret','personalAccessToken','certificate','secretKey','token'].value" + value: '******' exclude: - types: - Kubernetes::* @@ -59,6 +62,10 @@ spec: patch.status.containerStatuses.size() > 0 && has(patch.status.containerStatuses[0].restartCount) type: PodCrashLooping + - filter: > + change.change_type == 'diff' && + jq('.status.conditions[] | select(.type == "Healthy").message', patch).contains('Health check passed') + type: HealthCheckPassed exclude: - 'config_type == "Kubernetes::Endpoints" && details.message == "metadata.annotations.endpoints.kubernetes.io/last-change-trigger-time"' - 'config_type == "Kubernetes::Node" && has(details.message) && details.message == "status.images"' diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index 38ea6441..afe5799a 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -362,8 +362,8 @@ func (aws Scraper) ecsServices(ctx *AWSContext, config v1.AWS, client *ecs.Clien var relationships []v1.RelationshipResult // ECS Task Definition to ECS Service relationship relationships = append(relationships, v1.RelationshipResult{ - RelatedExternalID: v1.ExternalID{ExternalID: []string{*service.ServiceArn}, ConfigType: v1.AWSECSService}, - ConfigExternalID: v1.ExternalID{ExternalID: []string{*service.TaskDefinition}, ConfigType: v1.AWSECSTaskDefinition}, + RelatedExternalID: v1.ExternalID{ExternalID: *service.ServiceArn, ConfigType: v1.AWSECSService}, + ConfigExternalID: v1.ExternalID{ExternalID: *service.TaskDefinition, ConfigType: v1.AWSECSTaskDefinition}, Relationship: "ECSTaskDefinitionECSService", }) @@ -475,8 +475,8 @@ func (aws Scraper) ecsTasks(ctx *AWSContext, config v1.AWS, client *ecs.Client, Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSECSTask, taskID, map[string]string{"cluster": clusterName})}, RelationshipResults: []v1.RelationshipResult{ { - ConfigExternalID: v1.ExternalID{ExternalID: []string{*task.TaskDefinitionArn}, ConfigType: v1.AWSECSTaskDefinition}, - RelatedExternalID: v1.ExternalID{ExternalID: []string{*task.TaskArn}, ConfigType: v1.AWSECSTask}, + ConfigExternalID: v1.ExternalID{ExternalID: *task.TaskDefinitionArn, ConfigType: v1.AWSECSTaskDefinition}, + RelatedExternalID: v1.ExternalID{ExternalID: *task.TaskArn, ConfigType: v1.AWSECSTask}, Relationship: "ECSTaskDefinitionECSTask", }, }, @@ -675,12 +675,12 @@ func (aws Scraper) eksClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap } var relationships []v1.RelationshipResult - selfExternalID := v1.ExternalID{ExternalID: []string{lo.FromPtr(cluster.Cluster.Name)}, ConfigType: v1.AWSEKSCluster} + selfExternalID := v1.ExternalID{ExternalID: lo.FromPtr(cluster.Cluster.Name), ConfigType: v1.AWSEKSCluster} // EKS to instance roles relationship relationships = append(relationships, v1.RelationshipResult{ RelatedExternalID: selfExternalID, - ConfigExternalID: v1.ExternalID{ExternalID: []string{lo.FromPtr(cluster.Cluster.Arn)}, ConfigType: v1.AWSIAMRole}, + ConfigExternalID: v1.ExternalID{ExternalID: lo.FromPtr(cluster.Cluster.Arn), ConfigType: v1.AWSIAMRole}, Relationship: "EKSIAMRole", }) @@ -688,7 +688,7 @@ func (aws Scraper) eksClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap for _, subnetID := range cluster.Cluster.ResourcesVpcConfig.SubnetIds { relationships = append(relationships, v1.RelationshipResult{ RelatedExternalID: selfExternalID, - ConfigExternalID: v1.ExternalID{ExternalID: []string{subnetID}, ConfigType: v1.AWSEC2Subnet}, + ConfigExternalID: v1.ExternalID{ExternalID: subnetID, ConfigType: v1.AWSEC2Subnet}, Relationship: "SubnetEKS", }) } @@ -696,7 +696,7 @@ func (aws Scraper) eksClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap // EKS to security groups relationship relationships = append(relationships, v1.RelationshipResult{ RelatedExternalID: selfExternalID, - ConfigExternalID: v1.ExternalID{ExternalID: []string{lo.FromPtr(cluster.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId)}, ConfigType: v1.AWSEC2SecurityGroup}, + ConfigExternalID: v1.ExternalID{ExternalID: lo.FromPtr(cluster.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId), ConfigType: v1.AWSEC2SecurityGroup}, Relationship: "EKSSecuritygroups", }) @@ -784,8 +784,8 @@ func (aws Scraper) availabilityZones(ctx *AWSContext, config v1.AWS, results *v1 Name: lo.FromPtr(az.ZoneName), Parents: []v1.ConfigExternalKey{{Type: v1.AWSRegion, ExternalID: lo.FromPtr(az.RegionName)}}, RelationshipResults: []v1.RelationshipResult{{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{lo.FromPtr(az.ZoneId)}, ConfigType: v1.AWSAvailabilityZoneID, ScraperID: "all"}, - RelatedExternalID: v1.ExternalID{ExternalID: []string{lo.FromPtr(az.ZoneName)}, ConfigType: v1.AWSAvailabilityZone}, + ConfigExternalID: v1.ExternalID{ExternalID: lo.FromPtr(az.ZoneId), ConfigType: v1.AWSAvailabilityZoneID, ScraperID: "all"}, + RelatedExternalID: v1.ExternalID{ExternalID: lo.FromPtr(az.ZoneName), ConfigType: v1.AWSAvailabilityZone}, }}, }) @@ -880,8 +880,8 @@ func (aws Scraper) account(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRes if *region.OptInStatus != "not-opted-in" { result.RelationshipResults = []v1.RelationshipResult{ { - RelatedExternalID: v1.ExternalID{ConfigType: v1.AWSAccount, ExternalID: []string{lo.FromPtr(ctx.Caller.Account)}}, - ConfigExternalID: v1.ExternalID{ConfigType: v1.AWSRegion, ExternalID: []string{*region.RegionName}}, + RelatedExternalID: v1.ExternalID{ConfigType: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}, + ConfigExternalID: v1.ExternalID{ConfigType: v1.AWSRegion, ExternalID: *region.RegionName}, }, } } @@ -992,11 +992,11 @@ func (aws Scraper) rds(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults for _, sg := range instance.VpcSecurityGroups { relationships = append(relationships, v1.RelationshipResult{ ConfigExternalID: v1.ExternalID{ - ExternalID: []string{*instance.DBInstanceIdentifier}, + ExternalID: *instance.DBInstanceIdentifier, ConfigType: v1.AWSRDSInstance, }, RelatedExternalID: v1.ExternalID{ - ExternalID: []string{*sg.VpcSecurityGroupId}, + ExternalID: *sg.VpcSecurityGroupId, ConfigType: v1.AWSEC2SecurityGroup, }, Relationship: "RDSSecurityGroup", @@ -1041,11 +1041,11 @@ func (aws Scraper) vpcs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResult // DHCPOptions relationship relationships = append(relationships, v1.RelationshipResult{ ConfigExternalID: v1.ExternalID{ - ExternalID: []string{*vpc.VpcId}, + ExternalID: *vpc.VpcId, ConfigType: v1.AWSEC2VPC, }, RelatedExternalID: v1.ExternalID{ - ExternalID: []string{*vpc.DhcpOptionsId}, + ExternalID: *vpc.DhcpOptionsId, ConfigType: v1.AWSEC2DHCPOptions, }, Relationship: "VPCDHCPOptions", @@ -1053,8 +1053,8 @@ func (aws Scraper) vpcs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResult // VPC to region relationship relationships = append(relationships, v1.RelationshipResult{ - RelatedExternalID: v1.ExternalID{ExternalID: []string{string(*vpc.VpcId)}, ConfigType: v1.AWSEC2VPC}, - ConfigExternalID: v1.ExternalID{ExternalID: []string{ctx.Session.Region}, ConfigType: v1.AWSRegion}, + RelatedExternalID: v1.ExternalID{ExternalID: string(*vpc.VpcId), ConfigType: v1.AWSEC2VPC}, + ConfigExternalID: v1.ExternalID{ExternalID: ctx.Session.Region, ConfigType: v1.AWSRegion}, Relationship: "RegionVPC", }) @@ -1096,14 +1096,14 @@ func (aws Scraper) instances(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR for _, r := range describeOutput.Reservations { for _, i := range r.Instances { selfExternalID := v1.ExternalID{ - ExternalID: []string{*i.InstanceId}, + ExternalID: *i.InstanceId, ConfigType: v1.AWSEC2Instance, } // SecurityGroup relationships for _, sg := range i.SecurityGroups { relationships = append(relationships, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{*sg.GroupId}, ConfigType: v1.AWSEC2SecurityGroup}, + ConfigExternalID: v1.ExternalID{ExternalID: *sg.GroupId, ConfigType: v1.AWSEC2SecurityGroup}, RelatedExternalID: selfExternalID, Relationship: "SecurityGroupInstance", }) @@ -1113,7 +1113,7 @@ func (aws Scraper) instances(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR for _, tag := range i.Tags { if *tag.Key == "aws:eks:cluster-name" { relationships = append(relationships, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{*tag.Value}, ConfigType: v1.AWSEKSCluster}, + ConfigExternalID: v1.ExternalID{ExternalID: *tag.Value, ConfigType: v1.AWSEKSCluster}, RelatedExternalID: selfExternalID, Relationship: "ClusterInstance", }) @@ -1124,14 +1124,14 @@ func (aws Scraper) instances(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR for _, vol := range i.BlockDeviceMappings { relationships = append(relationships, v1.RelationshipResult{ ConfigExternalID: selfExternalID, - RelatedExternalID: v1.ExternalID{ExternalID: []string{*vol.Ebs.VolumeId}, ConfigType: v1.AWSEBSVolume}, + RelatedExternalID: v1.ExternalID{ExternalID: *vol.Ebs.VolumeId, ConfigType: v1.AWSEBSVolume}, Relationship: "EC2InstanceVolume", }) } if i.IamInstanceProfile != nil { relationships = append(relationships, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{*i.IamInstanceProfile.Id}, ConfigType: v1.AWSIAMInstanceProfile}, + ConfigExternalID: v1.ExternalID{ExternalID: *i.IamInstanceProfile.Id, ConfigType: v1.AWSIAMInstanceProfile}, RelatedExternalID: selfExternalID, Relationship: "IAMInstanceProfileEC2Instance", }) @@ -1146,24 +1146,24 @@ func (aws Scraper) instances(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR relationships = append(relationships, v1.RelationshipResult{ ConfigExternalID: selfExternalID, - RelatedExternalID: v1.ExternalID{ExternalID: []string{"Kubernetes/Node//" + *i.PrivateDnsName}, ConfigType: "Kubernetes::Node"}, + RelatedExternalID: v1.ExternalID{ExternalID: "Kubernetes/Node//" + *i.PrivateDnsName, ConfigType: "Kubernetes::Node"}, Relationship: "InstanceKuberenetesNode", }) relationships = append(relationships, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{lo.FromPtr(i.SubnetId)}, ConfigType: v1.AWSEC2Subnet}, + ConfigExternalID: v1.ExternalID{ExternalID: lo.FromPtr(i.SubnetId), ConfigType: v1.AWSEC2Subnet}, RelatedExternalID: selfExternalID, Relationship: "SubnetInstance", }) relationships = append(relationships, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{ctx.Session.Region}, ConfigType: v1.AWSRegion}, + ConfigExternalID: v1.ExternalID{ExternalID: ctx.Session.Region, ConfigType: v1.AWSRegion}, RelatedExternalID: selfExternalID, Relationship: "RegionInstance", }) relationships = append(relationships, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{ctx.Subnets[lo.FromPtr(i.SubnetId)].Zone}, ConfigType: v1.AWSAvailabilityZone}, + ConfigExternalID: v1.ExternalID{ExternalID: ctx.Subnets[lo.FromPtr(i.SubnetId)].Zone, ConfigType: v1.AWSAvailabilityZone}, RelatedExternalID: selfExternalID, Relationship: "ZoneInstance", }) @@ -1379,11 +1379,11 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr for _, instance := range lb.Instances { relationships = append(relationships, v1.RelationshipResult{ ConfigExternalID: v1.ExternalID{ - ExternalID: []string{*instance.InstanceId}, + ExternalID: *instance.InstanceId, ConfigType: v1.AWSEC2Instance, }, RelatedExternalID: v1.ExternalID{ - ExternalID: []string{*lb.LoadBalancerName}, + ExternalID: *lb.LoadBalancerName, ConfigType: v1.AWSLoadBalancer, }, Relationship: "LoadBalancerInstance", @@ -1403,11 +1403,11 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr clusterName := strings.ReplaceAll(*tag.Key, clusterPrefix, "") relationships = append(relationships, v1.RelationshipResult{ ConfigExternalID: v1.ExternalID{ - ExternalID: []string{*lb.LoadBalancerName}, + ExternalID: *lb.LoadBalancerName, ConfigType: v1.AWSLoadBalancer, }, RelatedExternalID: v1.ExternalID{ - ExternalID: []string{clusterName}, + ExternalID: clusterName, ConfigType: v1.AWSEKSCluster, }, Relationship: "EKSLoadBalancer", @@ -1465,11 +1465,11 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr clusterName := strings.ReplaceAll(*tag.Key, clusterPrefix, "") relationships = append(relationships, v1.RelationshipResult{ ConfigExternalID: v1.ExternalID{ - ExternalID: []string{*lb.LoadBalancerArn}, + ExternalID: *lb.LoadBalancerArn, ConfigType: v1.AWSLoadBalancerV2, }, RelatedExternalID: v1.ExternalID{ - ExternalID: []string{clusterName}, + ExternalID: clusterName, ConfigType: v1.AWSEKSCluster, }, Relationship: "EKSLoadBalancer", @@ -1533,26 +1533,26 @@ func (aws Scraper) subnets(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRes } var relationships []v1.RelationshipResult - selfExternalID := v1.ExternalID{ExternalID: []string{lo.FromPtr(subnet.SubnetId)}, ConfigType: v1.AWSEC2Subnet} + selfExternalID := v1.ExternalID{ExternalID: lo.FromPtr(subnet.SubnetId), ConfigType: v1.AWSEC2Subnet} // Subnet to Region relationship relationships = append(relationships, v1.RelationshipResult{ RelatedExternalID: selfExternalID, - ConfigExternalID: v1.ExternalID{ExternalID: []string{ctx.Session.Region}, ConfigType: v1.AWSRegion}, + ConfigExternalID: v1.ExternalID{ExternalID: ctx.Session.Region, ConfigType: v1.AWSRegion}, Relationship: "RegionSubnet", }) // Subnet to availability zone relationship relationships = append(relationships, v1.RelationshipResult{ RelatedExternalID: selfExternalID, - ConfigExternalID: v1.ExternalID{ExternalID: []string{lo.FromPtr(subnet.AvailabilityZone)}, ConfigType: v1.AWSAvailabilityZone}, + ConfigExternalID: v1.ExternalID{ExternalID: lo.FromPtr(subnet.AvailabilityZone), ConfigType: v1.AWSAvailabilityZone}, Relationship: "AvailabilityZoneSubnet", }) // Subnet to availability zone relationship relationships = append(relationships, v1.RelationshipResult{ RelatedExternalID: selfExternalID, - ConfigExternalID: v1.ExternalID{ExternalID: []string{lo.FromPtr(subnet.AvailabilityZoneId)}, ConfigType: v1.AWSAvailabilityZoneID}, + ConfigExternalID: v1.ExternalID{ExternalID: lo.FromPtr(subnet.AvailabilityZoneId), ConfigType: v1.AWSAvailabilityZoneID}, Relationship: "AvailabilityZoneIDSubnet", }) @@ -1635,8 +1635,8 @@ func (aws Scraper) iamProfiles(ctx *AWSContext, config v1.AWS, results *v1.Scrap var relationships []v1.RelationshipResult for _, role := range profile.Roles { relationships = append(relationships, v1.RelationshipResult{ - RelatedExternalID: v1.ExternalID{ExternalID: []string{lo.FromPtr(profile.InstanceProfileId)}, ConfigType: v1.AWSIAMInstanceProfile}, - ConfigExternalID: v1.ExternalID{ExternalID: []string{lo.FromPtr(role.Arn)}, ConfigType: v1.AWSIAMRole}, + RelatedExternalID: v1.ExternalID{ExternalID: lo.FromPtr(profile.InstanceProfileId), ConfigType: v1.AWSIAMInstanceProfile}, + ConfigExternalID: v1.ExternalID{ExternalID: lo.FromPtr(role.Arn), ConfigType: v1.AWSIAMRole}, Relationship: "IAMRoleInstanceProfile", }) } @@ -1794,8 +1794,8 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { // the default parent should be moved to soft relationship defaultParent := r.Parents[0] (*results)[i].RelationshipResults = append((*results)[i].RelationshipResults, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ConfigType: defaultParent.Type, ExternalID: []string{defaultParent.ExternalID}}, - RelatedExternalID: v1.ExternalID{ConfigType: r.Type, ExternalID: []string{r.ID}}, + ConfigExternalID: v1.ExternalID{ConfigType: defaultParent.Type, ExternalID: defaultParent.ExternalID}, + RelatedExternalID: v1.ExternalID{ConfigType: r.Type, ExternalID: r.ID}, }) } diff --git a/scrapers/azure/azure.go b/scrapers/azure/azure.go index 1afd2621..963aa71a 100644 --- a/scrapers/azure/azure.go +++ b/scrapers/azure/azure.go @@ -218,18 +218,18 @@ func (azure Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { if relateSubscription { results[i].RelationshipResults = append(results[i].RelationshipResults, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{"/subscriptions/" + azure.config.SubscriptionID}, ConfigType: ConfigTypePrefix + "SUBSCRIPTION"}, - RelatedExternalID: v1.ExternalID{ExternalID: []string{r.ID}, ConfigType: r.Type}, + ConfigExternalID: v1.ExternalID{ExternalID: "/subscriptions/" + azure.config.SubscriptionID, ConfigType: ConfigTypePrefix + "SUBSCRIPTION"}, + RelatedExternalID: v1.ExternalID{ExternalID: r.ID, ConfigType: r.Type}, Relationship: "Subscription" + strings.TrimPrefix(r.Type, ConfigTypePrefix), }) } if relateResourceGroup && extractResourceGroup(r.ID) != "" { results[i].RelationshipResults = append(results[i].RelationshipResults, v1.RelationshipResult{ - RelatedExternalID: v1.ExternalID{ExternalID: []string{r.ID}, ConfigType: r.Type}, + RelatedExternalID: v1.ExternalID{ExternalID: r.ID, ConfigType: r.Type}, ConfigExternalID: v1.ExternalID{ ConfigType: ConfigTypePrefix + "MICROSOFT.RESOURCES/RESOURCEGROUPS", - ExternalID: []string{fmt.Sprintf("/subscriptions/%s/resourcegroups/%s", azure.config.SubscriptionID, extractResourceGroup(r.ID))}, + ExternalID: fmt.Sprintf("/subscriptions/%s/resourcegroups/%s", azure.config.SubscriptionID, extractResourceGroup(r.ID)), }, Relationship: "Resourcegroup" + strings.TrimPrefix(r.Type, ConfigTypePrefix), }) diff --git a/scrapers/kubernetes/kubernetes.go b/scrapers/kubernetes/kubernetes.go index 42c14852..6c0b3bf6 100644 --- a/scrapers/kubernetes/kubernetes.go +++ b/scrapers/kubernetes/kubernetes.go @@ -273,7 +273,7 @@ func ExtractResults(ctx *KubernetesContext, objs []*unstructured.Unstructured) v if address.TargetRef != nil { if address.TargetRef.Kind != "Service" { relationships = append(relationships, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{alias("Service", obj.GetNamespace(), obj.GetName())}, ConfigType: ConfigTypePrefix + "Service"}, + ConfigExternalID: v1.ExternalID{ExternalID: alias("Service", obj.GetNamespace(), obj.GetName()), ConfigType: ConfigTypePrefix + "Service"}, RelatedConfigID: string(address.TargetRef.UID), Relationship: fmt.Sprintf("Service%s", address.TargetRef.Kind), }) @@ -295,7 +295,7 @@ func ExtractResults(ctx *KubernetesContext, objs []*unstructured.Unstructured) v Relationship: "NodePod", }.WithConfig( ctx.GetID("", "Node", nodeName), - v1.ExternalID{ExternalID: []string{nodeExternalID}, ConfigType: ConfigTypePrefix + "Node"}, + v1.ExternalID{ExternalID: nodeExternalID, ConfigType: ConfigTypePrefix + "Node"}, )) } } @@ -306,7 +306,7 @@ func ExtractResults(ctx *KubernetesContext, objs []*unstructured.Unstructured) v Relationship: "Namespace" + obj.GetKind(), }.WithConfig( ctx.GetID("", "Namespace", obj.GetNamespace()), - v1.ExternalID{ExternalID: []string{alias("Namespace", "", obj.GetNamespace())}, ConfigType: ConfigTypePrefix + "Namespace"}, + v1.ExternalID{ExternalID: alias("Namespace", "", obj.GetNamespace()), ConfigType: ConfigTypePrefix + "Namespace"}, )) } @@ -341,7 +341,7 @@ func ExtractResults(ctx *KubernetesContext, objs []*unstructured.Unstructured) v ConfigID: id.String(), }.WithRelated( ctx.GetID(obj.GetNamespace(), obj.GetKind(), obj.GetName()), - v1.ExternalID{ExternalID: []string{string(obj.GetUID())}, ConfigType: getConfigTypePrefix(obj.GetAPIVersion()) + obj.GetKind()}, + v1.ExternalID{ExternalID: string(obj.GetUID()), ConfigType: getConfigTypePrefix(obj.GetAPIVersion()) + obj.GetKind()}, ) relationships = append(relationships, rel) @@ -375,7 +375,7 @@ func ExtractResults(ctx *KubernetesContext, objs []*unstructured.Unstructured) v if strings.HasSuffix(hostname, "elb.amazonaws.com") { relationships = append(relationships, v1.RelationshipResult{ ConfigID: string(obj.GetUID()), - RelatedExternalID: v1.ExternalID{ExternalID: []string{hostname}, ConfigType: v1.AWSLoadBalancer}, + RelatedExternalID: v1.ExternalID{ExternalID: hostname, ConfigType: v1.AWSLoadBalancer}, }) } } diff --git a/scrapers/runscrapers_test.go b/scrapers/runscrapers_test.go index 0bb14ee5..874db51a 100644 --- a/scrapers/runscrapers_test.go +++ b/scrapers/runscrapers_test.go @@ -429,14 +429,14 @@ var _ = Describe("Scrapers test", Ordered, func() { dummyCI := models.ConfigItem{ ID: configItemID, ConfigClass: "Test", - Type: lo.ToPtr("Test"), + Type: "Test", ScraperID: &dummyScraper.ID, } configItemID2 := uuid.New().String() dummyCI2 := models.ConfigItem{ ID: configItemID2, ConfigClass: "Test", - Type: lo.ToPtr("Test"), + Type: "Test", ScraperID: &dummyScraper.ID, } err = ctx.DB().Create(&dummyCI).Error diff --git a/scrapers/terraform/terraform.go b/scrapers/terraform/terraform.go index 09356d09..a8761de3 100644 --- a/scrapers/terraform/terraform.go +++ b/scrapers/terraform/terraform.go @@ -146,8 +146,8 @@ func awsProvider(externalID string, resource Resource) []v1.RelationshipResult { } results = append(results, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ConfigType: ConfigType, ExternalID: []string{externalID}}, - RelatedExternalID: v1.ExternalID{ExternalID: []string{arn}, ScraperID: "all"}, + ConfigExternalID: v1.ExternalID{ConfigType: ConfigType, ExternalID: externalID}, + RelatedExternalID: v1.ExternalID{ExternalID: arn, ScraperID: "all"}, }) }