chore: change ci.type to non pointer & make external id lookup a string

api/cache.go

@@ -82,21 +82,16 @@ func (t *TempCache) Insert(item models.ConfigItem) {
 	}
 
 	scraperID := lo.FromPtr(item.ScraperID).String()
-	for _, id := range item.ExternalID {
-		if item.Type != nil {
-			t.aliases[strings.ToLower(*item.Type)+strings.ToLower(id)+scraperID] = item.ID
-		} else {
-			t.aliases[strings.ToLower(id)+scraperID] = item.ID
-		}
+	for _, extID := range item.ExternalID {
+		key := v1.ExternalID{ConfigType: item.Type, ExternalID: extID, ScraperID: scraperID}.Key()
+		t.aliases[key] = item.ID
+
+		// Remove from nonFound cache
+		delete(t.notFound, key)
 	}
 
 	t.items[strings.ToLower(item.ID)] = item
 	delete(t.notFound, strings.ToLower(item.ID))
-	delete(t.notFound, v1.ExternalID{
-		ConfigType: lo.FromPtr(item.Type),
-		ExternalID: []string(item.ExternalID),
-		ScraperID:  lo.FromPtr(item.ScraperID).String(),
-	}.Key())
 }
 
 func (t *TempCache) Get(ctx ScrapeContext, id string) (*models.ConfigItem, error) {

api/v1/types.go

@@ -91,7 +91,7 @@ func (c ScraperSpec) IsDebug() bool {
 
 type ExternalID struct {
 	ConfigType string
-	ExternalID []string
+	ExternalID string
 
 	// Scraper id of the config
 	// If left empty, the scraper id is the requester's scraper id.
@@ -106,16 +106,16 @@ func (e ExternalID) GetKubernetesUID() string {
 		strings.HasPrefix(configType, "argo::") ||
 		strings.HasPrefix(configType, "flux::") {
 
-		if uuid.Validate(e.ExternalID[0]) == nil {
-			return e.ExternalID[0]
+		if uuid.Validate(e.ExternalID) == nil {
+			return e.ExternalID
 		}
 	}
 
 	return ""
 }
 
 func (e ExternalID) Find(db *gorm.DB) *gorm.DB {
-	query := db.Limit(1).Order("updated_at DESC").Where("deleted_at IS NULL").Where("external_id @> ?", pq.StringArray(e.ExternalID))
+	query := db.Limit(1).Order("updated_at DESC").Where("deleted_at IS NULL").Where("external_id @> ?", pq.StringArray([]string{e.ExternalID}))
 	if e.ConfigType != "" {
 		query = query.Where("type = ?", e.ConfigType)
 	}
@@ -126,14 +126,14 @@ func (e ExternalID) Find(db *gorm.DB) *gorm.DB {
 }
 
 func (e ExternalID) Key() string {
-	return fmt.Sprintf("%s%s%s", strings.ToLower(e.ConfigType), strings.ToLower(strings.Join(e.ExternalID, ",")), e.ScraperID)
+	return strings.ToLower(fmt.Sprintf("%s%s%s", e.ConfigType, e.ExternalID, e.ScraperID))
 }
 
 func (e ExternalID) String() string {
 	if e.ScraperID != "" {
-		return fmt.Sprintf("scraper_id=%s type=%s externalids=%s", e.ScraperID, e.ConfigType, strings.Join(e.ExternalID, ","))
+		return fmt.Sprintf("scraper_id=%s type=%s externalids=%s", e.ScraperID, e.ConfigType, e.ExternalID)
 	}
-	return fmt.Sprintf("type=%s externalids=%s", e.ConfigType, strings.Join(e.ExternalID, ","))
+	return fmt.Sprintf("type=%s externalids=%s", e.ConfigType, e.ExternalID)
 }
 
 func (e ExternalID) IsEmpty() bool {

chart/crds/configs.flanksource.com_scrapeconfigs.yaml

@@ -5669,6 +5669,9 @@ spec:
                               type: object
                             endpoint:
                               type: string
+                            skipTLSVerify:
+                              description: Skip TLS verify
+                              type: boolean
                           type: object
                         local:
                           type: string

config/schemas/scrape_config.schema.json

@@ -766,6 +766,9 @@
         "credentials": {
           "$ref": "#/$defs/EnvVar"
         },
+        "skipTLSVerify": {
+          "type": "boolean"
+        },
         "bucket": {
           "type": "string"
         }

db/config.go

@@ -134,7 +134,7 @@ func NewConfigItemFromResult(ctx api.ScrapeContext, result v1.ScrapeResult) (*mo
 		ExternalID:      append([]string{result.ID}, result.Aliases...),
 		ID:              utils.Deref(result.ConfigID),
 		ConfigClass:     result.ConfigClass,
-		Type:            &result.Type,
+		Type:            result.Type,
 		Name:            &result.Name,
 		Source:          &result.Source,
 		Labels:          &result.Labels,

db/models/config_change.go

@@ -40,7 +40,7 @@ type ConfigChange struct {
 
 func (c ConfigChange) GetExternalID() v1.ExternalID {
 	return v1.ExternalID{
-		ExternalID: []string{c.ExternalID},
+		ExternalID: c.ExternalID,
 		ConfigType: c.ConfigType,
 	}
 }

db/models/config_item.go

@@ -20,7 +20,7 @@ type ConfigItem struct {
 	ScraperID       *uuid.UUID            `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"`
 	ConfigClass     string                `gorm:"column:config_class;default:''" json:"config_class"  `
 	ExternalID      pq.StringArray        `gorm:"column:external_id;type:[]text" json:"external_id,omitempty"  `
-	Type            *string               `gorm:"column:type;default:null" json:"type,omitempty"  `
+	Type            string                `gorm:"column:type" json:"type,omitempty"  `
 	Status          *string               `gorm:"column:status;default:null" json:"status,omitempty"  `
 	Ready           bool                  `json:"ready,omitempty"  `
 	Health          *models.Health        `json:"health,omitempty"`
@@ -50,21 +50,21 @@ type ConfigItem struct {
 
 func (ci ConfigItem) Label() string {
 	if len(ci.ExternalID) == 0 {
-		return fmt.Sprintf("%s/%s id=%s ", lo.FromPtr(ci.Type), lo.FromPtr(ci.Name), ci.ID)
+		return fmt.Sprintf("%s/%s id=%s ", ci.Type, lo.FromPtr(ci.Name), ci.ID)
 	}
 	if ci.ID == ci.ExternalID[0] {
-		return fmt.Sprintf("%s/%s id=%s", lo.FromPtr(ci.Type), lo.FromPtr(ci.Name), ci.ID)
+		return fmt.Sprintf("%s/%s id=%s", ci.Type, lo.FromPtr(ci.Name), ci.ID)
 	}
 
-	return fmt.Sprintf("%s/%s id=%s external=%s", lo.FromPtr(ci.Type), lo.FromPtr(ci.Name), ci.ID, ci.ExternalID[0])
+	return fmt.Sprintf("%s/%s id=%s external=%s", ci.Type, lo.FromPtr(ci.Name), ci.ID, ci.ExternalID[0])
 }
 
 func (ci ConfigItem) String() string {
 	if len(ci.ExternalID) == 0 {
-		return fmt.Sprintf("id=%s type=%s name=%s ", ci.ID, lo.FromPtr(ci.Type), lo.FromPtr(ci.Name))
+		return fmt.Sprintf("id=%s type=%s name=%s ", ci.ID, ci.Type, lo.FromPtr(ci.Name))
 	}
 
-	return fmt.Sprintf("id=%s type=%s name=%s external_id=%s", ci.ID, lo.FromPtr(ci.Type), lo.FromPtr(ci.Name), ci.ExternalID[0])
+	return fmt.Sprintf("id=%s type=%s name=%s external_id=%s", ci.ID, ci.Type, lo.FromPtr(ci.Name), ci.ExternalID[0])
 }
 
 func (ci ConfigItem) ConfigJSONStringMap() (map[string]interface{}, error) {

db/update.go

@@ -127,9 +127,9 @@ func updateCI(ctx api.ScrapeContext, summary *v1.ScrapeSummary, result v1.Scrape
 		ctx.Errorf("[%s] failed to check for changes: %v", ci, err)
 	} else if changeResult != nil {
 		if ctx.Logger.V(5).Enabled() {
-			ctx.Logger.V(5).Infof("[%s/%s] detected changes %v", *ci.Type, ci.ExternalID[0], lo.FromPtr(changeResult.Diff))
+			ctx.Logger.V(5).Infof("[%s/%s] detected changes %v", ci.Type, ci.ExternalID[0], lo.FromPtr(changeResult.Diff))
 		} else {
-			ctx.Logger.V(3).Infof("[%s/%s] detected changes", *ci.Type, ci.ExternalID[0])
+			ctx.Logger.V(3).Infof("[%s/%s] detected changes", ci.Type, ci.ExternalID[0])
 
 		}
 		result.Changes = []v1.ChangeResult{*changeResult}
@@ -195,7 +195,7 @@ func updateCI(ctx api.ScrapeContext, summary *v1.ScrapeSummary, result v1.Scrape
 	if !stringEqual(ci.Source, existing.Source) {
 		updates["source"] = ci.Source
 	}
-	if !stringEqual(ci.Type, existing.Type) {
+	if ci.Type != existing.Type {
 		updates["type"] = ci.Type
 	}
 	if !stringEqual(ci.Status, existing.Status) {
@@ -247,7 +247,7 @@ func updateCI(ctx api.ScrapeContext, summary *v1.ScrapeSummary, result v1.Scrape
 	// same config items
 	if lo.FromPtr(existing.ScraperID) != lo.FromPtr(ci.ScraperID) {
 		updates["scraper_id"] = ci.ScraperID
-		summary.AddWarning(*ci.Type, fmt.Sprintf("updated scraper_id of config[%s] from %s to %s", ci, existing.ScraperID, ci.ScraperID))
+		summary.AddWarning(ci.Type, fmt.Sprintf("updated scraper_id of config[%s] from %s to %s", ci, existing.ScraperID, ci.ScraperID))
 	}
 
 	if ci.Properties != nil && len(*ci.Properties) > 0 && (existing.Properties == nil || !mapEqual(ci.Properties.AsMap(), existing.Properties.AsMap())) {
@@ -399,7 +399,7 @@ var orphanCache = cache.New(60*time.Minute, 10*time.Minute)
 
 func upsertAnalysis(ctx api.ScrapeContext, result *v1.ScrapeResult) error {
 	analysis := result.AnalysisResult.ToConfigAnalysis()
-	ciID, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: analysis.ConfigType, ExternalID: []string{analysis.ExternalID}})
+	ciID, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: analysis.ConfigType, ExternalID: analysis.ExternalID})
 	if err != nil {
 		return err
 	} else if ciID == nil {
@@ -443,7 +443,7 @@ func SaveResults(ctx api.ScrapeContext, results []v1.ScrapeResult) (v1.ScrapeSum
 
 func syncCRDChanges(ctx api.ScrapeContext, configs []*models.ConfigItem) error {
 	for _, config := range configs {
-		if !strings.HasPrefix(*config.Type, api.MissionControlConfigTypePrefix) {
+		if !strings.HasPrefix(config.Type, api.MissionControlConfigTypePrefix) {
 			continue
 		}
 
@@ -468,7 +468,7 @@ func syncCRDChanges(ctx api.ScrapeContext, configs []*models.ConfigItem) error {
 			return err
 		}
 
-		switch strings.TrimPrefix(*config.Type, api.MissionControlConfigTypePrefix) {
+		switch strings.TrimPrefix(config.Type, api.MissionControlConfigTypePrefix) {
 		case "ScrapeConfig":
 			scrapeConfig := dutyModels.ConfigScraper{
 				Name:   fmt.Sprintf("%s/%s", namespace, *config.Name),
@@ -576,7 +576,7 @@ func saveResults(ctx api.ScrapeContext, results []v1.ScrapeResult) (v1.ScrapeSum
 		return summary, fmt.Errorf("failed to create config items: %w", dutydb.ErrorDetails(err))
 	}
 	for _, config := range newConfigs {
-		summary.AddInserted(*config.Type)
+		summary.AddInserted(config.Type)
 	}
 
 	// nonUpdatedConfigs are existing configs that were not updated in this scrape.
@@ -591,9 +591,9 @@ func saveResults(ctx api.ScrapeContext, results []v1.ScrapeResult) (v1.ScrapeSum
 		}
 
 		if updated {
-			summary.AddUpdated(*updateArg.Existing.Type)
+			summary.AddUpdated(updateArg.Existing.Type)
 		} else {
-			summary.AddUnchanged(*updateArg.Existing.Type)
+			summary.AddUnchanged(updateArg.Existing.Type)
 			nonUpdatedConfigs = append(nonUpdatedConfigs, updateArg.Existing.ID)
 		}
 
@@ -790,7 +790,7 @@ func generateConfigChange(ctx api.ScrapeContext, newConf, prev models.ConfigItem
 	}
 
 	return &v1.ChangeResult{
-		ConfigType: lo.FromPtr(newConf.Type),
+		ConfigType: newConf.Type,
 		ChangeType: "diff",
 		ExternalID: newConf.ExternalID[0],
 		Diff:       &diff,
@@ -811,7 +811,7 @@ func relationshipSelectorToResults(ctx dutyContext.Context, inputs []v1.ScrapeRe
 
 			for _, id := range linkedConfigIDs {
 				rel := v1.RelationshipResult{
-					ConfigExternalID: v1.ExternalID{ExternalID: []string{input.ID}, ConfigType: input.Type},
+					ConfigExternalID: v1.ExternalID{ExternalID: input.ID, ConfigType: input.Type},
 					RelatedConfigID:  id.String(),
 				}
 
@@ -954,7 +954,7 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime
 				return nil, nil, nil, nil, allChangeSummary, fmt.Errorf("config item %s has no external id", ci)
 			}
 
-			parentExternalKey := configExternalKey{externalID: ci.ExternalID[0], parentType: lo.FromPtr(ci.Type)}
+			parentExternalKey := configExternalKey{externalID: ci.ExternalID[0], parentType: ci.Type}
 			parentTypeToConfigMap[parentExternalKey] = ci.ID
 
 			existing := &models.ConfigItem{}
@@ -963,7 +963,7 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime
 					return nil, nil, nil, nil, allChangeSummary, fmt.Errorf("unable to lookup existing config(%s): %w", ci, err)
 				}
 			} else {
-				if existing, err = ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: *ci.Type, ExternalID: []string{ci.ExternalID[0]}}); err != nil {
+				if existing, err = ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: ci.Type, ExternalID: ci.ExternalID[0]}); err != nil {
 					return nil, nil, nil, nil, allChangeSummary, fmt.Errorf("unable to lookup external id(%s): %w", ci, err)
 				}
 			}
@@ -993,8 +993,8 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime
 		} else {
 			if !changeSummary.IsEmpty() {
 				var configType string
-				if ci != nil && ci.Type != nil {
-					configType = lo.FromPtr(ci.Type)
+				if ci != nil {
+					configType = ci.Type
 				} else if len(result.Changes) > 0 {
 					configType = result.Changes[0].ConfigType
 				}
@@ -1067,7 +1067,7 @@ func setConfigProbableParents(ctx api.ScrapeContext, parentTypeToConfigMap map[c
 				continue
 			}
 
-			if foundParent, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: parent.Type, ExternalID: []string{parent.ExternalID}}); err != nil {
+			if foundParent, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: parent.Type, ExternalID: parent.ExternalID}); err != nil {
 				return err
 			} else if foundParent != nil {
 				// Ignore self parent reference
@@ -1095,7 +1095,7 @@ func setConfigChildren(ctx api.ScrapeContext, allConfigs models.ConfigItems) err
 				continue
 			}
 
-			found, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: child.Type, ExternalID: []string{child.ExternalID}})
+			found, err := ctx.TempCache().Find(ctx, v1.ExternalID{ConfigType: child.Type, ExternalID: child.ExternalID})
 			if err != nil {
 				return err
 			}

fixtures/kubernetes.yaml

@@ -44,6 +44,9 @@ spec:
           - selector: 'config_type == "Kubernetes::Certificate"'
             jsonpath: .spec.commonName
             value: md5sum
+          - selector: config_class == 'Connection'
+            jsonpath: "$..['password','bearer','clientSecret','personalAccessToken','certificate','secretKey','token'].value"
+            value: '******'
         exclude:
           - types:
               - Kubernetes::*
@@ -59,6 +62,10 @@ spec:
                 patch.status.containerStatuses.size() > 0 &&
                 has(patch.status.containerStatuses[0].restartCount)
               type: PodCrashLooping
+            - filter: >
+                change.change_type == 'diff' &&
+                jq('.status.conditions[] | select(.type == "Healthy").message', patch).contains('Health check passed')
+              type: HealthCheckPassed
           exclude:
             - 'config_type == "Kubernetes::Endpoints" && details.message == "metadata.annotations.endpoints.kubernetes.io/last-change-trigger-time"'
             - 'config_type == "Kubernetes::Node" && has(details.message) && details.message == "status.images"'