Skip to content

Commit

Permalink
Merge pull request #2368 from opengovern/fix-queries
Browse files Browse the repository at this point in the history 
fix: update views and queries structure
  • Loading branch information
@artaasadi
artaasadi authored Jan 6, 2025
2 parents b73bc08 + 722f1e3 commit a75bca7
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 73 deletions.
60 changes: 19 additions & 41 deletions jobs/post-install-job/job/migrations/compliance/git_parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ type GitParser struct {
queryViews []models.QueryView
coreServiceQueries []models.Query
controlsPolicies map[string]db.Policy
namedPolicies map[string]NamedPolicy
namedPolicies map[string]NamedQuery
Comparison *git.ComparisonResultGrouped

manualRemediationMap map[string]string
Expand Down Expand Up @@ -75,7 +75,7 @@ func (g *GitParser) ExtractNamedQueries() error {
return err
}

var item NamedPolicy
var item NamedQuery
err = yaml.Unmarshal(content, &item)
if err != nil {
g.logger.Error("failure in unmarshal", zap.String("path", path), zap.Error(err))
Expand Down Expand Up @@ -315,7 +315,7 @@ func (g *GitParser) parseControlFile(content []byte, path string) error {
for _, it := range query.IntegrationTypes {
integrationTypes = append(integrationTypes, string(it))
}
listOfTables, err := utils.ExtractTableRefsFromPolicy(types.PolicyLanguageSQL, query.Policy.QueryToExecute)
listOfTables, err := utils.ExtractTableRefsFromPolicy(types.PolicyLanguageSQL, query.Query)
if err != nil {
g.logger.Error("failed to extract table refs from query", zap.String("query-id", control.ID), zap.Error(err))
return nil
Expand All @@ -328,13 +328,10 @@ func (g *GitParser) parseControlFile(content []byte, path string) error {
}

var primaryResource string
if query.Policy.PrimaryTable != nil {
primaryResource = *query.Policy.PrimaryTable
}

p := db.Policy{
ID: control.ID,
Definition: query.Policy.QueryToExecute,
Definition: query.Query,
IntegrationType: integrationTypes,
PrimaryResource: primaryResource,
ListOfResources: listOfTables,
Expand Down Expand Up @@ -804,45 +801,26 @@ func (g *GitParser) ExtractQueryViews(viewsPath string) error {
}

qv := models.QueryView{
ID: obj.ID,
Title: obj.Title,
Description: obj.Description,
Dependencies: obj.Dependencies,
ID: obj.ID,
Title: obj.Title,
Description: obj.Description,
}

if obj.Query != nil {
listOfTables, err := utils.ExtractTableRefsFromPolicy(types.PolicyLanguageSQL, obj.Query.QueryToExecute)
if err != nil {
g.logger.Error("failed to extract table refs from query", zap.String("query-id", obj.ID), zap.Error(err))
listOfTables = obj.Query.ListOfTables
}

q := models.Query{
ID: obj.ID,
QueryToExecute: obj.Query.QueryToExecute,
PrimaryTable: obj.Query.PrimaryTable,
ListOfTables: listOfTables,
Engine: obj.Query.Engine,
Global: obj.Query.Global,
}
for _, parameter := range obj.Query.Parameters {
q.Parameters = append(q.Parameters, models.QueryParameter{
QueryID: obj.ID,
Key: parameter.Key,
Required: parameter.Required,
})
listOfTables, err := utils.ExtractTableRefsFromPolicy(types.PolicyLanguageSQL, obj.Query)
if err != nil {
g.logger.Error("failed to extract table refs from query", zap.String("query-id", obj.ID), zap.Error(err))
}

if parameter.DefaultValue != "" {
g.policyParamValues = append(g.policyParamValues, models.PolicyParameterValues{
Key: parameter.Key,
Value: parameter.DefaultValue,
})
}
}
g.coreServiceQueries = append(g.coreServiceQueries, q)
qv.QueryID = &obj.ID
q := models.Query{
ID: obj.ID,
QueryToExecute: obj.Query,
ListOfTables: listOfTables,
Engine: "sql",
}

g.coreServiceQueries = append(g.coreServiceQueries, q)
qv.QueryID = &obj.ID

g.queryViews = append(g.queryViews, qv)

return nil
Expand Down
29 changes: 5 additions & 24 deletions jobs/post-install-job/job/migrations/compliance/populate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func (m Migration) Run(ctx context.Context, conf config.MigratorConfig, logger *
logger: logger,
frameworksChildren: make(map[string][]string),
controlsPolicies: make(map[string]db.Policy),
namedPolicies: make(map[string]NamedPolicy),
namedPolicies: make(map[string]NamedQuery),
}
if err := p.ExtractCompliance(config.ComplianceGitPath, config.ControlEnrichmentGitPath); err != nil {
logger.Error("failed to extract controls and benchmarks", zap.Error(err))
Expand Down Expand Up @@ -359,7 +359,7 @@ func populateFinderItem(logger *zap.Logger, tx *gorm.DB, path string, info fs.Fi
return err
}

var item NamedPolicy
var item NamedQuery
err = yaml.Unmarshal(content, &item)
if err != nil {
logger.Error("failure in unmarshal", zap.String("path", path), zap.Error(err))
Expand Down Expand Up @@ -400,33 +400,16 @@ func populateFinderItem(logger *zap.Logger, tx *gorm.DB, path string, info fs.Fi
QueryID: &id,
}
queryParams := []models.QueryParameter{}
for _, qp := range item.Policy.Parameters {
queryParams = append(queryParams, models.QueryParameter{
Key: qp.Key,
Required: qp.Required,
QueryID: dbMetric.ID,
})
if qp.DefaultValue != "" {
queryParamObj := models.PolicyParameterValues{
Key: qp.Key,
Value: qp.DefaultValue,
}
QueryParameters = append(QueryParameters, queryParamObj)
}
}
listOfTables, err := utils.ExtractTableRefsFromPolicy("sql", item.Policy.QueryToExecute)
listOfTables, err := utils.ExtractTableRefsFromPolicy("sql", item.Query)
if err != nil {
logger.Error("failed to extract table refs from query", zap.String("query-id", dbMetric.ID), zap.Error(err))
listOfTables = item.Policy.ListOfTables
}
query := models.Query{
ID: dbMetric.ID,
QueryToExecute: item.Policy.QueryToExecute,
PrimaryTable: item.Policy.PrimaryTable,
QueryToExecute: item.Query,
ListOfTables: listOfTables,
Engine: item.Policy.Engine,
Engine: "sql",
Parameters: queryParams,
Global: item.Policy.Global,
}
err = tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "id"}}, // key column
Expand Down Expand Up @@ -455,8 +438,6 @@ func populateFinderItem(logger *zap.Logger, tx *gorm.DB, path string, info fs.Fi
return err
}

// logger.Info("parsed the tags", zap.String("id", id), zap.Any("tags", tags))

if len(tags) > 0 {
for _, tag := range tags {
err = tx.Model(&models.NamedQueryTag{}).Create(&tag).Error
Expand Down
15 changes: 7 additions & 8 deletions jobs/post-install-job/job/migrations/compliance/translator_model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,19 +58,18 @@ type Control struct {
}

type QueryView struct {
ID string `json:"id" yaml:"ID"`
Title string `json:"title" yaml:"Title"`
Description string `json:"description" yaml:"Description"`
Query *shared.Query `json:"query" yaml:"Policy"`

Dependencies []string `json:"dependencies" yaml:"Dependencies"`
ID string `json:"id" yaml:"id"`
Title string `json:"title" yaml:"title"`
Description string `json:"description" yaml:"description"`
Query string `json:"query" yaml:"query"`
Tags map[string][]string `json:"tags" yaml:"tags"`
}

type NamedPolicy struct {
type NamedQuery struct {
ID string `json:"id" yaml:"id"`
Title string `json:"title" yaml:"title"`
Description string `json:"description" yaml:"description"`
IntegrationTypes []integration.Type `json:"integration_type" yaml:"integration_type"`
Policy shared.Query `json:"policy" yaml:"policy"`
Query string `json:"query" yaml:"query"`
Tags map[string][]string `json:"tags" yaml:"tags"`
}

0 comments on commit a75bca7

Please sign in to comment.