Skip to content

Commit

Permalink
Merge pull request #2352 from opengovern/fix-policy
Browse files Browse the repository at this point in the history
fix: fix schedulers
artaasadi authored Jan 4, 2025
2 parents 782f7e6 + fa55999 commit 7cbea4d
Showing 6 changed files with 57 additions and 77 deletions.
12 changes: 6 additions & 6 deletions jobs/query-validator-job/job.go
Original file line number Diff line number Diff line change
@@ -31,8 +31,8 @@ type Job struct {
Parameters []api.QueryParameter `json:"parameters"`
Query string `json:"query"`
IntegrationType []integration.Type `json:"integration_type"`
PrimaryTable *string `json:"primary_table"`
ListOfTables []string `json:"list_of_tables"`
PrimaryResource *string `json:"primary_resource"`
ListOfResources []string `json:"list_of_resources"`
}

func (w *Worker) RunJob(ctx context.Context, job Job) error {
@@ -46,12 +46,12 @@ func (w *Worker) RunJob(ctx context.Context, job Job) error {
if job.QueryType == QueryTypeComplianceControl {
w.logger.Info("QueryTypeComplianceControl")
queryResourceType := ""
if job.PrimaryTable != nil || len(job.ListOfTables) == 1 {
if job.PrimaryResource != nil || len(job.ListOfResources) == 1 {
tableName := ""
if job.PrimaryTable != nil {
tableName = *job.PrimaryTable
if job.PrimaryResource != nil {
tableName = *job.PrimaryResource
} else {
tableName = job.ListOfTables[0]
tableName = job.ListOfResources[0]
}
if tableName != "" {
queryResourceType, _, err = GetResourceTypeFromTableName(tableName, job.IntegrationType)
38 changes: 19 additions & 19 deletions services/compliance/api/control.go
Original file line number Diff line number Diff line change
@@ -50,7 +50,7 @@ type ControlSummary struct {
}

type ControlTrendDatapoint struct {
Timestamp int `json:"timestamp" example:"1686346668"` // Time
Timestamp int `json:"timestamp"` // Time
FailedResourcesCount int `json:"failedResourcesCount"`
TotalResourcesCount int `json:"totalResourcesCount"`
FailedIntegrationCount int `json:"failedIntegrationCount"`
@@ -63,8 +63,8 @@ type ControlsFilterSummaryRequest struct {
RootBenchmark []string `json:"root_benchmark"`
ParentBenchmark []string `json:"parent_benchmark"`
HasParameters *bool `json:"has_parameters"`
PrimaryTable []string `json:"primary_table"`
ListOfTables []string `json:"list_of_tables"`
PrimaryResource []string `json:"primary_resource"`
ListOfResources []string `json:"list_of_resources"`
Tags map[string][]string `json:"tags"`
TagsRegex *string `json:"tags_regex"`
ComplianceResultFilters *ComplianceResultFilters `json:"compliance_result_filters"`
@@ -76,8 +76,8 @@ type ListControlsFilterRequest struct {
RootBenchmark []string `json:"root_benchmark"`
ParentBenchmark []string `json:"parent_benchmark"`
HasParameters *bool `json:"has_parameters"`
PrimaryTable []string `json:"primary_table"`
ListOfTables []string `json:"list_of_tables"`
PrimaryResource []string `json:"primary_resource"`
ListOfResources []string `json:"list_of_resources"`
Tags map[string][]string `json:"tags"`
TagsRegex *string `json:"tags_regex"`
ComplianceResultFilters *ComplianceResultFilters `json:"compliance_result_filters"`
@@ -100,11 +100,11 @@ type ListControlsFilterResultControl struct {
IntegrationType []integration.Type `json:"integration_type"`
Severity types.ComplianceResultSeverity `json:"severity"`
Tags map[string][]string `json:"tags"`
Query struct {
PrimaryTable string `json:"primary_table"`
ListOfTables []string `json:"list_of_tables"`
Parameters []QueryParameter `json:"parameters"`
} `json:"query"`
Policy struct {
PrimaryResource string `json:"primary_resource"`
ListOfResources []string `json:"list_of_resources"`
Parameters []QueryParameter `json:"parameters"`
} `json:"policy"`
ComplianceResultsSummary struct {
IncidentCount int64 `json:"incident_count"`
NonIncidentCount int64 `json:"non_incident_count"`
@@ -120,8 +120,8 @@ type ControlsFilterSummaryResult struct {
IntegrationTypes []string `json:"integration_types"`
Severity []string `json:"severity"`
Tags map[string][]string `json:"tags"`
PrimaryTable []string `json:"primary_table"`
ListOfTables []string `json:"list_of_tables"`
PrimaryResource []string `json:"primary_resource"`
ListOfResources []string `json:"list_of_resources"`
}

type ControlTagsResult struct {
@@ -140,13 +140,13 @@ type GetControlDetailsResponse struct {
Description string `json:"description"`
IntegrationType []integration.Type `json:"integrationType"`
Severity string `json:"severity"`
Query struct {
Language string `json:"language"`
Definition string `json:"definition"`
PrimaryTable string `json:"primaryTable"`
ListOfTables []string `json:"listOfTables"`
Parameters []QueryParameter `json:"parameters"`
} `json:"query"`
Policy struct {
Language string `json:"language"`
Definition string `json:"definition"`
PrimaryResource string `json:"primaryResource"`
ListOfResources []string `json:"listOfResources"`
Parameters []QueryParameter `json:"parameters"`
} `json:"policy"`
Tags map[string][]string `json:"tags"`
Benchmarks *struct {
Roots []string `json:"roots"`
12 changes: 6 additions & 6 deletions services/compliance/http_routes.go
Original file line number Diff line number Diff line change
@@ -2280,7 +2280,7 @@ func (h *HttpHandler) ListControlsFiltered(echoCtx echo.Context) error {
}

controls, err := h.db.ListControlsByFilter(ctx, nil, req.IntegrationTypes, req.Severity, benchmarks, req.Tags, req.HasParameters,
req.PrimaryTable, req.ListOfTables, nil)
req.PrimaryResource, req.ListOfResources, nil)
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest, err.Error())
}
@@ -2362,7 +2362,7 @@ func (h *HttpHandler) ListControlsFiltered(echoCtx echo.Context) error {
IntegrationType: integration_type.ParseTypes(control.IntegrationType),
Severity: control.Severity,
Tags: filterTagsByRegex(req.TagsRegex, model.TrimPrivateTags(control.GetTagsMap())),
Query: struct {
Policy: struct {

Check failure on line 2365 in services/compliance/http_routes.go

GitHub Actions / build

cannot use struct{PrimaryTable string `json:"primary_table"`; ListOfTables []string `json:"list_of_tables"`; Parameters []api.QueryParameter `json:"parameters"`}{…} (value of type struct{PrimaryTable string "json:\"primary_table\""; ListOfTables []string "json:\"list_of_tables\""; Parameters []"github.com/opengovern/opencomply/services/compliance/api".QueryParameter "json:\"parameters\""}) as struct{PrimaryResource string "json:\"primary_resource\""; ListOfResources []string "json:\"list_of_resources\""; Parameters []"github.com/opengovern/opencomply/services/compliance/api".QueryParameter "json:\"parameters\""} value in struct literal
PrimaryTable string `json:"primary_table"`
ListOfTables []string `json:"list_of_tables"`
Parameters []api.QueryParameter `json:"parameters"`
@@ -2373,7 +2373,7 @@ func (h *HttpHandler) ListControlsFiltered(echoCtx echo.Context) error {
},
}
for _, p := range control.Policy.Parameters {
apiControl.Query.Parameters = append(apiControl.Query.Parameters, p.ToApi())
apiControl.Policy.Parameters = append(apiControl.Policy.Parameters, p.ToApi())
}

h.logger.Info("ListControlsByFilter", zap.Strings("benchmarks", benchmarks))
@@ -2420,10 +2420,10 @@ func (h *HttpHandler) ListControlsFiltered(echoCtx echo.Context) error {
uniqueIntegrationTypes[c.String()] = true
}
uniqueSeverities[apiControl.Severity.String()] = true
for _, t := range apiControl.Query.ListOfTables {
for _, t := range apiControl.Policy.ListOfResources {
uniqueListOfTables[t] = true
}
uniquePrimaryTables[apiControl.Query.PrimaryTable] = true
uniquePrimaryTables[apiControl.Policy.PrimaryResource] = true

for k, vs := range apiControl.Tags {
if _, ok := uniqueTags[k]; !ok {
@@ -2580,7 +2580,7 @@ func (h *HttpHandler) GetControlDetails(echoCtx echo.Context) error {
Description: control.Description,
IntegrationType: integration_type.ParseTypes(control.IntegrationType),
Severity: control.Severity.String(),
Query: struct {
Policy: struct {

Check failure on line 2583 in services/compliance/http_routes.go

GitHub Actions / build

cannot use struct{Language string `json:"language"`; Definition string `json:"definition"`; PrimaryTable string `json:"primaryTable"`; ListOfTables []string `json:"listOfTables"`; Parameters []api.QueryParameter `json:"parameters"`}{…} (value of type struct{Language string "json:\"language\""; Definition string "json:\"definition\""; PrimaryTable string "json:\"primaryTable\""; ListOfTables []string "json:\"listOfTables\""; Parameters []"github.com/opengovern/opencomply/services/compliance/api".QueryParameter "json:\"parameters\""}) as struct{Language string "json:\"language\""; Definition string "json:\"definition\""; PrimaryResource string "json:\"primaryResource\""; ListOfResources []string "json:\"listOfResources\""; Parameters []"github.com/opengovern/opencomply/services/compliance/api".QueryParameter "json:\"parameters\""} value in struct literal
Language string `json:"language"`
Definition string `json:"definition"`
PrimaryTable string `json:"primaryTable"`
56 changes: 18 additions & 38 deletions services/describe/schedulers/compliance/trigger.go
Original file line number Diff line number Diff line change
@@ -67,7 +67,7 @@ func (s *JobScheduler) buildRunners(
if control.Policy == nil {
continue
}
if connector != nil && len(control.Policy.IntegrationType) > 0 && !control.Policy.Global {
if connector != nil && len(control.Policy.IntegrationType) > 0 {
supportsConnector := false
for _, c := range control.Policy.IntegrationType {
if *connector == c {
@@ -87,44 +87,24 @@ func (s *JobScheduler) buildRunners(
ControlID: control.ID,
ControlSeverity: control.Severity,
}
if control.Policy.Global == true {
runnerJob := model.ComplianceRunner{
FrameworkID: rootBenchmarkID,
QueryID: control.Policy.ID,
IntegrationID: nil,
ResourceCollectionID: resourceCollectionID,
ParentJobID: parentJobID,
StartedAt: time.Time{},
RetryCount: 0,
Status: runner.ComplianceRunnerCreated,
FailureMessage: "",
TriggerType: triggerType,
}
err = runnerJob.SetCallers([]runner.Caller{callers})
if err != nil {
return nil, nil, err
}
globalRunners = append(globalRunners, &runnerJob)
} else {
runnerJob := model.ComplianceRunner{
FrameworkID: rootBenchmarkID,
QueryID: control.Policy.ID,
IntegrationID: connectionID,
ResourceCollectionID: resourceCollectionID,
ParentJobID: parentJobID,
StartedAt: time.Time{},
RetryCount: 0,
Status: runner.ComplianceRunnerCreated,
FailureMessage: "",
TriggerType: triggerType,
}
err = runnerJob.SetCallers([]runner.Caller{callers})
if err != nil {
return nil, nil, err
}
runners = append(runners, &runnerJob)
}

runnerJob := model.ComplianceRunner{
FrameworkID: rootBenchmarkID,
QueryID: control.Policy.ID,
IntegrationID: connectionID,
ResourceCollectionID: resourceCollectionID,
ParentJobID: parentJobID,
StartedAt: time.Time{},
RetryCount: 0,
Status: runner.ComplianceRunnerCreated,
FailureMessage: "",
TriggerType: triggerType,
}
err = runnerJob.SetCallers([]runner.Caller{callers})
if err != nil {
return nil, nil, err
}
runners = append(runners, &runnerJob)
}

uniqueMap := map[string]*model.ComplianceRunner{}
4 changes: 2 additions & 2 deletions services/describe/schedulers/query-runner/publisher.go
Original file line number Diff line number Diff line change
@@ -51,8 +51,8 @@ func (s *JobScheduler) runPublisher(ctx context.Context) error {
query = namedQuery.Query.QueryToExecute
parameters = namedQuery.Query.Parameters
} else if controlQuery != nil {
query = controlQuery.Query.Definition
for _, qp := range controlQuery.Query.Parameters {
query = controlQuery.Policy.Definition
for _, qp := range controlQuery.Policy.Parameters {
parameters = append(parameters, inventoryApi.QueryParameter{
Key: qp.Key,
Required: qp.Required,
12 changes: 6 additions & 6 deletions services/describe/schedulers/query-validator/publisher.go
Original file line number Diff line number Diff line change
@@ -53,8 +53,8 @@ func (s *JobScheduler) runPublisher(ctx context.Context) error {
}
jobMsg.Query = namedQuery.Query.QueryToExecute
jobMsg.Parameters = namedQuery.Query.Parameters
jobMsg.ListOfTables = namedQuery.Query.ListOfTables
jobMsg.PrimaryTable = namedQuery.Query.PrimaryTable
jobMsg.ListOfResources = namedQuery.Query.ListOfTables
jobMsg.PrimaryResource = namedQuery.Query.PrimaryTable
jobMsg.IntegrationType = namedQuery.IntegrationTypes
} else if job.QueryType == queryvalidator.QueryTypeComplianceControl {
jobMsg.QueryType = queryvalidator.QueryTypeComplianceControl
@@ -63,17 +63,17 @@ func (s *JobScheduler) runPublisher(ctx context.Context) error {
if err != nil {
s.logger.Error("Get Control Error", zap.Error(err))
}
jobMsg.Query = controlQuery.Query.Definition
jobMsg.Query = controlQuery.Policy.Definition
var parameters []inventoryApi.QueryParameter
for _, qp := range controlQuery.Query.Parameters {
for _, qp := range controlQuery.Policy.Parameters {
parameters = append(parameters, inventoryApi.QueryParameter{
Key: qp.Key,
Required: qp.Required,
})
}
jobMsg.Parameters = parameters
jobMsg.ListOfTables = controlQuery.Query.ListOfTables
jobMsg.PrimaryTable = controlQuery.Query.PrimaryTable
jobMsg.ListOfResources = controlQuery.Policy.ListOfResources
jobMsg.PrimaryResource = &controlQuery.Policy.PrimaryResource
jobMsg.IntegrationType = controlQuery.IntegrationType
} else {
_ = s.db.UpdateQueryValidatorJobStatus(job.ID, queryvalidator.QueryValidatorFailed, "query ID not found")

0 comments on commit 7cbea4d

Please sign in to comment.