Skip to content

Commit

Permalink
Merge pull request #2260 from opengovern/fix-control-summary
Browse files Browse the repository at this point in the history 
fix: merge compliance jobs tables
  • Loading branch information
@artaasadi
artaasadi authored Dec 17, 2024
2 parents fb27ff8 + 9dc2885 commit c8d3bb7
Show file tree
Hide file tree
Showing 18 changed files with 271 additions and 471 deletions.
74 changes: 32 additions & 42 deletions jobs/compliance-quick-run-job/job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ import (
)

type AuditJob struct {
JobID uint
FrameworkID string
IntegrationIDs []string
IncludeResult []string
JobID uint
FrameworkID string
IntegrationID string
IncludeResult []string

JobReportControlSummary *types.ComplianceJobReportControlSummary
JobReportControlView *types.ComplianceJobReportControlView
Expand All @@ -31,7 +31,7 @@ type AuditJob struct {

type JobResult struct {
JobID uint
Status model.ComplianceQuickRunStatus
Status model.ComplianceJobStatus
FailureMessage string
}

Expand All @@ -40,11 +40,11 @@ func (w *Worker) RunJob(ctx context.Context, job *AuditJob) error {
Controls: make(map[string]types.AuditControlResult),
ComplianceSummary: make(map[types.ComplianceStatus]uint64),
JobSummary: types.JobSummary{
JobID: job.JobID,
FrameworkID: job.FrameworkID,
Auditable: false,
JobStartedAt: time.Now(),
IntegrationIDs: job.IntegrationIDs,
JobID: job.JobID,
FrameworkID: job.FrameworkID,
Auditable: false,
JobStartedAt: time.Now(),
IntegrationID: job.IntegrationID,
},
}
job.JobReportControlSummary = &types.ComplianceJobReportControlSummary{
Expand All @@ -55,50 +55,40 @@ func (w *Worker) RunJob(ctx context.Context, job *AuditJob) error {
FailedControls: 0,
},
JobSummary: types.JobSummary{
JobID: job.JobID,
FrameworkID: job.FrameworkID,
Auditable: false,
JobStartedAt: time.Now(),
IntegrationIDs: job.IntegrationIDs,
JobID: job.JobID,
FrameworkID: job.FrameworkID,
Auditable: false,
JobStartedAt: time.Now(),
IntegrationID: job.IntegrationID,
},
}
job.JobReportResourceView = &types.ComplianceJobReportResourceView{
Integrations: make(map[string]types.AuditIntegrationResult),
ComplianceSummary: make(map[types.ComplianceStatus]uint64),
JobSummary: types.JobSummary{
JobID: job.JobID,
FrameworkID: job.FrameworkID,
Auditable: false,
JobStartedAt: time.Now(),
IntegrationIDs: job.IntegrationIDs,
JobID: job.JobID,
FrameworkID: job.FrameworkID,
Auditable: false,
JobStartedAt: time.Now(),
IntegrationID: job.IntegrationID,
},
}

totalControls := make(map[string]bool)
failedControls := make(map[string]bool)
if len(job.IntegrationIDs) > 0 {
for _, integrationID := range job.IntegrationIDs {
err := w.RunJobForIntegration(ctx, job, integrationID, &totalControls, &failedControls)
if err != nil {
w.logger.Error("failed to run audit job for integration", zap.String("integration_id", integrationID), zap.Error(err))
return err
}
w.logger.Info("audit job for integration completed", zap.String("integration_id", integrationID))
}
} else {
err := w.RunJobForIntegration(ctx, job, "all", &totalControls, &failedControls)
if err != nil {
w.logger.Error("failed to run audit job for all integrations", zap.Error(err))
return err
}
w.logger.Info("audit job for all integration completed")

err := w.RunJobForIntegration(ctx, job, job.IntegrationID, &totalControls, &failedControls)
if err != nil {
w.logger.Error("failed to run audit job for integration", zap.String("integration_id", job.IntegrationID), zap.Error(err))
return err
}
w.logger.Info("audit job for integration completed", zap.String("integration_id", job.IntegrationID))

keys, idx := job.JobReportControlView.KeysAndIndex()
job.JobReportControlView.EsID = es.HashOf(keys...)
job.JobReportControlView.EsIndex = idx

err := sendDataToOpensearch(w.esClient.ES(), *job.JobReportControlView)
err = sendDataToOpensearch(w.esClient.ES(), *job.JobReportControlView)
if err != nil {
return err
}
Expand Down Expand Up @@ -136,9 +126,9 @@ func (w *Worker) RunJobForIntegration(ctx context.Context, job *AuditJob, integr
include["alarm"] = true
}

job.JobReportControlView.JobSummary.IntegrationIDs = append(job.JobReportControlView.JobSummary.IntegrationIDs, integrationId)
job.JobReportResourceView.JobSummary.IntegrationIDs = append(job.JobReportResourceView.JobSummary.IntegrationIDs, integrationId)
job.JobReportControlSummary.JobSummary.IntegrationIDs = append(job.JobReportControlSummary.JobSummary.IntegrationIDs, integrationId)
job.JobReportControlView.JobSummary.IntegrationID = integrationId
job.JobReportResourceView.JobSummary.IntegrationID = integrationId
job.JobReportControlSummary.JobSummary.IntegrationID = integrationId

job.JobReportResourceView.Integrations[integrationId] = types.AuditIntegrationResult{
ResourceTypes: make(map[string]types.AuditResourceTypesResult),
Expand Down Expand Up @@ -169,8 +159,8 @@ func (w *Worker) RunJobForIntegration(ctx context.Context, job *AuditJob, integr
queryJob := QueryJob{
AuditJobID: job.JobID,
ExecutionPlan: ExecutionPlan{
Query: *control.Query,
IntegrationIDs: job.IntegrationIDs,
Query: *control.Query,
IntegrationID: job.IntegrationID,
},
}
queryResults, err := w.RunQuery(ctx, queryJob)
Expand Down
12 changes: 6 additions & 6 deletions jobs/compliance-quick-run-job/query_result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ type QueryResult struct {
type ExecutionPlan struct {
Query complianceApi.Query

IntegrationIDs []string
IntegrationID string
}

type QueryJob struct {
Expand All @@ -41,7 +41,7 @@ type QueryJob struct {
func (w *Worker) RunQuery(ctx context.Context, j QueryJob) ([]QueryResult, error) {
w.logger.Info("Running query",
zap.String("query_id", j.ExecutionPlan.Query.ID),
zap.Strings("integration_ids", j.ExecutionPlan.IntegrationIDs),
zap.String("integration_ids", j.ExecutionPlan.IntegrationID),
)

queryParams, err := w.metadataClient.ListQueryParameters(&httpclient.Context{Ctx: ctx, UserRole: authApi.AdminRole})
Expand All @@ -59,15 +59,15 @@ func (w *Worker) RunQuery(ctx context.Context, j QueryJob) ([]QueryResult, error
w.logger.Error("required query parameter not found",
zap.String("key", param.Key),
zap.String("query_id", j.ExecutionPlan.Query.ID),
zap.Strings("integration_ids", j.ExecutionPlan.IntegrationIDs),
zap.String("integration_id", j.ExecutionPlan.IntegrationID),
)
return nil, fmt.Errorf("required query parameter not found: %s for query: %s", param.Key, j.ExecutionPlan.Query.ID)
}
if _, ok := queryParamMap[param.Key]; !ok && !param.Required {
w.logger.Info("optional query parameter not found",
zap.String("key", param.Key),
zap.String("query_id", j.ExecutionPlan.Query.ID),
zap.Strings("integration_ids", j.ExecutionPlan.IntegrationIDs),
zap.String("integration_id", j.ExecutionPlan.IntegrationID),
)
queryParamMap[param.Key] = ""
}
Expand Down Expand Up @@ -106,7 +106,7 @@ func (w *Worker) runSqlWorkerJob(ctx context.Context, j QueryJob, queryParamMap
w.logger.Error("failed to execute query template",
zap.Error(err),
zap.String("query_id", j.ExecutionPlan.Query.ID),
zap.Strings("integration_ids", j.ExecutionPlan.IntegrationIDs),
zap.String("integration_id", j.ExecutionPlan.IntegrationID),
zap.Uint("job_id", j.AuditJobID),
)
return nil, fmt.Errorf("failed to execute query template: %w for query: %s", err, j.ExecutionPlan.Query.ID)
Expand All @@ -119,7 +119,7 @@ func (w *Worker) runSqlWorkerJob(ctx context.Context, j QueryJob, queryParamMap
zap.String("query", queryOutput.String()))
res, err := w.steampipeConn.QueryAll(ctx, queryOutput.String())
if err != nil {
w.logger.Error("failed to run query", zap.Error(err), zap.String("query_id", j.ExecutionPlan.Query.ID), zap.Strings("integration_ids", j.ExecutionPlan.IntegrationIDs))
w.logger.Error("failed to run query", zap.Error(err), zap.String("query_id", j.ExecutionPlan.Query.ID), zap.String("integration_id", j.ExecutionPlan.IntegrationID))
return nil, err
}

Expand Down
6 changes: 3 additions & 3 deletions jobs/compliance-quick-run-job/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,16 +166,16 @@ func (w *Worker) ProcessMessage(ctx context.Context, msg jetstream.Msg) (err err

result := JobResult{
JobID: job.JobID,
Status: model.ComplianceQuickRunStatusInProgress,
Status: model.ComplianceJobRunnersInProgress,
FailureMessage: "",
}

defer func() {
if err != nil {
result.FailureMessage = err.Error()
result.Status = model.ComplianceQuickRunStatusFailed
result.Status = model.ComplianceJobFailed
} else {
result.Status = model.ComplianceQuickRunStatusSucceeded
result.Status = model.ComplianceJobSucceeded
}

resultJson, err := json.Marshal(result)
Expand Down
6 changes: 3 additions & 3 deletions jobs/compliance-summarizer-job/job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,14 +237,14 @@ func (w *Worker) RunJob(ctx context.Context, j types2.Job) error {
integrations = append(integrations, i)
}

controlView.JobSummary.IntegrationIDs = integrations
controlView.JobSummary.IntegrationID = integrations

Check failure on line 240 in jobs/compliance-summarizer-job/job.go

View workflow job for this annotation

GitHub Actions / build 

cannot use integrations (variable of type []string) as string value in assignment
keys, idx = controlView.KeysAndIndex()
controlView.EsID = es2.HashOf(keys...)
controlView.EsIndex = idx

err = sendDataToOpensearch(w.esClient.ES(), controlView)

resourceView.JobSummary.IntegrationIDs = integrations
resourceView.JobSummary.IntegrationID = integrations

Check failure on line 247 in jobs/compliance-summarizer-job/job.go

View workflow job for this annotation

GitHub Actions / build 

cannot use integrations (variable of type []string) as string value in assignment
keys, idx = resourceView.KeysAndIndex()
resourceView.EsID = es2.HashOf(keys...)
resourceView.EsIndex = idx
Expand All @@ -254,7 +254,7 @@ func (w *Worker) RunJob(ctx context.Context, j types2.Job) error {
return err
}

controlSummary.JobSummary.IntegrationIDs = integrations
controlSummary.JobSummary.IntegrationID = integrations

Check failure on line 257 in jobs/compliance-summarizer-job/job.go

View workflow job for this annotation

GitHub Actions / build 

cannot use integrations (variable of type []string) as string value in assignment
controlSummary.ControlScore.TotalControls = int64(len(totalControls))
controlSummary.ControlScore.FailedControls = int64(len(failedControls))
keys, idx = controlSummary.KeysAndIndex()
Expand Down
10 changes: 5 additions & 5 deletions pkg/types/compliance_job_report_control_view.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ type AuditControlResult struct {
}

type JobSummary struct {
JobID uint `json:"job_id"`
Auditable bool `json:"auditable"`
FrameworkID string `json:"framework_id"`
JobStartedAt time.Time `json:"job_started_at"`
IntegrationIDs []string `json:"integration_ids"`
JobID uint `json:"job_id"`
Auditable bool `json:"auditable"`
FrameworkID string `json:"framework_id"`
JobStartedAt time.Time `json:"job_started_at"`
IntegrationID string `json:"integration_id"`
}
2 changes: 2 additions & 0 deletions services/describe/api/jobs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ type GetComplianceJobsHistoryRequest struct {

type GetComplianceJobsHistoryResponse struct {
JobId uint `json:"job_id"`
WithIncidents bool `json:"with_incidents"`
BenchmarkId string `json:"benchmark_id"`
JobStatus ComplianceJobStatus `json:"job_status"`
DateTime time.Time `json:"date_time"`
Expand Down Expand Up @@ -292,6 +293,7 @@ type GetDescribeJobsHistoryByIntegrationRequest struct {

type GetComplianceJobsHistoryByIntegrationRequest struct {
IntegrationInfo IntegrationInfoFilter `json:"integration_info"`
WithIncidents *bool `json:"with_incidents"`
BenchmarkId []string `json:"benchmark_id"`
JobStatus []string `json:"job_status"`
StartTime time.Time `json:"start_time"`
Expand Down
Loading

0 comments on commit c8d3bb7

Please sign in to comment.