From 1f4eb6dcd8b9922a4b2ec0aa0f633e58819f9a99 Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Thu, 7 Mar 2024 13:58:13 +0100 Subject: [PATCH 1/3] fix incognito scans Signed-off-by: Ivan Milchev --- apps/cnquery/cmd/scan.go | 45 +++++++++++----------- explorer/scan/local_scanner.go | 69 +++++++++++++++++++++++----------- 2 files changed, 69 insertions(+), 45 deletions(-) diff --git a/apps/cnquery/cmd/scan.go b/apps/cnquery/cmd/scan.go index e995179b0b..0dfabcc349 100644 --- a/apps/cnquery/cmd/scan.go +++ b/apps/cnquery/cmd/scan.go @@ -237,31 +237,28 @@ func getCobraScanConfig(cmd *cobra.Command, runtime *providers.Runtime, cliRes * conf.Inventory.ApplyCategory(inventory.AssetCategory_CATEGORY_CICD) } - var serviceAccount *upstream.ServiceAccountCredentials - if !conf.IsIncognito { - serviceAccount = opts.GetServiceCredential() - if serviceAccount != nil { - // TODO: determine if this needs migrating - // // determine information about the client - // sysInfo, err := sysinfo.GatherSystemInfo() - // if err != nil { - // log.Warn().Err(err).Msg("could not gather client information") - // } - // plugins = append(plugins, defaultRangerPlugins(sysInfo, opts.GetFeatures())...) - - log.Info().Msg("using service account credentials") - conf.runtime.UpstreamConfig = &upstream.UpstreamConfig{ - SpaceMrn: opts.GetParentMrn(), - ApiEndpoint: opts.UpstreamApiEndpoint(), - ApiProxy: opts.APIProxy, - Incognito: conf.IsIncognito, - Creds: serviceAccount, - } - providers.DefaultRuntime().UpstreamConfig = conf.runtime.UpstreamConfig - } else { - log.Warn().Msg("No credentials provided. Switching to --incognito mode.") - conf.IsIncognito = true + serviceAccount := opts.GetServiceCredential() + if serviceAccount != nil { + // TODO: determine if this needs migrating + // // determine information about the client + // sysInfo, err := sysinfo.GatherSystemInfo() + // if err != nil { + // log.Warn().Err(err).Msg("could not gather client information") + // } + // plugins = append(plugins, defaultRangerPlugins(sysInfo, opts.GetFeatures())...) + + log.Info().Msg("using service account credentials") + conf.runtime.UpstreamConfig = &upstream.UpstreamConfig{ + SpaceMrn: opts.GetParentMrn(), + ApiEndpoint: opts.UpstreamApiEndpoint(), + ApiProxy: opts.APIProxy, + Incognito: conf.IsIncognito, + Creds: serviceAccount, } + providers.DefaultRuntime().UpstreamConfig = conf.runtime.UpstreamConfig + } else { + log.Warn().Msg("No credentials provided. Switching to --incognito mode.") + conf.IsIncognito = true } if len(conf.QueryPackPaths) > 0 && !conf.IsIncognito { diff --git a/explorer/scan/local_scanner.go b/explorer/scan/local_scanner.go index 4c5502abb9..deee3f3854 100644 --- a/explorer/scan/local_scanner.go +++ b/explorer/scan/local_scanner.go @@ -113,18 +113,21 @@ func (s *LocalScanner) Run(ctx context.Context, job *Job) (*explorer.ReportColle // returns the upstream config for the job. If the job has a specified config, it has precedence // over the automatically detected one func (s *LocalScanner) getUpstreamConfig(inv *inventory.Inventory, incognito bool) (*upstream.UpstreamConfig, error) { - jobCreds := inv.GetSpec().GetUpstreamCredentials() - if s.upstream == nil && jobCreds == nil { - return nil, errors.New("no default or job upstream config provided") + var res *upstream.UpstreamConfig + if s.upstream != nil { + res = proto.Clone(s.upstream).(*upstream.UpstreamConfig) + } else { + res = &upstream.UpstreamConfig{} } - u := proto.Clone(s.upstream).(*upstream.UpstreamConfig) - u.Incognito = incognito + + jobCreds := inv.GetSpec().GetUpstreamCredentials() + res.Incognito = incognito if jobCreds != nil { - u.ApiEndpoint = jobCreds.GetApiEndpoint() - u.Creds = jobCreds - u.SpaceMrn = jobCreds.GetParentMrn() + res.ApiEndpoint = jobCreds.GetApiEndpoint() + res.Creds = jobCreds + res.SpaceMrn = jobCreds.GetParentMrn() } - return u, nil + return res, nil } func (s *LocalScanner) RunIncognito(ctx context.Context, job *Job) (*explorer.ReportCollection, error) { @@ -218,6 +221,24 @@ func (s *LocalScanner) distributeJob(job *Job, ctx context.Context, upstream *up } }() + if job.Bundle == nil && upstream != nil && upstream.Creds != nil { + client, err := upstream.InitClient() + if err != nil { + return nil, err + } + + services, err := explorer.NewRemoteServices(client.ApiEndpoint, client.Plugins, client.HttpClient) + if err != nil { + return nil, err + } + + bundle, err := services.GetBundle(ctx, &explorer.Mrn{Mrn: upstream.Creds.ParentMrn}) + if err != nil { + return nil, err + } + job.Bundle = bundle + } + // plan scan jobs reporter := NewAggregateReporter() // if we had asset errors we want to place them into the reporter @@ -430,13 +451,16 @@ func (s *localAssetScanner) prepareAsset() error { var hub explorer.QueryHub = s.services var conductor explorer.QueryConductor = s.services - // if we are using upstream we get the bundle from there - if s.job.UpstreamConfig != nil && !s.job.UpstreamConfig.Incognito { + // if we are using upstream we get the bundle from there. If we are in incognito mode, + // we should still use the upstream bundle but without reporting the results back + if !s.job.UpstreamConfig.Incognito { return nil } - if err := s.ensureBundle(); err != nil { - return err + if s.job.Bundle == nil { + if err := s.ensureBundle(); err != nil { + return err + } } if s.job.Bundle == nil { @@ -561,13 +585,16 @@ func (s *localAssetScanner) runQueryPack() (*AssetReport, error) { var conductor explorer.QueryConductor = s.services log.Debug().Str("asset", s.job.Asset.Mrn).Msg("client> request bundle for asset") - assetBundle, err := hub.GetBundle(s.job.Ctx, &explorer.Mrn{Mrn: s.job.Asset.Mrn}) - if err != nil { - return nil, err + // If we run in debug mode, download the asset bundle and dump it to disk + if val, ok := os.LookupEnv("DEBUG"); ok && (val == "1" || val == "true") { + assetBundle, err := hub.GetBundle(s.job.Ctx, &explorer.Mrn{Mrn: s.job.Asset.Mrn}) + if err != nil { + return nil, err + } + log.Debug().Msg("client> got bundle") + logger.TraceJSON(assetBundle) + logger.DebugDumpJSON("assetBundle", assetBundle) } - log.Debug().Msg("client> got bundle") - logger.TraceJSON(assetBundle) - logger.DebugDumpJSON("assetBundle", assetBundle) rawFilters, err := hub.GetFilters(s.job.Ctx, &explorer.Mrn{Mrn: s.job.Asset.Mrn}) if err != nil { @@ -626,8 +653,8 @@ func (s *localAssetScanner) runQueryPack() (*AssetReport, error) { } ar := &AssetReport{ - Mrn: s.job.Asset.Mrn, - Bundle: assetBundle, + Mrn: s.job.Asset.Mrn, + // Bundle: assetBundle, Resolved: resolvedPack, } From b66b9b7bc5a7870421c572e4bc53cdca9fa6c9f3 Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Thu, 7 Mar 2024 14:03:13 +0100 Subject: [PATCH 2/3] set bundle in reporter Signed-off-by: Ivan Milchev --- explorer/scan/local_scanner.go | 13 +++++++++---- explorer/scan/reporter.go | 7 +++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/explorer/scan/local_scanner.go b/explorer/scan/local_scanner.go index deee3f3854..eb3a01c27a 100644 --- a/explorer/scan/local_scanner.go +++ b/explorer/scan/local_scanner.go @@ -221,6 +221,8 @@ func (s *LocalScanner) distributeJob(job *Job, ctx context.Context, upstream *up } }() + // plan scan jobs + reporter := NewAggregateReporter() if job.Bundle == nil && upstream != nil && upstream.Creds != nil { client, err := upstream.InitClient() if err != nil { @@ -237,10 +239,9 @@ func (s *LocalScanner) distributeJob(job *Job, ctx context.Context, upstream *up return nil, err } job.Bundle = bundle + reporter.AddBundle(bundle) } - // plan scan jobs - reporter := NewAggregateReporter() // if we had asset errors we want to place them into the reporter for i := range discoveredAssets.Errors { reporter.AddScanError(discoveredAssets.Errors[i].Asset, discoveredAssets.Errors[i].Err) @@ -461,6 +462,11 @@ func (s *localAssetScanner) prepareAsset() error { if err := s.ensureBundle(); err != nil { return err } + + // add asset bundle to the reporter + if s.job.Reporter != nil && s.job.Bundle != nil { + s.job.Reporter.AddBundle(s.job.Bundle) + } } if s.job.Bundle == nil { @@ -653,8 +659,7 @@ func (s *localAssetScanner) runQueryPack() (*AssetReport, error) { } ar := &AssetReport{ - Mrn: s.job.Asset.Mrn, - // Bundle: assetBundle, + Mrn: s.job.Asset.Mrn, Resolved: resolvedPack, } diff --git a/explorer/scan/reporter.go b/explorer/scan/reporter.go index ff90e38222..a9244ddc1c 100644 --- a/explorer/scan/reporter.go +++ b/explorer/scan/reporter.go @@ -11,12 +11,12 @@ import ( type Reporter interface { AddReport(asset *inventory.Asset, results *AssetReport) + AddBundle(bundle *explorer.Bundle) AddScanError(asset *inventory.Asset, err error) } type AssetReport struct { Mrn string - Bundle *explorer.Bundle Report *explorer.Report Resolved *explorer.ResolvedPack } @@ -42,7 +42,10 @@ func (r *AggregateReporter) AddReport(asset *inventory.Asset, results *AssetRepo r.assets[asset.Mrn] = &explorer.Asset{Name: asset.Name, Mrn: asset.Mrn} r.assetReports[asset.Mrn] = results.Report r.resolved[asset.Mrn] = results.Resolved - r.bundle = results.Bundle +} + +func (r *AggregateReporter) AddBundle(bundle *explorer.Bundle) { + r.bundle = bundle } func (r *AggregateReporter) AddScanError(asset *inventory.Asset, err error) { From 938ed2b221c23f1cc1fe458a09b0147a2e682343 Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Thu, 7 Mar 2024 17:28:41 +0100 Subject: [PATCH 3/3] remove todo Signed-off-by: Ivan Milchev --- apps/cnquery/cmd/scan.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/apps/cnquery/cmd/scan.go b/apps/cnquery/cmd/scan.go index 0dfabcc349..5895051671 100644 --- a/apps/cnquery/cmd/scan.go +++ b/apps/cnquery/cmd/scan.go @@ -239,14 +239,6 @@ func getCobraScanConfig(cmd *cobra.Command, runtime *providers.Runtime, cliRes * serviceAccount := opts.GetServiceCredential() if serviceAccount != nil { - // TODO: determine if this needs migrating - // // determine information about the client - // sysInfo, err := sysinfo.GatherSystemInfo() - // if err != nil { - // log.Warn().Err(err).Msg("could not gather client information") - // } - // plugins = append(plugins, defaultRangerPlugins(sysInfo, opts.GetFeatures())...) - log.Info().Msg("using service account credentials") conf.runtime.UpstreamConfig = &upstream.UpstreamConfig{ SpaceMrn: opts.GetParentMrn(),