Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🐛 fix incognito scans #3518

Merged
merged 3 commits into from
Mar 7, 2024
Merged

🐛 fix incognito scans #3518

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1f4eb6d
fix incognito scans
imilchev Mar 7, 2024
b66b9b7
set bundle in reporter
imilchev Mar 7, 2024
938ed2b
remove todo
imilchev Mar 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 21 additions & 24 deletions apps/cnquery/cmd/scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,31 +237,28 @@ func getCobraScanConfig(cmd *cobra.Command, runtime *providers.Runtime, cliRes *
conf.Inventory.ApplyCategory(inventory.AssetCategory_CATEGORY_CICD)
}

var serviceAccount *upstream.ServiceAccountCredentials
if !conf.IsIncognito {
serviceAccount = opts.GetServiceCredential()
if serviceAccount != nil {
// TODO: determine if this needs migrating
// // determine information about the client
// sysInfo, err := sysinfo.GatherSystemInfo()
// if err != nil {
// log.Warn().Err(err).Msg("could not gather client information")
// }
// plugins = append(plugins, defaultRangerPlugins(sysInfo, opts.GetFeatures())...)

log.Info().Msg("using service account credentials")
conf.runtime.UpstreamConfig = &upstream.UpstreamConfig{
SpaceMrn: opts.GetParentMrn(),
ApiEndpoint: opts.UpstreamApiEndpoint(),
ApiProxy: opts.APIProxy,
Incognito: conf.IsIncognito,
Creds: serviceAccount,
}
providers.DefaultRuntime().UpstreamConfig = conf.runtime.UpstreamConfig
} else {
log.Warn().Msg("No credentials provided. Switching to --incognito mode.")
conf.IsIncognito = true
serviceAccount := opts.GetServiceCredential()
if serviceAccount != nil {
// TODO: determine if this needs migrating
imilchev marked this conversation as resolved.
Show resolved Hide resolved
// // determine information about the client
// sysInfo, err := sysinfo.GatherSystemInfo()
// if err != nil {
// log.Warn().Err(err).Msg("could not gather client information")
// }
// plugins = append(plugins, defaultRangerPlugins(sysInfo, opts.GetFeatures())...)

log.Info().Msg("using service account credentials")
conf.runtime.UpstreamConfig = &upstream.UpstreamConfig{
SpaceMrn: opts.GetParentMrn(),
ApiEndpoint: opts.UpstreamApiEndpoint(),
ApiProxy: opts.APIProxy,
Incognito: conf.IsIncognito,
Creds: serviceAccount,
}
providers.DefaultRuntime().UpstreamConfig = conf.runtime.UpstreamConfig
} else {
log.Warn().Msg("No credentials provided. Switching to --incognito mode.")
conf.IsIncognito = true
}

if len(conf.QueryPackPaths) > 0 && !conf.IsIncognito {
Expand Down
72 changes: 52 additions & 20 deletions explorer/scan/local_scanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,18 +113,21 @@ func (s *LocalScanner) Run(ctx context.Context, job *Job) (*explorer.ReportColle
// returns the upstream config for the job. If the job has a specified config, it has precedence
// over the automatically detected one
func (s *LocalScanner) getUpstreamConfig(inv *inventory.Inventory, incognito bool) (*upstream.UpstreamConfig, error) {
jobCreds := inv.GetSpec().GetUpstreamCredentials()
if s.upstream == nil && jobCreds == nil {
return nil, errors.New("no default or job upstream config provided")
var res *upstream.UpstreamConfig
if s.upstream != nil {
res = proto.Clone(s.upstream).(*upstream.UpstreamConfig)
} else {
res = &upstream.UpstreamConfig{}
}
u := proto.Clone(s.upstream).(*upstream.UpstreamConfig)
u.Incognito = incognito

jobCreds := inv.GetSpec().GetUpstreamCredentials()
res.Incognito = incognito
if jobCreds != nil {
u.ApiEndpoint = jobCreds.GetApiEndpoint()
u.Creds = jobCreds
u.SpaceMrn = jobCreds.GetParentMrn()
res.ApiEndpoint = jobCreds.GetApiEndpoint()
res.Creds = jobCreds
res.SpaceMrn = jobCreds.GetParentMrn()
}
return u, nil
return res, nil
}

func (s *LocalScanner) RunIncognito(ctx context.Context, job *Job) (*explorer.ReportCollection, error) {
Expand Down Expand Up @@ -220,6 +223,25 @@ func (s *LocalScanner) distributeJob(job *Job, ctx context.Context, upstream *up

// plan scan jobs
reporter := NewAggregateReporter()
if job.Bundle == nil && upstream != nil && upstream.Creds != nil {
client, err := upstream.InitClient()
if err != nil {
return nil, err
}

services, err := explorer.NewRemoteServices(client.ApiEndpoint, client.Plugins, client.HttpClient)
if err != nil {
return nil, err
}

bundle, err := services.GetBundle(ctx, &explorer.Mrn{Mrn: upstream.Creds.ParentMrn})
if err != nil {
return nil, err
}
job.Bundle = bundle
reporter.AddBundle(bundle)
}

// if we had asset errors we want to place them into the reporter
for i := range discoveredAssets.Errors {
reporter.AddScanError(discoveredAssets.Errors[i].Asset, discoveredAssets.Errors[i].Err)
Expand Down Expand Up @@ -430,13 +452,21 @@ func (s *localAssetScanner) prepareAsset() error {
var hub explorer.QueryHub = s.services
var conductor explorer.QueryConductor = s.services

// if we are using upstream we get the bundle from there
if s.job.UpstreamConfig != nil && !s.job.UpstreamConfig.Incognito {
// if we are using upstream we get the bundle from there. If we are in incognito mode,
// we should still use the upstream bundle but without reporting the results back
if !s.job.UpstreamConfig.Incognito {
return nil
}

if err := s.ensureBundle(); err != nil {
return err
if s.job.Bundle == nil {
if err := s.ensureBundle(); err != nil {
return err
}

// add asset bundle to the reporter
if s.job.Reporter != nil && s.job.Bundle != nil {
s.job.Reporter.AddBundle(s.job.Bundle)
}
}

if s.job.Bundle == nil {
Expand Down Expand Up @@ -561,13 +591,16 @@ func (s *localAssetScanner) runQueryPack() (*AssetReport, error) {
var conductor explorer.QueryConductor = s.services

log.Debug().Str("asset", s.job.Asset.Mrn).Msg("client> request bundle for asset")
assetBundle, err := hub.GetBundle(s.job.Ctx, &explorer.Mrn{Mrn: s.job.Asset.Mrn})
if err != nil {
return nil, err
// If we run in debug mode, download the asset bundle and dump it to disk
if val, ok := os.LookupEnv("DEBUG"); ok && (val == "1" || val == "true") {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we not use the same logic as in here

cnquery/logger/debug.go

Lines 44 to 46 in 1b98d7f

if !log.Debug().Enabled() {
return
}
?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we only dump this if the env var DEBUG=1 is set. I think log.Debug().Enabled() is also true when we just have --log-level debug. I don't think we want to have these files in that case.

That same condition exists in cnspec for dumping the bundle there too. I just used the same approach

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

temporarily ok, but we need to extract this into a function. Otherwise it will be hard to change that in the future https://github.com/mondoohq/cnquery/blob/main/logger/debug.go#L77

assetBundle, err := hub.GetBundle(s.job.Ctx, &explorer.Mrn{Mrn: s.job.Asset.Mrn})
if err != nil {
return nil, err
}
log.Debug().Msg("client> got bundle")
logger.TraceJSON(assetBundle)
logger.DebugDumpJSON("assetBundle", assetBundle)
}
log.Debug().Msg("client> got bundle")
logger.TraceJSON(assetBundle)
logger.DebugDumpJSON("assetBundle", assetBundle)

rawFilters, err := hub.GetFilters(s.job.Ctx, &explorer.Mrn{Mrn: s.job.Asset.Mrn})
if err != nil {
Expand Down Expand Up @@ -627,7 +660,6 @@ func (s *localAssetScanner) runQueryPack() (*AssetReport, error) {

ar := &AssetReport{
Mrn: s.job.Asset.Mrn,
Bundle: assetBundle,
Resolved: resolvedPack,
}

Expand Down
7 changes: 5 additions & 2 deletions explorer/scan/reporter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ import (

type Reporter interface {
AddReport(asset *inventory.Asset, results *AssetReport)
AddBundle(bundle *explorer.Bundle)
AddScanError(asset *inventory.Asset, err error)
}

type AssetReport struct {
Mrn string
Bundle *explorer.Bundle
Report *explorer.Report
Resolved *explorer.ResolvedPack
}
Expand All @@ -42,7 +42,10 @@ func (r *AggregateReporter) AddReport(asset *inventory.Asset, results *AssetRepo
r.assets[asset.Mrn] = &explorer.Asset{Name: asset.Name, Mrn: asset.Mrn}
r.assetReports[asset.Mrn] = results.Report
r.resolved[asset.Mrn] = results.Resolved
r.bundle = results.Bundle
}

func (r *AggregateReporter) AddBundle(bundle *explorer.Bundle) {
r.bundle = bundle
}

func (r *AggregateReporter) AddScanError(asset *inventory.Asset, err error) {
Expand Down
Loading