From 78f4b85311462e8f25aa79c32aa26310d38dfe00 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:49:14 +0100 Subject: [PATCH] pkg/cwhub - refact Downloader (#3382) * pkg/cwhub - refact Downloader * single method interfaces * lint --- cmd/crowdsec-cli/clicapi/capi.go | 2 +- cmd/crowdsec-cli/cliconsole/console.go | 2 +- cmd/crowdsec-cli/clihub/hub.go | 20 +-- cmd/crowdsec-cli/cliitem/item.go | 53 ++++---- cmd/crowdsec-cli/cliitem/suggest.go | 4 +- cmd/crowdsec-cli/clilapi/context.go | 6 +- cmd/crowdsec-cli/clilapi/status.go | 2 +- cmd/crowdsec-cli/clisetup/setup.go | 8 +- cmd/crowdsec-cli/clisimulation/simulation.go | 2 +- cmd/crowdsec-cli/clisupport/support.go | 4 +- cmd/crowdsec-cli/config_backup.go | 2 +- cmd/crowdsec-cli/config_restore.go | 6 +- cmd/crowdsec-cli/require/require.go | 9 +- cmd/crowdsec/serve.go | 4 +- go.mod | 2 +- go.sum | 6 +- pkg/cwhub/cwhub_test.go | 16 +-- pkg/cwhub/doc.go | 11 +- pkg/cwhub/download.go | 124 +++++++++++++++++++ pkg/cwhub/download_test.go | 50 ++++++++ pkg/cwhub/errors.go | 19 --- pkg/cwhub/fetch.go | 44 +------ pkg/cwhub/hub.go | 11 +- pkg/cwhub/hub_test.go | 35 +++--- pkg/cwhub/itemupgrade_test.go | 8 +- pkg/cwhub/remote.go | 87 ------------- pkg/hubops/download.go | 16 +-- pkg/hubtest/hubtest.go | 8 +- pkg/hubtest/hubtest_item.go | 2 +- pkg/leakybucket/buckets_test.go | 3 +- pkg/setup/install.go | 14 ++- 31 files changed, 307 insertions(+), 273 deletions(-) create mode 100644 pkg/cwhub/download.go create mode 100644 pkg/cwhub/download_test.go delete mode 100644 pkg/cwhub/errors.go delete mode 100644 pkg/cwhub/remote.go diff --git a/cmd/crowdsec-cli/clicapi/capi.go b/cmd/crowdsec-cli/clicapi/capi.go index 2cce01c7d3e..120acca8b59 100644 --- a/cmd/crowdsec-cli/clicapi/capi.go +++ b/cmd/crowdsec-cli/clicapi/capi.go @@ -261,7 +261,7 @@ func (cli *cliCapi) newStatusCmd() *cobra.Command { Args: cobra.MinimumNArgs(0), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { - hub, err := require.Hub(cli.cfg(), nil, nil) + hub, err := require.Hub(cli.cfg(), nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/cliconsole/console.go b/cmd/crowdsec-cli/cliconsole/console.go index 448ddcee7fa..dbbe2487cd4 100644 --- a/cmd/crowdsec-cli/cliconsole/console.go +++ b/cmd/crowdsec-cli/cliconsole/console.go @@ -114,7 +114,7 @@ func (cli *cliConsole) enroll(ctx context.Context, key string, name string, over } } - hub, err := require.Hub(cfg, nil, nil) + hub, err := require.Hub(cfg, nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/clihub/hub.go b/cmd/crowdsec-cli/clihub/hub.go index 49ccd761285..9571b3d866d 100644 --- a/cmd/crowdsec-cli/clihub/hub.go +++ b/cmd/crowdsec-cli/clihub/hub.go @@ -93,7 +93,7 @@ func (cli *cliHub) newListCmd() *cobra.Command { Args: cobra.NoArgs, DisableAutoGenTag: true, RunE: func(_ *cobra.Command, _ []string) error { - hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger()) + hub, err := require.Hub(cli.cfg(), log.StandardLogger()) if err != nil { return err } @@ -110,15 +110,15 @@ func (cli *cliHub) newListCmd() *cobra.Command { func (cli *cliHub) update(ctx context.Context, withContent bool) error { local := cli.cfg().Hub - remote := require.RemoteHub(ctx, cli.cfg()) - // don't use require.Hub because if there is no index file, it would fail - hub, err := cwhub.NewHub(local, remote, log.StandardLogger()) + hub, err := cwhub.NewHub(local, log.StandardLogger()) if err != nil { return err } - if err := hub.Update(ctx, withContent); err != nil { + indexProvider := require.HubDownloader(ctx, cli.cfg()) + + if err := hub.Update(ctx, indexProvider, withContent); err != nil { return fmt.Errorf("failed to update hub: %w", err) } @@ -166,16 +166,18 @@ cscli hub update --with-content`, func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force bool) error { cfg := cli.cfg() - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) + hub, err := require.Hub(cfg, log.StandardLogger()) if err != nil { return err } plan := hubops.NewActionPlan(hub) + contentProvider := require.HubDownloader(ctx, cfg) + for _, itemType := range cwhub.ItemTypes { for _, item := range hub.GetInstalledByType(itemType, true) { - plan.AddCommand(hubops.NewDownloadCommand(item, force)) + plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)) } } @@ -196,9 +198,9 @@ func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force boo func (cli *cliHub) newUpgradeCmd() *cobra.Command { var ( - yes bool + yes bool dryRun bool - force bool + force bool ) cmd := &cobra.Command{ diff --git a/cmd/crowdsec-cli/cliitem/item.go b/cmd/crowdsec-cli/cliitem/item.go index 05e52d18dd3..637bd3023cf 100644 --- a/cmd/crowdsec-cli/cliitem/item.go +++ b/cmd/crowdsec-cli/cliitem/item.go @@ -71,13 +71,15 @@ func (cli cliItem) NewCommand() *cobra.Command { func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun bool, downloadOnly bool, force bool, ignoreError bool) error { cfg := cli.cfg() - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) + hub, err := require.Hub(cfg, log.StandardLogger()) if err != nil { return err } plan := hubops.NewActionPlan(hub) + contentProvider := require.HubDownloader(ctx, cfg) + for _, name := range args { item := hub.GetItem(cli.name, name) if item == nil { @@ -91,7 +93,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun continue } - if err = plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil { + if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { return err } @@ -121,7 +123,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun func (cli cliItem) newInstallCmd() *cobra.Command { var ( - yes bool + yes bool dryRun bool downloadOnly bool force bool @@ -180,6 +182,7 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil { return nil, err } + if purge { if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil { return nil, err @@ -211,12 +214,11 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil { return nil, err - } + if purge { if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil { return nil, err - } } } @@ -224,11 +226,10 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b return plan, nil } - func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun bool, purge bool, force bool, all bool) error { cfg := cli.cfg() - hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger()) + hub, err := require.Hub(cli.cfg(), log.StandardLogger()) if err != nil { return err } @@ -253,7 +254,7 @@ func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun b func (cli cliItem) newRemoveCmd() *cobra.Command { var ( - yes bool + yes bool dryRun bool purge bool force bool @@ -290,12 +291,12 @@ func (cli cliItem) newRemoveCmd() *cobra.Command { return cmd } -func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bool) (*hubops.ActionPlan, error) { +func (cli cliItem) upgradePlan(hub *cwhub.Hub, contentProvider cwhub.ContentProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) { plan := hubops.NewActionPlan(hub) if all { for _, item := range hub.GetInstalledByType(cli.name, true) { - if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil { + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { return nil, err } } @@ -313,7 +314,7 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bo return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name) } - if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil { + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { return nil, err } } @@ -324,12 +325,14 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bo func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun bool, force bool, all bool) error { cfg := cli.cfg() - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) + hub, err := require.Hub(cfg, log.StandardLogger()) if err != nil { return err } - plan, err := cli.upgradePlan(hub, args, force, all) + contentProvider := require.HubDownloader(ctx, cfg) + + plan, err := cli.upgradePlan(hub, contentProvider, args, force, all) if err != nil { return err } @@ -349,10 +352,10 @@ func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun func (cli cliItem) newUpgradeCmd() *cobra.Command { var ( - yes bool + yes bool dryRun bool - all bool - force bool + all bool + force bool ) cmd := &cobra.Command{ @@ -390,13 +393,13 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff cfg.Cscli.PrometheusUrl = url } - remote := (*cwhub.RemoteHubCfg)(nil) + var contentProvider cwhub.ContentProvider if diff { - remote = require.RemoteHub(ctx, cfg) + contentProvider = require.HubDownloader(ctx, cfg) } - hub, err := require.Hub(cfg, remote, log.StandardLogger()) + hub, err := require.Hub(cfg, log.StandardLogger()) if err != nil { return err } @@ -408,7 +411,7 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff } if diff { - fmt.Println(cli.whyTainted(ctx, hub, item, rev)) + fmt.Println(cli.whyTainted(ctx, hub, contentProvider, item, rev)) continue } @@ -462,7 +465,7 @@ func (cli cliItem) newInspectCmd() *cobra.Command { func (cli cliItem) list(args []string, all bool) error { cfg := cli.cfg() - hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger()) + hub, err := require.Hub(cli.cfg(), log.StandardLogger()) if err != nil { return err } @@ -498,7 +501,7 @@ func (cli cliItem) newListCmd() *cobra.Command { } // return the diff between the installed version and the latest version -func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool) (string, error) { +func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, contentProvider cwhub.ContentProvider, reverse bool) (string, error) { if !item.State.Installed { return "", fmt.Errorf("'%s' is not installed", item.FQName()) } @@ -509,7 +512,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool) } defer os.Remove(dest.Name()) - _, remoteURL, err := item.FetchContentTo(ctx, dest.Name()) + _, remoteURL, err := item.FetchContentTo(ctx, contentProvider, dest.Name()) if err != nil { return "", err } @@ -540,7 +543,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool) return fmt.Sprintf("%s", diff), nil } -func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, item *cwhub.Item, reverse bool) string { +func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, item *cwhub.Item, reverse bool) string { if !item.State.Installed { return fmt.Sprintf("# %s is not installed", item.FQName()) } @@ -565,7 +568,7 @@ func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, item *cwhub.I ret = append(ret, err.Error()) } - diff, err := cli.itemDiff(ctx, sub, reverse) + diff, err := cli.itemDiff(ctx, sub, contentProvider, reverse) if err != nil { ret = append(ret, err.Error()) } diff --git a/cmd/crowdsec-cli/cliitem/suggest.go b/cmd/crowdsec-cli/cliitem/suggest.go index 5b080722af9..b0f19b6993c 100644 --- a/cmd/crowdsec-cli/cliitem/suggest.go +++ b/cmd/crowdsec-cli/cliitem/suggest.go @@ -37,7 +37,7 @@ func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) str } func compAllItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) { - hub, err := require.Hub(cfg(), nil, nil) + hub, err := require.Hub(cfg(), nil) if err != nil { return nil, cobra.ShellCompDirectiveDefault } @@ -56,7 +56,7 @@ func compAllItems(itemType string, args []string, toComplete string, cfg configG } func compInstalledItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) { - hub, err := require.Hub(cfg(), nil, nil) + hub, err := require.Hub(cfg(), nil) if err != nil { return nil, cobra.ShellCompDirectiveDefault } diff --git a/cmd/crowdsec-cli/clilapi/context.go b/cmd/crowdsec-cli/clilapi/context.go index 20ceb2b9596..0730ba2b2a9 100644 --- a/cmd/crowdsec-cli/clilapi/context.go +++ b/cmd/crowdsec-cli/clilapi/context.go @@ -59,7 +59,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user `, DisableAutoGenTag: true, RunE: func(_ *cobra.Command, _ []string) error { - hub, err := require.Hub(cli.cfg(), nil, nil) + hub, err := require.Hub(cli.cfg(), nil) if err != nil { return err } @@ -101,7 +101,7 @@ func (cli *cliLapi) newContextStatusCmd() *cobra.Command { DisableAutoGenTag: true, RunE: func(_ *cobra.Command, _ []string) error { cfg := cli.cfg() - hub, err := require.Hub(cfg, nil, nil) + hub, err := require.Hub(cfg, nil) if err != nil { return err } @@ -153,7 +153,7 @@ cscli lapi context detect crowdsecurity/sshd-logs return fmt.Errorf("failed to init expr helpers: %w", err) } - hub, err := require.Hub(cfg, nil, nil) + hub, err := require.Hub(cfg, nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/clilapi/status.go b/cmd/crowdsec-cli/clilapi/status.go index 6ff88834602..039c75e585d 100644 --- a/cmd/crowdsec-cli/clilapi/status.go +++ b/cmd/crowdsec-cli/clilapi/status.go @@ -102,7 +102,7 @@ func (cli *cliLapi) newStatusCmd() *cobra.Command { Args: cobra.MinimumNArgs(0), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { - hub, err := require.Hub(cli.cfg(), nil, nil) + hub, err := require.Hub(cli.cfg(), nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/clisetup/setup.go b/cmd/crowdsec-cli/clisetup/setup.go index 4cb423e484c..77c357e7251 100644 --- a/cmd/crowdsec-cli/clisetup/setup.go +++ b/cmd/crowdsec-cli/clisetup/setup.go @@ -95,7 +95,7 @@ func (cli *cliSetup) newDetectCmd() *cobra.Command { func (cli *cliSetup) newInstallHubCmd() *cobra.Command { var ( - yes bool + yes bool dryRun bool ) @@ -289,14 +289,16 @@ func (cli *cliSetup) install(ctx context.Context, yes bool, dryRun bool, fromFil cfg := cli.cfg() - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) + hub, err := require.Hub(cfg, log.StandardLogger()) if err != nil { return err } verbose := (cfg.Cscli.Output == "raw") - return setup.InstallHubItems(ctx, hub, input, yes, dryRun, verbose) + contentProvider := require.HubDownloader(ctx, cfg) + + return setup.InstallHubItems(ctx, hub, contentProvider, input, yes, dryRun, verbose) } func (cli *cliSetup) validate(fromFile string) error { diff --git a/cmd/crowdsec-cli/clisimulation/simulation.go b/cmd/crowdsec-cli/clisimulation/simulation.go index 8136aa213c3..c06db56f200 100644 --- a/cmd/crowdsec-cli/clisimulation/simulation.go +++ b/cmd/crowdsec-cli/clisimulation/simulation.go @@ -71,7 +71,7 @@ func (cli *cliSimulation) newEnableCmd() *cobra.Command { Example: `cscli simulation enable`, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - hub, err := require.Hub(cli.cfg(), nil, nil) + hub, err := require.Hub(cli.cfg(), nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/clisupport/support.go b/cmd/crowdsec-cli/clisupport/support.go index a21d00e3b20..5f6032a17bd 100644 --- a/cmd/crowdsec-cli/clisupport/support.go +++ b/cmd/crowdsec-cli/clisupport/support.go @@ -491,9 +491,9 @@ func (cli *cliSupport) dump(ctx context.Context, outFile string) error { skipAgent = true } - hub, err := require.Hub(cfg, nil, nil) + hub, err := require.Hub(cfg, nil) if err != nil { - log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected") + log.Warn("Could not init hub, running on LAPI? Hub related information will not be collected") // XXX: lapi status check requires scenarios, will return an error } diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index d23aff80a78..faac786ebdc 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -15,7 +15,7 @@ import ( ) func (cli *cliConfig) backupHub(dirPath string) error { - hub, err := require.Hub(cli.cfg(), nil, nil) + hub, err := require.Hub(cli.cfg(), nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 8884fa448d2..b5fbf36b2b4 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -18,11 +18,13 @@ import ( func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error { cfg := cli.cfg() - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), nil) + hub, err := require.Hub(cfg, nil) if err != nil { return err } + contentProvider := require.HubDownloader(ctx, cfg) + for _, itype := range cwhub.ItemTypes { itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) if _, err = os.Stat(itemDirectory); err != nil { @@ -53,7 +55,7 @@ func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error { plan := hubops.NewActionPlan(hub) - if err = plan.AddCommand(hubops.NewDownloadCommand(item, false)); err != nil { + if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, false)); err != nil { return err } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index 7b3410021c1..a44e76ae47d 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -82,12 +82,11 @@ func Notifications(c *csconfig.Config) error { return nil } -// RemoteHub returns the configuration required to download hub index and items: url, branch, etc. -func RemoteHub(ctx context.Context, c *csconfig.Config) *cwhub.RemoteHubCfg { +func HubDownloader(ctx context.Context, c *csconfig.Config) *cwhub.Downloader { // set branch in config, and log if necessary branch := HubBranch(ctx, c) urlTemplate := HubURLTemplate(c) - remote := &cwhub.RemoteHubCfg{ + remote := &cwhub.Downloader{ Branch: branch, URLTemplate: urlTemplate, IndexPath: ".index.json", @@ -98,7 +97,7 @@ func RemoteHub(ctx context.Context, c *csconfig.Config) *cwhub.RemoteHubCfg { // Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items. // If no remote parameter is provided, the hub can only be used for local operations. -func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) { +func Hub(c *csconfig.Config, logger *logrus.Logger) (*cwhub.Hub, error) { local := c.Hub if local == nil { @@ -110,7 +109,7 @@ func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) logger.SetOutput(io.Discard) } - hub, err := cwhub.NewHub(local, remote, logger) + hub, err := cwhub.NewHub(local, logger) if err != nil { return nil, err } diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 14602c425fe..62b721befdb 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -85,7 +85,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } if !cConfig.DisableAgent { - hub, err := cwhub.NewHub(cConfig.Hub, nil, log.StandardLogger()) + hub, err := cwhub.NewHub(cConfig.Hub, log.StandardLogger()) if err != nil { return nil, err } @@ -387,7 +387,7 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error { } if !cConfig.DisableAgent { - hub, err := cwhub.NewHub(cConfig.Hub, nil, log.StandardLogger()) + hub, err := cwhub.NewHub(cConfig.Hub, log.StandardLogger()) if err != nil { return err } diff --git a/go.mod b/go.mod index e437bbd688a..aa723b38409 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/creack/pty v1.1.21 // indirect github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f + github.com/crowdsecurity/go-cs-lib v0.0.16 github.com/crowdsecurity/grokky v0.2.2 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index d092956d0a8..e7f181d7d24 100644 --- a/go.sum +++ b/go.sum @@ -107,10 +107,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 h1:hyrYw3h github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607/go.mod h1:br36fEqurGYZQGit+iDYsIzW0FF6VufMbDzyyLxEuPA= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241203101722-e557f9809413 h1:VIedap4s3mXM4+tM2NMm7R3E/kn79ayLZaLHDqPYVCc= -github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241203101722-e557f9809413/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g= -github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f h1:Pd+O4UK78uQtTqbvYX+nHvqZ7TffD51uC4q0RE/podk= -github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g= +github.com/crowdsecurity/go-cs-lib v0.0.16 h1:2/htodjwc/sfsv4deX8F/2Fzg1bOI8w3O1/BPSvvsB0= +github.com/crowdsecurity/go-cs-lib v0.0.16/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g= github.com/crowdsecurity/grokky v0.2.2 h1:yALsI9zqpDArYzmSSxfBq2dhYuGUTKMJq8KOEIAsuo4= github.com/crowdsecurity/grokky v0.2.2/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 1b5dee34dd3..94a1d6ef6fd 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -57,18 +57,18 @@ func testHub(t *testing.T, update bool) *Hub { os.RemoveAll(tmpDir) }) - remote := &RemoteHubCfg{ - Branch: "master", - URLTemplate: mockURLTemplate, - IndexPath: ".index.json", - } - - hub, err := NewHub(local, remote, log.StandardLogger()) + hub, err := NewHub(local, log.StandardLogger()) require.NoError(t, err) if update { + indexProvider := &Downloader{ + Branch: "master", + URLTemplate: mockURLTemplate, + IndexPath: ".index.json", + } + ctx := context.Background() - err := hub.Update(ctx, false) + err := hub.Update(ctx, indexProvider, false) require.NoError(t, err) } diff --git a/pkg/cwhub/doc.go b/pkg/cwhub/doc.go index a1ee9d37ee7..b85d7634da4 100644 --- a/pkg/cwhub/doc.go +++ b/pkg/cwhub/doc.go @@ -85,25 +85,20 @@ // return fmt.Errorf("collection not found") // } // -// To provide the remote hub configuration, use the second parameter of NewHub(): +// Some commands require an object to provide the hub index, or contents: // -// remoteHub := cwhub.RemoteHubCfg{ +// indexProvider := cwhub.Downloader{ // URLTemplate: "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s", // Branch: "master", // IndexPath: ".index.json", // } // -// hub, err := cwhub.NewHub(localHub, remoteHub, logger) -// if err != nil { -// return fmt.Errorf("unable to initialize hub: %w", err) -// } -// // The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two // placeholders: the branch and the file path (it will be an index or an item). // // Before calling hub.Load(), you can update the index file by calling the Update() method: // -// err := hub.Update(context.Background()) +// err := hub.Update(context.Background(), indexProvider) // if err != nil { // return fmt.Errorf("unable to update hub index: %w", err) // } diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go new file mode 100644 index 00000000000..48cb2382668 --- /dev/null +++ b/pkg/cwhub/download.go @@ -0,0 +1,124 @@ +package cwhub + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + + "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/go-cs-lib/downloader" +) + +// Downloader is used to retrieve index and items from a remote hub, with cache control. +type Downloader struct { + Branch string + URLTemplate string + IndexPath string +} + +// IndexProvider retrieves and writes .index.json +type IndexProvider interface { + FetchIndex(ctx context.Context, indexFile string, withContent bool, logger *logrus.Logger) (bool, error) +} + +// ContentProvider retrieves and writes the YAML files with the item content. +type ContentProvider interface { + FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error) +} + +// urlTo builds the URL to download a file from the remote hub. +func (d *Downloader) urlTo(remotePath string) (string, error) { + // the template must contain two string placeholders + if fmt.Sprintf(d.URLTemplate, "%s", "%s") != d.URLTemplate { + return "", fmt.Errorf("invalid URL template '%s'", d.URLTemplate) + } + + return fmt.Sprintf(d.URLTemplate, d.Branch, remotePath), nil +} + +// addURLParam adds a parameter with a value (ex. "with_content=true") to the URL if it's not already present. +func addURLParam(rawURL string, param string, value string) (string, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + query := parsedURL.Query() + + if _, exists := query[param]; !exists { + query.Add(param, value) + } + + parsedURL.RawQuery = query.Encode() + + return parsedURL.String(), nil +} + +// FetchIndex downloads the index from the hub and writes it to the filesystem. +// It uses a temporary file to avoid partial downloads, and won't overwrite the original +// if it has not changed. +func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withContent bool, logger *logrus.Logger) (bool, error) { + url, err := d.urlTo(d.IndexPath) + if err != nil { + return false, fmt.Errorf("failed to build hub index request: %w", err) + } + + if withContent { + url, err = addURLParam(url, "with_content", "true") + if err != nil { + return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err) + } + } + + downloaded, err := downloader. + New(). + WithHTTPClient(HubClient). + ToFile(destPath). + WithETagFn(downloader.SHA256). + CompareContent(). + WithLogger(logger.WithField("url", url)). + BeforeRequest(func(_ *http.Request) { + fmt.Println("Downloading " + destPath) + }). + Download(ctx, url) + if err != nil { + return false, err + } + + return downloaded, nil +} + +// FetchContent downloads the content to the specified path, through a temporary file +// to avoid partial downloads. +// If the hash does not match, it will not overwrite and log a warning. +func (d *Downloader) FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error) { + url, err := d.urlTo(remotePath) + if err != nil { + return false, "", fmt.Errorf("failed to build request: %w", err) + } + + downloaded, err := downloader. + New(). + WithHTTPClient(HubClient). + ToFile(destPath). + WithETagFn(downloader.SHA256). + WithMakeDirs(true). + WithLogger(logger.WithField("url", url)). + CompareContent(). + VerifyHash("sha256", wantHash). + Download(ctx, url) + + var hasherr downloader.HashMismatchError + + switch { + case errors.As(err, &hasherr): + logger.Warnf("%s. The index file is outdated, please run 'cscli hub update' and try again", err.Error()) + case err != nil: + return false, "", err + } + + return downloaded, url, nil +} diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go new file mode 100644 index 00000000000..ec07862abcf --- /dev/null +++ b/pkg/cwhub/download_test.go @@ -0,0 +1,50 @@ +package cwhub + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetchIndex(t *testing.T) { + ctx := context.Background() + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("with_content") == "true" { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`Hi I'm an index with content`)) + } else { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`Hi I'm a regular index`)) + } + })) + defer mockServer.Close() + + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", + IndexPath: "index.txt", + } + + logger := logrus.New() + logger.Out = io.Discard + + destPath := filepath.Join(t.TempDir(), "index.txt") + withContent := true + + downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, logger) + require.NoError(t, err) + assert.True(t, downloaded) + + content, err := os.ReadFile(destPath) + require.NoError(t, err) + assert.Equal(t, "Hi I'm an index with content", string(content)) +} diff --git a/pkg/cwhub/errors.go b/pkg/cwhub/errors.go deleted file mode 100644 index b0be444fcba..00000000000 --- a/pkg/cwhub/errors.go +++ /dev/null @@ -1,19 +0,0 @@ -package cwhub - -import ( - "errors" - "fmt" -) - -// ErrNilRemoteHub is returned when trying to download with a local-only configuration. -var ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers") - -// IndexNotFoundError is returned when the remote hub index is not found. -type IndexNotFoundError struct { - URL string - Branch string -} - -func (e IndexNotFoundError) Error() string { - return fmt.Sprintf("index not found at %s, branch '%s'. Please check the .cscli.hub_branch value if you specified it in config.yaml, or use 'master' if not sure", e.URL, e.Branch) -} diff --git a/pkg/cwhub/fetch.go b/pkg/cwhub/fetch.go index 92198e63ef1..dd1a520d7e2 100644 --- a/pkg/cwhub/fetch.go +++ b/pkg/cwhub/fetch.go @@ -1,24 +1,15 @@ package cwhub -// Install, upgrade and remove items from the hub to the local configuration - import ( "context" "crypto" "encoding/base64" "encoding/hex" - "errors" "fmt" "os" "path/filepath" - - "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/go-cs-lib/downloader" - ) - // writeEmbeddedContentTo writes the embedded content to the specified path and checks the hash. // If the content is base64 encoded, it will be decoded before writing. Check for item.Content // before calling this method. @@ -56,40 +47,9 @@ func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error { return nil } -// writeRemoteContentTo downloads the content to the specified path and checks the hash. -func (i *Item) writeRemoteContentTo(ctx context.Context, destPath, wantHash string) (bool, string, error) { - url, err := i.hub.remote.urlTo(i.RemotePath) - if err != nil { - return false, "", fmt.Errorf("failed to build request: %w", err) - } - - d := downloader. - New(). - WithHTTPClient(HubClient). - ToFile(destPath). - WithETagFn(downloader.SHA256). - WithMakeDirs(true). - WithLogger(logrus.WithField("url", url)). - CompareContent(). - VerifyHash("sha256", wantHash) - - hasherr := downloader.HashMismatchError{} - - downloaded, err := d.Download(ctx, url) - - switch { - case errors.As(err, &hasherr): - i.hub.logger.Warnf("%s. The index file is outdated, please run 'cscli hub update' and try again", err.Error()) - case err != nil: - return false, "", err - } - - return downloaded, url, nil -} - // FetchContentTo writes the last version of the item's YAML file to the specified path. // Returns whether the file was downloaded, and the remote url for feedback purposes. -func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, string, error) { +func (i *Item) FetchContentTo(ctx context.Context, contentProvider ContentProvider, destPath string) (bool, string, error) { wantHash := i.latestHash() if wantHash == "" { return false, "", fmt.Errorf("%s: latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again", i.FQName()) @@ -104,5 +64,5 @@ func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, strin return true, fmt.Sprintf("(embedded in %s)", i.hub.local.HubIndexFile), nil } - return i.writeRemoteContentTo(ctx, destPath, wantHash) + return contentProvider.FetchContent(ctx, i.RemotePath, destPath, wantHash, i.hub.logger) } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 55469fed711..3722ceaafcd 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -22,7 +22,6 @@ type Hub struct { items HubItems // Items read from HubDir and InstallDir pathIndex map[string]*Item local *csconfig.LocalHubCfg - remote *RemoteHubCfg logger *logrus.Logger Warnings []string // Warnings encountered during sync } @@ -35,8 +34,7 @@ func (h *Hub) GetDataDir() string { // NewHub returns a new Hub instance with local and (optionally) remote configuration. // The hub is not synced automatically. Load() must be called to read the index, sync the local state, // and check for unmanaged items. -// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set. -func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, logger *logrus.Logger) (*Hub, error) { +func NewHub(local *csconfig.LocalHubCfg, logger *logrus.Logger) (*Hub, error) { if local == nil { return nil, errors.New("no hub configuration found") } @@ -48,7 +46,6 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, logger *logrus.Lo hub := &Hub{ local: local, - remote: remote, logger: logger, pathIndex: make(map[string]*Item, 0), } @@ -158,13 +155,13 @@ func (h *Hub) ItemStats() []string { // Update downloads the latest version of the index and writes it to disk if it changed. // It cannot be called after Load() unless the hub is completely empty. -func (h *Hub) Update(ctx context.Context, withContent bool) error { +func (h *Hub) Update(ctx context.Context, indexProvider IndexProvider, withContent bool) error { if len(h.pathIndex) > 0 { // if this happens, it's a bug. return errors.New("cannot update hub after items have been loaded") } - downloaded, err := h.remote.fetchIndex(ctx, h.local.HubIndexFile, withContent) + downloaded, err := indexProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger) if err != nil { return err } @@ -238,6 +235,7 @@ func (h *Hub) GetItemsByType(itemType string, sorted bool) []*Item { } idx := 0 + for _, item := range items { ret[idx] = item idx += 1 @@ -269,6 +267,7 @@ func (h *Hub) GetInstalledListForAPI() []string { ret := make([]string, len(scenarios)+len(appsecRules)) idx := 0 + for _, item := range scenarios { ret[idx] = item.Name idx += 1 diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 727b9a18fdf..c2b949b7cdf 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -13,18 +13,19 @@ import ( func TestInitHubUpdate(t *testing.T) { hub := envSetup(t) - remote := &RemoteHubCfg{ - URLTemplate: mockURLTemplate, - Branch: "master", - IndexPath: ".index.json", - } - _, err := NewHub(hub.local, remote, nil) + _, err := NewHub(hub.local, nil) require.NoError(t, err) ctx := context.Background() - err = hub.Update(ctx, false) + indexProvider := &Downloader{ + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", + } + + err = hub.Update(ctx, indexProvider, false) require.NoError(t, err) err = hub.Load() @@ -48,29 +49,29 @@ func TestUpdateIndex(t *testing.T) { hub := envSetup(t) - hub.remote = &RemoteHubCfg{ + hub.local.HubIndexFile = tmpIndex.Name() + + ctx := context.Background() + + indexProvider := &Downloader{ URLTemplate: "x", Branch: "", IndexPath: "", } - hub.local.HubIndexFile = tmpIndex.Name() - - ctx := context.Background() - - err = hub.Update(ctx, false) + err = hub.Update(ctx, indexProvider, false) cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") // bad domain fmt.Println("Test 'bad domain'") - hub.remote = &RemoteHubCfg{ + indexProvider = &Downloader{ URLTemplate: "https://baddomain/crowdsecurity/%s/%s", Branch: "master", IndexPath: ".index.json", } - err = hub.Update(ctx, false) + err = hub.Update(ctx, indexProvider, false) require.NoError(t, err) // XXX: this is not failing // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") @@ -78,7 +79,7 @@ func TestUpdateIndex(t *testing.T) { // bad target path fmt.Println("Test 'bad target path'") - hub.remote = &RemoteHubCfg{ + indexProvider = &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", IndexPath: ".index.json", @@ -86,6 +87,6 @@ func TestUpdateIndex(t *testing.T) { hub.local.HubIndexFile = "/does/not/exist/index.json" - err = hub.Update(ctx, false) + err = hub.Update(ctx, indexProvider, false) cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:") } diff --git a/pkg/cwhub/itemupgrade_test.go b/pkg/cwhub/itemupgrade_test.go index e523a222d69..da02837e972 100644 --- a/pkg/cwhub/itemupgrade_test.go +++ b/pkg/cwhub/itemupgrade_test.go @@ -38,7 +38,7 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" pushUpdateToCollectionInHub() - remote := &RemoteHubCfg{ + remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", IndexPath: ".index.json", @@ -98,7 +98,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.NoError(t, err) require.True(t, didRemove) - remote := &RemoteHubCfg{ + remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", IndexPath: ".index.json", @@ -132,7 +132,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { } // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test. -func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub { +func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *Downloader) *Hub { hub, err := NewHub(local, remote, nil) require.NoError(t, err) @@ -170,7 +170,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te require.NoError(t, err) require.True(t, didRemove) - remote := &RemoteHubCfg{ + remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", IndexPath: ".index.json", diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go deleted file mode 100644 index c96471b390c..00000000000 --- a/pkg/cwhub/remote.go +++ /dev/null @@ -1,87 +0,0 @@ -package cwhub - -import ( - "context" - "fmt" - "net/http" - "net/url" - - "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/go-cs-lib/downloader" -) - -// RemoteHubCfg is used to retrieve index and items from the remote hub. -type RemoteHubCfg struct { - Branch string - URLTemplate string - IndexPath string -} - -// urlTo builds the URL to download a file from the remote hub. -func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { - if r == nil { - return "", ErrNilRemoteHub - } - - // the template must contain two string placeholders - if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate { - return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate) - } - - return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil -} - -// addURLParam adds a parameter with a value (ex. "with_content=true") to the URL if it's not already present. -func addURLParam(rawURL string, param string, value string) (string, error) { - parsedURL, err := url.Parse(rawURL) - if err != nil { - return "", fmt.Errorf("failed to parse URL: %w", err) - } - - query := parsedURL.Query() - - if _, exists := query[param]; !exists { - query.Add(param, value) - } - - parsedURL.RawQuery = query.Encode() - - return parsedURL.String(), nil -} - -// fetchIndex downloads the index from the hub and returns the content. -func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string, withContent bool) (bool, error) { - if r == nil { - return false, ErrNilRemoteHub - } - - url, err := r.urlTo(r.IndexPath) - if err != nil { - return false, fmt.Errorf("failed to build hub index request: %w", err) - } - - if withContent { - url, err = addURLParam(url, "with_content", "true") - if err != nil { - return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err) - } - } - - downloaded, err := downloader. - New(). - WithHTTPClient(HubClient). - ToFile(destPath). - WithETagFn(downloader.SHA256). - CompareContent(). - WithLogger(logrus.WithField("url", url)). - BeforeRequest(func(_ *http.Request) { - fmt.Println("Downloading "+destPath) - }). - Download(ctx, url) - if err != nil { - return false, err - } - - return downloaded, nil -} diff --git a/pkg/hubops/download.go b/pkg/hubops/download.go index 4a722efdb77..72aed542115 100644 --- a/pkg/hubops/download.go +++ b/pkg/hubops/download.go @@ -9,9 +9,9 @@ import ( "os" "time" + "github.com/fatih/color" "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" - "github.com/fatih/color" "github.com/crowdsecurity/go-cs-lib/downloader" @@ -19,19 +19,19 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) - // DownloadCommand handles the downloading of hub items. // It ensures that items are fetched from the hub (or from the index file if it also has content) // managing dependencies and verifying the integrity of downloaded content. // This is used by "cscli install" and "cscli upgrade". // Tainted items require the force parameter, local items are skipped. type DownloadCommand struct { - Item *cwhub.Item - Force bool + Item *cwhub.Item + Force bool + contentProvider cwhub.ContentProvider } -func NewDownloadCommand(item *cwhub.Item, force bool) *DownloadCommand { - return &DownloadCommand{Item: item, Force: force} +func NewDownloadCommand(item *cwhub.Item, contentProvider cwhub.ContentProvider, force bool) *DownloadCommand { + return &DownloadCommand{Item: item, Force: force, contentProvider: contentProvider} } func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) { @@ -60,7 +60,7 @@ func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) { } for sub := range i.LatestDependencies().SubItems(plan.hub) { - if err := plan.AddCommand(NewDownloadCommand(sub, c.Force)); err != nil { + if err := plan.AddCommand(NewDownloadCommand(sub, c.contentProvider, c.Force)); err != nil { return false, err } @@ -158,7 +158,7 @@ func (c *DownloadCommand) Run(ctx context.Context, plan *ActionPlan) error { return err } - downloaded, _, err := i.FetchContentTo(ctx, finalPath) + downloaded, _, err := i.FetchContentTo(ctx, c.contentProvider, finalPath) if err != nil { return fmt.Errorf("%s: %w", i.FQName(), err) } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index 93f5abaa879..ce9efcec601 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -14,8 +14,8 @@ type HubTest struct { CrowdSecPath string CscliPath string HubPath string - HubTestPath string //generic parser/scenario tests .tests - HubAppsecTestPath string //dir specific to appsec tests .appsec-tests + HubTestPath string // generic parser/scenario tests .tests + HubAppsecTestPath string // dir specific to appsec tests .appsec-tests HubIndexFile string TemplateConfigPath string TemplateProfilePath string @@ -93,7 +93,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT InstallDataDir: HubTestPath, } - hub, err := cwhub.NewHub(local, nil, nil) + hub, err := cwhub.NewHub(local, nil) if err != nil { return HubTest{}, err } @@ -130,7 +130,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT InstallDataDir: HubTestPath, } - hub, err := cwhub.NewHub(local, nil, nil) + hub, err := cwhub.NewHub(local, nil) if err != nil { return HubTest{}, err } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index d999d15ba6e..e8bc56f650a 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -212,7 +212,7 @@ func (t *HubTestItem) InstallHub() error { } // load installed hub - hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, nil) + hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil) if err != nil { return err } diff --git a/pkg/leakybucket/buckets_test.go b/pkg/leakybucket/buckets_test.go index 8bb7a3d4c47..90a751160cb 100644 --- a/pkg/leakybucket/buckets_test.go +++ b/pkg/leakybucket/buckets_test.go @@ -46,7 +46,7 @@ func TestBucket(t *testing.T) { InstallDataDir: testdata, } - hub, err := cwhub.NewHub(hubCfg, nil, nil) + hub, err := cwhub.NewHub(hubCfg, nil) require.NoError(t, err) err = hub.Load() @@ -140,6 +140,7 @@ func testOneBucket(t *testing.T, hub *cwhub.Hub, dir string, tomb *tomb.Tomb) er } scenarios := []*cwhub.Item{} + for _, x := range stages { // XXX: LoadBuckets should take an interface, BucketProvider ScenarioProvider or w/e item := &cwhub.Item{ diff --git a/pkg/setup/install.go b/pkg/setup/install.go index dcefe744a76..42634672912 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -48,7 +48,7 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) { } // InstallHubItems installs the objects recommended in a setup file. -func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dryRun, verbose bool) error { +func InstallHubItems(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, input []byte, yes, dryRun, verbose bool) error { setupEnvelope, err := decodeSetup(input, false) if err != nil { return err @@ -71,7 +71,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry return fmt.Errorf("collection %s not found", collection) } - plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) + if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } @@ -83,7 +84,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry return fmt.Errorf("parser %s not found", parser) } - plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) + if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } @@ -95,7 +97,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry return fmt.Errorf("scenario %s not found", scenario) } - plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) + if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } @@ -107,7 +110,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry return fmt.Errorf("postoverflow %s not found", postoverflow) } - plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) + if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) }