From 07165aab0ecfa3bddacf4e49e49a407f3d9366f3 Mon Sep 17 00:00:00 2001 From: marco Date: Thu, 26 Dec 2024 14:11:19 +0100 Subject: [PATCH] pkg/cwhub - refact Downloader --- cmd/crowdsec-cli/clicapi/capi.go | 2 +- cmd/crowdsec-cli/cliconsole/console.go | 2 +- cmd/crowdsec-cli/clihub/hub.go | 16 +-- cmd/crowdsec-cli/cliitem/item.go | 38 ++++--- cmd/crowdsec-cli/cliitem/suggest.go | 4 +- cmd/crowdsec-cli/clilapi/context.go | 6 +- cmd/crowdsec-cli/clilapi/status.go | 2 +- cmd/crowdsec-cli/clisetup/setup.go | 6 +- cmd/crowdsec-cli/clisimulation/simulation.go | 2 +- cmd/crowdsec-cli/clisupport/support.go | 2 +- cmd/crowdsec-cli/config_backup.go | 2 +- cmd/crowdsec-cli/config_restore.go | 6 +- cmd/crowdsec-cli/require/require.go | 9 +- cmd/crowdsec/serve.go | 4 +- go.mod | 2 +- go.sum | 6 +- pkg/cwhub/cwhub_test.go | 16 +-- pkg/cwhub/doc.go | 11 +- pkg/cwhub/download.go | 110 +++++++++++++++++++ pkg/cwhub/errors.go | 19 ---- pkg/cwhub/fetch.go | 43 +------- pkg/cwhub/hub.go | 14 ++- pkg/cwhub/hub_test.go | 35 +++--- pkg/cwhub/itemupgrade_test.go | 8 +- pkg/cwhub/remote.go | 87 --------------- pkg/hubops/download.go | 9 +- pkg/hubtest/hubtest.go | 4 +- pkg/hubtest/hubtest_item.go | 2 +- pkg/leakybucket/buckets_test.go | 2 +- pkg/setup/install.go | 10 +- 30 files changed, 225 insertions(+), 254 deletions(-) create mode 100644 pkg/cwhub/download.go delete mode 100644 pkg/cwhub/errors.go delete mode 100644 pkg/cwhub/remote.go diff --git a/cmd/crowdsec-cli/clicapi/capi.go b/cmd/crowdsec-cli/clicapi/capi.go index 2cce01c7d3e..120acca8b59 100644 --- a/cmd/crowdsec-cli/clicapi/capi.go +++ b/cmd/crowdsec-cli/clicapi/capi.go @@ -261,7 +261,7 @@ func (cli *cliCapi) newStatusCmd() *cobra.Command { Args: cobra.MinimumNArgs(0), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { - hub, err := require.Hub(cli.cfg(), nil, nil) + hub, err := require.Hub(cli.cfg(), nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/cliconsole/console.go b/cmd/crowdsec-cli/cliconsole/console.go index 448ddcee7fa..dbbe2487cd4 100644 --- a/cmd/crowdsec-cli/cliconsole/console.go +++ b/cmd/crowdsec-cli/cliconsole/console.go @@ -114,7 +114,7 @@ func (cli *cliConsole) enroll(ctx context.Context, key string, name string, over } } - hub, err := require.Hub(cfg, nil, nil) + hub, err := require.Hub(cfg, nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/clihub/hub.go b/cmd/crowdsec-cli/clihub/hub.go index 49ccd761285..160293746d5 100644 --- a/cmd/crowdsec-cli/clihub/hub.go +++ b/cmd/crowdsec-cli/clihub/hub.go @@ -93,7 +93,7 @@ func (cli *cliHub) newListCmd() *cobra.Command { Args: cobra.NoArgs, DisableAutoGenTag: true, RunE: func(_ *cobra.Command, _ []string) error { - hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger()) + hub, err := require.Hub(cli.cfg(), log.StandardLogger()) if err != nil { return err } @@ -110,15 +110,15 @@ func (cli *cliHub) newListCmd() *cobra.Command { func (cli *cliHub) update(ctx context.Context, withContent bool) error { local := cli.cfg().Hub - remote := require.RemoteHub(ctx, cli.cfg()) - // don't use require.Hub because if there is no index file, it would fail - hub, err := cwhub.NewHub(local, remote, log.StandardLogger()) + hub, err := cwhub.NewHub(local, log.StandardLogger()) if err != nil { return err } - if err := hub.Update(ctx, withContent); err != nil { + hubProvider := require.HubDownloader(ctx, cli.cfg()) + + if err := hub.Update(ctx, hubProvider, withContent); err != nil { return fmt.Errorf("failed to update hub: %w", err) } @@ -166,16 +166,18 @@ cscli hub update --with-content`, func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force bool) error { cfg := cli.cfg() - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) + hub, err := require.Hub(cfg, log.StandardLogger()) if err != nil { return err } plan := hubops.NewActionPlan(hub) + hubProvider := require.HubDownloader(ctx, cfg) + for _, itemType := range cwhub.ItemTypes { for _, item := range hub.GetInstalledByType(itemType, true) { - plan.AddCommand(hubops.NewDownloadCommand(item, force)) + plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)) } } diff --git a/cmd/crowdsec-cli/cliitem/item.go b/cmd/crowdsec-cli/cliitem/item.go index 05e52d18dd3..55e39030f76 100644 --- a/cmd/crowdsec-cli/cliitem/item.go +++ b/cmd/crowdsec-cli/cliitem/item.go @@ -71,13 +71,15 @@ func (cli cliItem) NewCommand() *cobra.Command { func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun bool, downloadOnly bool, force bool, ignoreError bool) error { cfg := cli.cfg() - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) + hub, err := require.Hub(cfg, log.StandardLogger()) if err != nil { return err } plan := hubops.NewActionPlan(hub) + hubProvider := require.HubDownloader(ctx, cfg) + for _, name := range args { item := hub.GetItem(cli.name, name) if item == nil { @@ -91,7 +93,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun continue } - if err = plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil { + if err = plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil { return err } @@ -228,7 +230,7 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun bool, purge bool, force bool, all bool) error { cfg := cli.cfg() - hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger()) + hub, err := require.Hub(cli.cfg(), log.StandardLogger()) if err != nil { return err } @@ -290,12 +292,12 @@ func (cli cliItem) newRemoveCmd() *cobra.Command { return cmd } -func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bool) (*hubops.ActionPlan, error) { +func (cli cliItem) upgradePlan(hub *cwhub.Hub, hubProvider cwhub.HubProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) { plan := hubops.NewActionPlan(hub) if all { for _, item := range hub.GetInstalledByType(cli.name, true) { - if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil { + if err := plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil { return nil, err } } @@ -313,7 +315,7 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bo return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name) } - if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil { + if err := plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil { return nil, err } } @@ -324,12 +326,14 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bo func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun bool, force bool, all bool) error { cfg := cli.cfg() - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) + hub, err := require.Hub(cfg, log.StandardLogger()) if err != nil { return err } - plan, err := cli.upgradePlan(hub, args, force, all) + hubProvider := require.HubDownloader(ctx, cfg) + + plan, err := cli.upgradePlan(hub, hubProvider, args, force, all) if err != nil { return err } @@ -390,13 +394,13 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff cfg.Cscli.PrometheusUrl = url } - remote := (*cwhub.RemoteHubCfg)(nil) + var hubProvider cwhub.HubProvider if diff { - remote = require.RemoteHub(ctx, cfg) + hubProvider = require.HubDownloader(ctx, cfg) } - hub, err := require.Hub(cfg, remote, log.StandardLogger()) + hub, err := require.Hub(cfg, log.StandardLogger()) if err != nil { return err } @@ -408,7 +412,7 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff } if diff { - fmt.Println(cli.whyTainted(ctx, hub, item, rev)) + fmt.Println(cli.whyTainted(ctx, hub, hubProvider, item, rev)) continue } @@ -462,7 +466,7 @@ func (cli cliItem) newInspectCmd() *cobra.Command { func (cli cliItem) list(args []string, all bool) error { cfg := cli.cfg() - hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger()) + hub, err := require.Hub(cli.cfg(), log.StandardLogger()) if err != nil { return err } @@ -498,7 +502,7 @@ func (cli cliItem) newListCmd() *cobra.Command { } // return the diff between the installed version and the latest version -func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool) (string, error) { +func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, hubProvider cwhub.HubProvider, reverse bool) (string, error) { if !item.State.Installed { return "", fmt.Errorf("'%s' is not installed", item.FQName()) } @@ -509,7 +513,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool) } defer os.Remove(dest.Name()) - _, remoteURL, err := item.FetchContentTo(ctx, dest.Name()) + _, remoteURL, err := item.FetchContentTo(ctx, hubProvider, dest.Name()) if err != nil { return "", err } @@ -540,7 +544,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool) return fmt.Sprintf("%s", diff), nil } -func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, item *cwhub.Item, reverse bool) string { +func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, hubProvider cwhub.HubProvider, item *cwhub.Item, reverse bool) string { if !item.State.Installed { return fmt.Sprintf("# %s is not installed", item.FQName()) } @@ -565,7 +569,7 @@ func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, item *cwhub.I ret = append(ret, err.Error()) } - diff, err := cli.itemDiff(ctx, sub, reverse) + diff, err := cli.itemDiff(ctx, sub, hubProvider, reverse) if err != nil { ret = append(ret, err.Error()) } diff --git a/cmd/crowdsec-cli/cliitem/suggest.go b/cmd/crowdsec-cli/cliitem/suggest.go index 5b080722af9..b0f19b6993c 100644 --- a/cmd/crowdsec-cli/cliitem/suggest.go +++ b/cmd/crowdsec-cli/cliitem/suggest.go @@ -37,7 +37,7 @@ func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) str } func compAllItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) { - hub, err := require.Hub(cfg(), nil, nil) + hub, err := require.Hub(cfg(), nil) if err != nil { return nil, cobra.ShellCompDirectiveDefault } @@ -56,7 +56,7 @@ func compAllItems(itemType string, args []string, toComplete string, cfg configG } func compInstalledItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) { - hub, err := require.Hub(cfg(), nil, nil) + hub, err := require.Hub(cfg(), nil) if err != nil { return nil, cobra.ShellCompDirectiveDefault } diff --git a/cmd/crowdsec-cli/clilapi/context.go b/cmd/crowdsec-cli/clilapi/context.go index 20ceb2b9596..0730ba2b2a9 100644 --- a/cmd/crowdsec-cli/clilapi/context.go +++ b/cmd/crowdsec-cli/clilapi/context.go @@ -59,7 +59,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user `, DisableAutoGenTag: true, RunE: func(_ *cobra.Command, _ []string) error { - hub, err := require.Hub(cli.cfg(), nil, nil) + hub, err := require.Hub(cli.cfg(), nil) if err != nil { return err } @@ -101,7 +101,7 @@ func (cli *cliLapi) newContextStatusCmd() *cobra.Command { DisableAutoGenTag: true, RunE: func(_ *cobra.Command, _ []string) error { cfg := cli.cfg() - hub, err := require.Hub(cfg, nil, nil) + hub, err := require.Hub(cfg, nil) if err != nil { return err } @@ -153,7 +153,7 @@ cscli lapi context detect crowdsecurity/sshd-logs return fmt.Errorf("failed to init expr helpers: %w", err) } - hub, err := require.Hub(cfg, nil, nil) + hub, err := require.Hub(cfg, nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/clilapi/status.go b/cmd/crowdsec-cli/clilapi/status.go index 6ff88834602..039c75e585d 100644 --- a/cmd/crowdsec-cli/clilapi/status.go +++ b/cmd/crowdsec-cli/clilapi/status.go @@ -102,7 +102,7 @@ func (cli *cliLapi) newStatusCmd() *cobra.Command { Args: cobra.MinimumNArgs(0), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { - hub, err := require.Hub(cli.cfg(), nil, nil) + hub, err := require.Hub(cli.cfg(), nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/clisetup/setup.go b/cmd/crowdsec-cli/clisetup/setup.go index 4cb423e484c..a189e56e704 100644 --- a/cmd/crowdsec-cli/clisetup/setup.go +++ b/cmd/crowdsec-cli/clisetup/setup.go @@ -289,14 +289,16 @@ func (cli *cliSetup) install(ctx context.Context, yes bool, dryRun bool, fromFil cfg := cli.cfg() - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger()) + hub, err := require.Hub(cfg, log.StandardLogger()) if err != nil { return err } verbose := (cfg.Cscli.Output == "raw") - return setup.InstallHubItems(ctx, hub, input, yes, dryRun, verbose) + hubProvider := require.HubDownloader(ctx, cfg) + + return setup.InstallHubItems(ctx, hub, hubProvider, input, yes, dryRun, verbose) } func (cli *cliSetup) validate(fromFile string) error { diff --git a/cmd/crowdsec-cli/clisimulation/simulation.go b/cmd/crowdsec-cli/clisimulation/simulation.go index 8136aa213c3..c06db56f200 100644 --- a/cmd/crowdsec-cli/clisimulation/simulation.go +++ b/cmd/crowdsec-cli/clisimulation/simulation.go @@ -71,7 +71,7 @@ func (cli *cliSimulation) newEnableCmd() *cobra.Command { Example: `cscli simulation enable`, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - hub, err := require.Hub(cli.cfg(), nil, nil) + hub, err := require.Hub(cli.cfg(), nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/clisupport/support.go b/cmd/crowdsec-cli/clisupport/support.go index a21d00e3b20..0f4d503da54 100644 --- a/cmd/crowdsec-cli/clisupport/support.go +++ b/cmd/crowdsec-cli/clisupport/support.go @@ -491,7 +491,7 @@ func (cli *cliSupport) dump(ctx context.Context, outFile string) error { skipAgent = true } - hub, err := require.Hub(cfg, nil, nil) + hub, err := require.Hub(cfg, nil) if err != nil { log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected") // XXX: lapi status check requires scenarios, will return an error diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index d23aff80a78..faac786ebdc 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -15,7 +15,7 @@ import ( ) func (cli *cliConfig) backupHub(dirPath string) error { - hub, err := require.Hub(cli.cfg(), nil, nil) + hub, err := require.Hub(cli.cfg(), nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 8884fa448d2..e9221df9f9e 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -18,11 +18,13 @@ import ( func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error { cfg := cli.cfg() - hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), nil) + hub, err := require.Hub(cfg, nil) if err != nil { return err } + hubProvider := require.HubDownloader(ctx, cfg) + for _, itype := range cwhub.ItemTypes { itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) if _, err = os.Stat(itemDirectory); err != nil { @@ -53,7 +55,7 @@ func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error { plan := hubops.NewActionPlan(hub) - if err = plan.AddCommand(hubops.NewDownloadCommand(item, false)); err != nil { + if err = plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, false)); err != nil { return err } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index 7b3410021c1..a44e76ae47d 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -82,12 +82,11 @@ func Notifications(c *csconfig.Config) error { return nil } -// RemoteHub returns the configuration required to download hub index and items: url, branch, etc. -func RemoteHub(ctx context.Context, c *csconfig.Config) *cwhub.RemoteHubCfg { +func HubDownloader(ctx context.Context, c *csconfig.Config) *cwhub.Downloader { // set branch in config, and log if necessary branch := HubBranch(ctx, c) urlTemplate := HubURLTemplate(c) - remote := &cwhub.RemoteHubCfg{ + remote := &cwhub.Downloader{ Branch: branch, URLTemplate: urlTemplate, IndexPath: ".index.json", @@ -98,7 +97,7 @@ func RemoteHub(ctx context.Context, c *csconfig.Config) *cwhub.RemoteHubCfg { // Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items. // If no remote parameter is provided, the hub can only be used for local operations. -func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) { +func Hub(c *csconfig.Config, logger *logrus.Logger) (*cwhub.Hub, error) { local := c.Hub if local == nil { @@ -110,7 +109,7 @@ func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) logger.SetOutput(io.Discard) } - hub, err := cwhub.NewHub(local, remote, logger) + hub, err := cwhub.NewHub(local, logger) if err != nil { return nil, err } diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 14602c425fe..62b721befdb 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -85,7 +85,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } if !cConfig.DisableAgent { - hub, err := cwhub.NewHub(cConfig.Hub, nil, log.StandardLogger()) + hub, err := cwhub.NewHub(cConfig.Hub, log.StandardLogger()) if err != nil { return nil, err } @@ -387,7 +387,7 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error { } if !cConfig.DisableAgent { - hub, err := cwhub.NewHub(cConfig.Hub, nil, log.StandardLogger()) + hub, err := cwhub.NewHub(cConfig.Hub, log.StandardLogger()) if err != nil { return err } diff --git a/go.mod b/go.mod index e437bbd688a..aa723b38409 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/creack/pty v1.1.21 // indirect github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f + github.com/crowdsecurity/go-cs-lib v0.0.16 github.com/crowdsecurity/grokky v0.2.2 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index d092956d0a8..e7f181d7d24 100644 --- a/go.sum +++ b/go.sum @@ -107,10 +107,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 h1:hyrYw3h github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607/go.mod h1:br36fEqurGYZQGit+iDYsIzW0FF6VufMbDzyyLxEuPA= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241203101722-e557f9809413 h1:VIedap4s3mXM4+tM2NMm7R3E/kn79ayLZaLHDqPYVCc= -github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241203101722-e557f9809413/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g= -github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f h1:Pd+O4UK78uQtTqbvYX+nHvqZ7TffD51uC4q0RE/podk= -github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g= +github.com/crowdsecurity/go-cs-lib v0.0.16 h1:2/htodjwc/sfsv4deX8F/2Fzg1bOI8w3O1/BPSvvsB0= +github.com/crowdsecurity/go-cs-lib v0.0.16/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g= github.com/crowdsecurity/grokky v0.2.2 h1:yALsI9zqpDArYzmSSxfBq2dhYuGUTKMJq8KOEIAsuo4= github.com/crowdsecurity/grokky v0.2.2/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 1b5dee34dd3..5e57a13a46d 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -57,18 +57,18 @@ func testHub(t *testing.T, update bool) *Hub { os.RemoveAll(tmpDir) }) - remote := &RemoteHubCfg{ - Branch: "master", - URLTemplate: mockURLTemplate, - IndexPath: ".index.json", - } - - hub, err := NewHub(local, remote, log.StandardLogger()) + hub, err := NewHub(local, log.StandardLogger()) require.NoError(t, err) if update { + hubProvider := &Downloader{ + Branch: "master", + URLTemplate: mockURLTemplate, + IndexPath: ".index.json", + } + ctx := context.Background() - err := hub.Update(ctx, false) + err := hub.Update(ctx, hubProvider, false) require.NoError(t, err) } diff --git a/pkg/cwhub/doc.go b/pkg/cwhub/doc.go index a1ee9d37ee7..f893e1fb1a9 100644 --- a/pkg/cwhub/doc.go +++ b/pkg/cwhub/doc.go @@ -85,25 +85,20 @@ // return fmt.Errorf("collection not found") // } // -// To provide the remote hub configuration, use the second parameter of NewHub(): +// Some commands require an object to provide the hub index, or contents: // -// remoteHub := cwhub.RemoteHubCfg{ +// hubProvider := cwhub.Downloader{ // URLTemplate: "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s", // Branch: "master", // IndexPath: ".index.json", // } // -// hub, err := cwhub.NewHub(localHub, remoteHub, logger) -// if err != nil { -// return fmt.Errorf("unable to initialize hub: %w", err) -// } -// // The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two // placeholders: the branch and the file path (it will be an index or an item). // // Before calling hub.Load(), you can update the index file by calling the Update() method: // -// err := hub.Update(context.Background()) +// err := hub.Update(context.Background(), hubProvider) // if err != nil { // return fmt.Errorf("unable to update hub index: %w", err) // } diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go new file mode 100644 index 00000000000..0ed419d4f6d --- /dev/null +++ b/pkg/cwhub/download.go @@ -0,0 +1,110 @@ +package cwhub + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + + "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/go-cs-lib/downloader" +) + +// Downloader is used to retrieve index and items from a remote hub. +type Downloader struct { + Branch string + URLTemplate string + IndexPath string +} + +// urlTo builds the URL to download a file from the remote hub. +func (d *Downloader) urlTo(remotePath string) (string, error) { + // the template must contain two string placeholders + if fmt.Sprintf(d.URLTemplate, "%s", "%s") != d.URLTemplate { + return "", fmt.Errorf("invalid URL template '%s'", d.URLTemplate) + } + + return fmt.Sprintf(d.URLTemplate, d.Branch, remotePath), nil +} + +// addURLParam adds a parameter with a value (ex. "with_content=true") to the URL if it's not already present. +func addURLParam(rawURL string, param string, value string) (string, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + query := parsedURL.Query() + + if _, exists := query[param]; !exists { + query.Add(param, value) + } + + parsedURL.RawQuery = query.Encode() + + return parsedURL.String(), nil +} + +// FetchIndex downloads the index from the hub and returns the content. +func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withContent bool, logger *logrus.Logger) (bool, error) { + url, err := d.urlTo(d.IndexPath) + if err != nil { + return false, fmt.Errorf("failed to build hub index request: %w", err) + } + + if withContent { + url, err = addURLParam(url, "with_content", "true") + if err != nil { + return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err) + } + } + + downloaded, err := downloader. + New(). + WithHTTPClient(HubClient). + ToFile(destPath). + WithETagFn(downloader.SHA256). + CompareContent(). + WithLogger(logger.WithField("url", url)). + BeforeRequest(func(_ *http.Request) { + fmt.Println("Downloading "+destPath) + }). + Download(ctx, url) + if err != nil { + return false, err + } + + return downloaded, nil +} + +// FetchContent downloads the content to the specified path and checks the hash. +func (d *Downloader) FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error) { + url, err := d.urlTo(remotePath) + if err != nil { + return false, "", fmt.Errorf("failed to build request: %w", err) + } + + downloaded, err := downloader. + New(). + WithHTTPClient(HubClient). + ToFile(destPath). + WithETagFn(downloader.SHA256). + WithMakeDirs(true). + WithLogger(logger.WithField("url", url)). + CompareContent(). + VerifyHash("sha256", wantHash). + Download(ctx, url) + + var hasherr downloader.HashMismatchError + + switch { + case errors.As(err, &hasherr): + logger.Warnf("%s. The index file is outdated, please run 'cscli hub update' and try again", err.Error()) + case err != nil: + return false, "", err + } + + return downloaded, url, nil +} diff --git a/pkg/cwhub/errors.go b/pkg/cwhub/errors.go deleted file mode 100644 index b0be444fcba..00000000000 --- a/pkg/cwhub/errors.go +++ /dev/null @@ -1,19 +0,0 @@ -package cwhub - -import ( - "errors" - "fmt" -) - -// ErrNilRemoteHub is returned when trying to download with a local-only configuration. -var ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers") - -// IndexNotFoundError is returned when the remote hub index is not found. -type IndexNotFoundError struct { - URL string - Branch string -} - -func (e IndexNotFoundError) Error() string { - return fmt.Sprintf("index not found at %s, branch '%s'. Please check the .cscli.hub_branch value if you specified it in config.yaml, or use 'master' if not sure", e.URL, e.Branch) -} diff --git a/pkg/cwhub/fetch.go b/pkg/cwhub/fetch.go index 92198e63ef1..138e0e5e6a2 100644 --- a/pkg/cwhub/fetch.go +++ b/pkg/cwhub/fetch.go @@ -1,21 +1,13 @@ package cwhub -// Install, upgrade and remove items from the hub to the local configuration - import ( "context" "crypto" "encoding/base64" "encoding/hex" - "errors" "fmt" "os" "path/filepath" - - "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/go-cs-lib/downloader" - ) @@ -56,40 +48,9 @@ func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error { return nil } -// writeRemoteContentTo downloads the content to the specified path and checks the hash. -func (i *Item) writeRemoteContentTo(ctx context.Context, destPath, wantHash string) (bool, string, error) { - url, err := i.hub.remote.urlTo(i.RemotePath) - if err != nil { - return false, "", fmt.Errorf("failed to build request: %w", err) - } - - d := downloader. - New(). - WithHTTPClient(HubClient). - ToFile(destPath). - WithETagFn(downloader.SHA256). - WithMakeDirs(true). - WithLogger(logrus.WithField("url", url)). - CompareContent(). - VerifyHash("sha256", wantHash) - - hasherr := downloader.HashMismatchError{} - - downloaded, err := d.Download(ctx, url) - - switch { - case errors.As(err, &hasherr): - i.hub.logger.Warnf("%s. The index file is outdated, please run 'cscli hub update' and try again", err.Error()) - case err != nil: - return false, "", err - } - - return downloaded, url, nil -} - // FetchContentTo writes the last version of the item's YAML file to the specified path. // Returns whether the file was downloaded, and the remote url for feedback purposes. -func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, string, error) { +func (i *Item) FetchContentTo(ctx context.Context, hubProvider HubProvider, destPath string) (bool, string, error) { wantHash := i.latestHash() if wantHash == "" { return false, "", fmt.Errorf("%s: latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again", i.FQName()) @@ -104,5 +65,5 @@ func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, strin return true, fmt.Sprintf("(embedded in %s)", i.hub.local.HubIndexFile), nil } - return i.writeRemoteContentTo(ctx, destPath, wantHash) + return hubProvider.FetchContent(ctx, i.RemotePath, destPath, wantHash, i.hub.logger) } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 55469fed711..ce56a780815 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -22,7 +22,6 @@ type Hub struct { items HubItems // Items read from HubDir and InstallDir pathIndex map[string]*Item local *csconfig.LocalHubCfg - remote *RemoteHubCfg logger *logrus.Logger Warnings []string // Warnings encountered during sync } @@ -35,8 +34,7 @@ func (h *Hub) GetDataDir() string { // NewHub returns a new Hub instance with local and (optionally) remote configuration. // The hub is not synced automatically. Load() must be called to read the index, sync the local state, // and check for unmanaged items. -// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set. -func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, logger *logrus.Logger) (*Hub, error) { +func NewHub(local *csconfig.LocalHubCfg, logger *logrus.Logger) (*Hub, error) { if local == nil { return nil, errors.New("no hub configuration found") } @@ -48,7 +46,6 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, logger *logrus.Lo hub := &Hub{ local: local, - remote: remote, logger: logger, pathIndex: make(map[string]*Item, 0), } @@ -156,15 +153,20 @@ func (h *Hub) ItemStats() []string { return ret } +type HubProvider interface { + FetchIndex(ctx context.Context, indexFile string, withContent bool, logger *logrus.Logger) (bool, error) + FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error) +} + // Update downloads the latest version of the index and writes it to disk if it changed. // It cannot be called after Load() unless the hub is completely empty. -func (h *Hub) Update(ctx context.Context, withContent bool) error { +func (h *Hub) Update(ctx context.Context, hubProvider HubProvider, withContent bool) error { if len(h.pathIndex) > 0 { // if this happens, it's a bug. return errors.New("cannot update hub after items have been loaded") } - downloaded, err := h.remote.fetchIndex(ctx, h.local.HubIndexFile, withContent) + downloaded, err := hubProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger) if err != nil { return err } diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 727b9a18fdf..dbde5a8a623 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -13,18 +13,19 @@ import ( func TestInitHubUpdate(t *testing.T) { hub := envSetup(t) - remote := &RemoteHubCfg{ - URLTemplate: mockURLTemplate, - Branch: "master", - IndexPath: ".index.json", - } - _, err := NewHub(hub.local, remote, nil) + _, err := NewHub(hub.local, nil) require.NoError(t, err) ctx := context.Background() - err = hub.Update(ctx, false) + hubProvider := &Downloader{ + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", + } + + err = hub.Update(ctx, hubProvider, false) require.NoError(t, err) err = hub.Load() @@ -48,29 +49,29 @@ func TestUpdateIndex(t *testing.T) { hub := envSetup(t) - hub.remote = &RemoteHubCfg{ + hub.local.HubIndexFile = tmpIndex.Name() + + ctx := context.Background() + + hubProvider := &Downloader{ URLTemplate: "x", Branch: "", IndexPath: "", } - hub.local.HubIndexFile = tmpIndex.Name() - - ctx := context.Background() - - err = hub.Update(ctx, false) + err = hub.Update(ctx, hubProvider, false) cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") // bad domain fmt.Println("Test 'bad domain'") - hub.remote = &RemoteHubCfg{ + hubProvider = &Downloader{ URLTemplate: "https://baddomain/crowdsecurity/%s/%s", Branch: "master", IndexPath: ".index.json", } - err = hub.Update(ctx, false) + err = hub.Update(ctx, hubProvider, false) require.NoError(t, err) // XXX: this is not failing // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") @@ -78,7 +79,7 @@ func TestUpdateIndex(t *testing.T) { // bad target path fmt.Println("Test 'bad target path'") - hub.remote = &RemoteHubCfg{ + hubProvider = &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", IndexPath: ".index.json", @@ -86,6 +87,6 @@ func TestUpdateIndex(t *testing.T) { hub.local.HubIndexFile = "/does/not/exist/index.json" - err = hub.Update(ctx, false) + err = hub.Update(ctx, hubProvider, false) cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:") } diff --git a/pkg/cwhub/itemupgrade_test.go b/pkg/cwhub/itemupgrade_test.go index e523a222d69..da02837e972 100644 --- a/pkg/cwhub/itemupgrade_test.go +++ b/pkg/cwhub/itemupgrade_test.go @@ -38,7 +38,7 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" pushUpdateToCollectionInHub() - remote := &RemoteHubCfg{ + remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", IndexPath: ".index.json", @@ -98,7 +98,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.NoError(t, err) require.True(t, didRemove) - remote := &RemoteHubCfg{ + remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", IndexPath: ".index.json", @@ -132,7 +132,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { } // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test. -func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub { +func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *Downloader) *Hub { hub, err := NewHub(local, remote, nil) require.NoError(t, err) @@ -170,7 +170,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te require.NoError(t, err) require.True(t, didRemove) - remote := &RemoteHubCfg{ + remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", IndexPath: ".index.json", diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go deleted file mode 100644 index c96471b390c..00000000000 --- a/pkg/cwhub/remote.go +++ /dev/null @@ -1,87 +0,0 @@ -package cwhub - -import ( - "context" - "fmt" - "net/http" - "net/url" - - "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/go-cs-lib/downloader" -) - -// RemoteHubCfg is used to retrieve index and items from the remote hub. -type RemoteHubCfg struct { - Branch string - URLTemplate string - IndexPath string -} - -// urlTo builds the URL to download a file from the remote hub. -func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { - if r == nil { - return "", ErrNilRemoteHub - } - - // the template must contain two string placeholders - if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate { - return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate) - } - - return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil -} - -// addURLParam adds a parameter with a value (ex. "with_content=true") to the URL if it's not already present. -func addURLParam(rawURL string, param string, value string) (string, error) { - parsedURL, err := url.Parse(rawURL) - if err != nil { - return "", fmt.Errorf("failed to parse URL: %w", err) - } - - query := parsedURL.Query() - - if _, exists := query[param]; !exists { - query.Add(param, value) - } - - parsedURL.RawQuery = query.Encode() - - return parsedURL.String(), nil -} - -// fetchIndex downloads the index from the hub and returns the content. -func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string, withContent bool) (bool, error) { - if r == nil { - return false, ErrNilRemoteHub - } - - url, err := r.urlTo(r.IndexPath) - if err != nil { - return false, fmt.Errorf("failed to build hub index request: %w", err) - } - - if withContent { - url, err = addURLParam(url, "with_content", "true") - if err != nil { - return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err) - } - } - - downloaded, err := downloader. - New(). - WithHTTPClient(HubClient). - ToFile(destPath). - WithETagFn(downloader.SHA256). - CompareContent(). - WithLogger(logrus.WithField("url", url)). - BeforeRequest(func(_ *http.Request) { - fmt.Println("Downloading "+destPath) - }). - Download(ctx, url) - if err != nil { - return false, err - } - - return downloaded, nil -} diff --git a/pkg/hubops/download.go b/pkg/hubops/download.go index 4a722efdb77..6ed173ac620 100644 --- a/pkg/hubops/download.go +++ b/pkg/hubops/download.go @@ -28,10 +28,11 @@ import ( type DownloadCommand struct { Item *cwhub.Item Force bool + hubProvider cwhub.HubProvider } -func NewDownloadCommand(item *cwhub.Item, force bool) *DownloadCommand { - return &DownloadCommand{Item: item, Force: force} +func NewDownloadCommand(item *cwhub.Item, hubProvider cwhub.HubProvider, force bool) *DownloadCommand { + return &DownloadCommand{Item: item, Force: force, hubProvider: hubProvider} } func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) { @@ -60,7 +61,7 @@ func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) { } for sub := range i.LatestDependencies().SubItems(plan.hub) { - if err := plan.AddCommand(NewDownloadCommand(sub, c.Force)); err != nil { + if err := plan.AddCommand(NewDownloadCommand(sub, c.hubProvider, c.Force)); err != nil { return false, err } @@ -158,7 +159,7 @@ func (c *DownloadCommand) Run(ctx context.Context, plan *ActionPlan) error { return err } - downloaded, _, err := i.FetchContentTo(ctx, finalPath) + downloaded, _, err := i.FetchContentTo(ctx, c.hubProvider, finalPath) if err != nil { return fmt.Errorf("%s: %w", i.FQName(), err) } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index 93f5abaa879..5fd1ce78884 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -93,7 +93,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT InstallDataDir: HubTestPath, } - hub, err := cwhub.NewHub(local, nil, nil) + hub, err := cwhub.NewHub(local, nil) if err != nil { return HubTest{}, err } @@ -130,7 +130,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT InstallDataDir: HubTestPath, } - hub, err := cwhub.NewHub(local, nil, nil) + hub, err := cwhub.NewHub(local, nil) if err != nil { return HubTest{}, err } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index d999d15ba6e..e8bc56f650a 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -212,7 +212,7 @@ func (t *HubTestItem) InstallHub() error { } // load installed hub - hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, nil) + hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil) if err != nil { return err } diff --git a/pkg/leakybucket/buckets_test.go b/pkg/leakybucket/buckets_test.go index 8bb7a3d4c47..d34583b5af2 100644 --- a/pkg/leakybucket/buckets_test.go +++ b/pkg/leakybucket/buckets_test.go @@ -46,7 +46,7 @@ func TestBucket(t *testing.T) { InstallDataDir: testdata, } - hub, err := cwhub.NewHub(hubCfg, nil, nil) + hub, err := cwhub.NewHub(hubCfg, nil) require.NoError(t, err) err = hub.Load() diff --git a/pkg/setup/install.go b/pkg/setup/install.go index dcefe744a76..d90b6177e00 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -48,7 +48,7 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) { } // InstallHubItems installs the objects recommended in a setup file. -func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dryRun, verbose bool) error { +func InstallHubItems(ctx context.Context, hub *cwhub.Hub, hubProvider cwhub.HubProvider, input []byte, yes, dryRun, verbose bool) error { setupEnvelope, err := decodeSetup(input, false) if err != nil { return err @@ -71,7 +71,7 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry return fmt.Errorf("collection %s not found", collection) } - plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, forceAction)) if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } @@ -83,7 +83,7 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry return fmt.Errorf("parser %s not found", parser) } - plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, forceAction)) if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } @@ -95,7 +95,7 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry return fmt.Errorf("scenario %s not found", scenario) } - plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, forceAction)) if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } @@ -107,7 +107,7 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry return fmt.Errorf("postoverflow %s not found", postoverflow) } - plan.AddCommand(hubops.NewDownloadCommand(item, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, forceAction)) if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) }