pkg/cwhub - refact Downloader

cmd/crowdsec-cli/clicapi/capi.go

@@ -261,7 +261,7 @@ func (cli *cliCapi) newStatusCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, nil)
+			hub, err := require.Hub(cli.cfg(), nil)
 			if err != nil {
 				return err
 			}

cmd/crowdsec-cli/cliconsole/console.go

@@ -114,7 +114,7 @@ func (cli *cliConsole) enroll(ctx context.Context, key string, name string, over
 		}
 	}
 
-	hub, err := require.Hub(cfg, nil, nil)
+	hub, err := require.Hub(cfg, nil)
 	if err != nil {
 		return err
 	}

cmd/crowdsec-cli/clihub/hub.go

@@ -93,7 +93,7 @@ func (cli *cliHub) newListCmd() *cobra.Command {
 		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
 		RunE: func(_ *cobra.Command, _ []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
+			hub, err := require.Hub(cli.cfg(), log.StandardLogger())
 			if err != nil {
 				return err
 			}
@@ -110,15 +110,15 @@ func (cli *cliHub) newListCmd() *cobra.Command {
 
 func (cli *cliHub) update(ctx context.Context, withContent bool) error {
 	local := cli.cfg().Hub
-	remote := require.RemoteHub(ctx, cli.cfg())
-
 	// don't use require.Hub because if there is no index file, it would fail
-	hub, err := cwhub.NewHub(local, remote, log.StandardLogger())
+	hub, err := cwhub.NewHub(local, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
-	if err := hub.Update(ctx, withContent); err != nil {
+	hubProvider := require.HubDownloader(ctx, cli.cfg())
+
+	if err := hub.Update(ctx, hubProvider, withContent); err != nil {
 		return fmt.Errorf("failed to update hub: %w", err)
 	}
 
@@ -166,16 +166,18 @@ cscli hub update --with-content`,
 func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force bool) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
+	hub, err := require.Hub(cfg, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
 	plan := hubops.NewActionPlan(hub)
 
+	hubProvider := require.HubDownloader(ctx, cfg)
+
 	for _, itemType := range cwhub.ItemTypes {
 		for _, item := range hub.GetInstalledByType(itemType, true) {
-			plan.AddCommand(hubops.NewDownloadCommand(item, force))
+			plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force))
 		}
 	}
 

cmd/crowdsec-cli/cliitem/item.go

@@ -71,13 +71,15 @@ func (cli cliItem) NewCommand() *cobra.Command {
 func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun bool, downloadOnly bool, force bool, ignoreError bool) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
+	hub, err := require.Hub(cfg, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
 	plan := hubops.NewActionPlan(hub)
 
+	hubProvider := require.HubDownloader(ctx, cfg)
+
 	for _, name := range args {
 		item := hub.GetItem(cli.name, name)
 		if item == nil {
@@ -91,7 +93,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun
 			continue
 		}
 
-		if err = plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil {
+		if err = plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil {
 			return err
 		}
 
@@ -228,7 +230,7 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b
 func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun bool, purge bool, force bool, all bool) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
+	hub, err := require.Hub(cli.cfg(), log.StandardLogger())
 	if err != nil {
 		return err
 	}
@@ -290,12 +292,12 @@ func (cli cliItem) newRemoveCmd() *cobra.Command {
 	return cmd
 }
 
-func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bool) (*hubops.ActionPlan, error) {
+func (cli cliItem) upgradePlan(hub *cwhub.Hub, hubProvider cwhub.HubProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) {
 	plan := hubops.NewActionPlan(hub)
 
 	if all {
 		for _, item := range hub.GetInstalledByType(cli.name, true) {
-			if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil {
+			if err := plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil {
 				return nil, err
 			}
 		}
@@ -313,7 +315,7 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bo
 			return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
 		}
 
-		if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil {
+		if err := plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil {
 			return nil, err
 		}
 	}
@@ -324,12 +326,14 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bo
 func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun bool, force bool, all bool) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
+	hub, err := require.Hub(cfg, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
-	plan, err := cli.upgradePlan(hub, args, force, all)
+	hubProvider := require.HubDownloader(ctx, cfg)
+
+	plan, err := cli.upgradePlan(hub, hubProvider, args, force, all)
 	if err != nil {
 		return err
 	}
@@ -390,13 +394,13 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff
 		cfg.Cscli.PrometheusUrl = url
 	}
 
-	remote := (*cwhub.RemoteHubCfg)(nil)
+	var hubProvider cwhub.HubProvider
 
 	if diff {
-		remote = require.RemoteHub(ctx, cfg)
+		hubProvider = require.HubDownloader(ctx, cfg)
 	}
 
-	hub, err := require.Hub(cfg, remote, log.StandardLogger())
+	hub, err := require.Hub(cfg, log.StandardLogger())
 	if err != nil {
 		return err
 	}
@@ -408,7 +412,7 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff
 		}
 
 		if diff {
-			fmt.Println(cli.whyTainted(ctx, hub, item, rev))
+			fmt.Println(cli.whyTainted(ctx, hub, hubProvider, item, rev))
 
 			continue
 		}
@@ -462,7 +466,7 @@ func (cli cliItem) newInspectCmd() *cobra.Command {
 func (cli cliItem) list(args []string, all bool) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
+	hub, err := require.Hub(cli.cfg(), log.StandardLogger())
 	if err != nil {
 		return err
 	}
@@ -498,7 +502,7 @@ func (cli cliItem) newListCmd() *cobra.Command {
 }
 
 // return the diff between the installed version and the latest version
-func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool) (string, error) {
+func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, hubProvider cwhub.HubProvider, reverse bool) (string, error) {
 	if !item.State.Installed {
 		return "", fmt.Errorf("'%s' is not installed", item.FQName())
 	}
@@ -509,7 +513,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool)
 	}
 	defer os.Remove(dest.Name())
 
-	_, remoteURL, err := item.FetchContentTo(ctx, dest.Name())
+	_, remoteURL, err := item.FetchContentTo(ctx, hubProvider, dest.Name())
 	if err != nil {
 		return "", err
 	}
@@ -540,7 +544,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool)
 	return fmt.Sprintf("%s", diff), nil
 }
 
-func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, item *cwhub.Item, reverse bool) string {
+func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, hubProvider cwhub.HubProvider, item *cwhub.Item, reverse bool) string {
 	if !item.State.Installed {
 		return fmt.Sprintf("# %s is not installed", item.FQName())
 	}
@@ -565,7 +569,7 @@ func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, item *cwhub.I
 			ret = append(ret, err.Error())
 		}
 
-		diff, err := cli.itemDiff(ctx, sub, reverse)
+		diff, err := cli.itemDiff(ctx, sub, hubProvider, reverse)
 		if err != nil {
 			ret = append(ret, err.Error())
 		}

cmd/crowdsec-cli/cliitem/suggest.go

@@ -37,7 +37,7 @@ func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) str
 }
 
 func compAllItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) {
-	hub, err := require.Hub(cfg(), nil, nil)
+	hub, err := require.Hub(cfg(), nil)
 	if err != nil {
 		return nil, cobra.ShellCompDirectiveDefault
 	}
@@ -56,7 +56,7 @@ func compAllItems(itemType string, args []string, toComplete string, cfg configG
 }
 
 func compInstalledItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) {
-	hub, err := require.Hub(cfg(), nil, nil)
+	hub, err := require.Hub(cfg(), nil)
 	if err != nil {
 		return nil, cobra.ShellCompDirectiveDefault
 	}

cmd/crowdsec-cli/clilapi/context.go

@@ -59,7 +59,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 		`,
 		DisableAutoGenTag: true,
 		RunE: func(_ *cobra.Command, _ []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, nil)
+			hub, err := require.Hub(cli.cfg(), nil)
 			if err != nil {
 				return err
 			}
@@ -101,7 +101,7 @@ func (cli *cliLapi) newContextStatusCmd() *cobra.Command {
 		DisableAutoGenTag: true,
 		RunE: func(_ *cobra.Command, _ []string) error {
 			cfg := cli.cfg()
-			hub, err := require.Hub(cfg, nil, nil)
+			hub, err := require.Hub(cfg, nil)
 			if err != nil {
 				return err
 			}
@@ -153,7 +153,7 @@ cscli lapi context detect crowdsecurity/sshd-logs
 				return fmt.Errorf("failed to init expr helpers: %w", err)
 			}
 
-			hub, err := require.Hub(cfg, nil, nil)
+			hub, err := require.Hub(cfg, nil)
 			if err != nil {
 				return err
 			}

cmd/crowdsec-cli/clilapi/status.go

@@ -102,7 +102,7 @@ func (cli *cliLapi) newStatusCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, nil)
+			hub, err := require.Hub(cli.cfg(), nil)
 			if err != nil {
 				return err
 			}

cmd/crowdsec-cli/clisetup/setup.go

@@ -289,14 +289,16 @@ func (cli *cliSetup) install(ctx context.Context, yes bool, dryRun bool, fromFil
 
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
+	hub, err := require.Hub(cfg, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
 	verbose := (cfg.Cscli.Output == "raw")
 
-	return setup.InstallHubItems(ctx, hub, input, yes, dryRun, verbose)
+	hubProvider := require.HubDownloader(ctx, cfg)
+
+	return setup.InstallHubItems(ctx, hub, hubProvider, input, yes, dryRun, verbose)
 }
 
 func (cli *cliSetup) validate(fromFile string) error {

cmd/crowdsec-cli/clisimulation/simulation.go

@@ -71,7 +71,7 @@ func (cli *cliSimulation) newEnableCmd() *cobra.Command {
 		Example:           `cscli simulation enable`,
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, nil)
+			hub, err := require.Hub(cli.cfg(), nil)
 			if err != nil {
 				return err
 			}

cmd/crowdsec-cli/clisupport/support.go

@@ -491,7 +491,7 @@ func (cli *cliSupport) dump(ctx context.Context, outFile string) error {
 		skipAgent = true
 	}
 
-	hub, err := require.Hub(cfg, nil, nil)
+	hub, err := require.Hub(cfg, nil)
 	if err != nil {
 		log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
 		// XXX: lapi status check requires scenarios, will return an error

cmd/crowdsec-cli/config_backup.go

@@ -15,7 +15,7 @@ import (
 )
 
 func (cli *cliConfig) backupHub(dirPath string) error {
-	hub, err := require.Hub(cli.cfg(), nil, nil)
+	hub, err := require.Hub(cli.cfg(), nil)
 	if err != nil {
 		return err
 	}

cmd/crowdsec-cli/config_restore.go

@@ -18,11 +18,13 @@ import (
 func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), nil)
+	hub, err := require.Hub(cfg, nil)
 	if err != nil {
 		return err
 	}
 
+	hubProvider := require.HubDownloader(ctx, cfg)
+
 	for _, itype := range cwhub.ItemTypes {
 		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
 		if _, err = os.Stat(itemDirectory); err != nil {
@@ -53,7 +55,7 @@ func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error {
 
 			plan := hubops.NewActionPlan(hub)
 
-			if err = plan.AddCommand(hubops.NewDownloadCommand(item, false)); err != nil {
+			if err = plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, false)); err != nil {
 				return err
 			}
 

cmd/crowdsec-cli/require/require.go

@@ -82,12 +82,11 @@ func Notifications(c *csconfig.Config) error {
 	return nil
 }
 
-// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
-func RemoteHub(ctx context.Context, c *csconfig.Config) *cwhub.RemoteHubCfg {
+func HubDownloader(ctx context.Context, c *csconfig.Config) *cwhub.Downloader {
 	// set branch in config, and log if necessary
 	branch := HubBranch(ctx, c)
 	urlTemplate := HubURLTemplate(c)
-	remote := &cwhub.RemoteHubCfg{
+	remote := &cwhub.Downloader{
 		Branch:      branch,
 		URLTemplate: urlTemplate,
 		IndexPath:   ".index.json",
@@ -98,7 +97,7 @@ func RemoteHub(ctx context.Context, c *csconfig.Config) *cwhub.RemoteHubCfg {
 
 // Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
 // If no remote parameter is provided, the hub can only be used for local operations.
-func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) {
+func Hub(c *csconfig.Config, logger *logrus.Logger) (*cwhub.Hub, error) {
 	local := c.Hub
 
 	if local == nil {
@@ -110,7 +109,7 @@ func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger)
 		logger.SetOutput(io.Discard)
 	}
 
-	hub, err := cwhub.NewHub(local, remote, logger)
+	hub, err := cwhub.NewHub(local, logger)
 	if err != nil {
 		return nil, err
 	}

cmd/crowdsec/serve.go

@@ -85,7 +85,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 	}
 
 	if !cConfig.DisableAgent {
-		hub, err := cwhub.NewHub(cConfig.Hub, nil, log.StandardLogger())
+		hub, err := cwhub.NewHub(cConfig.Hub, log.StandardLogger())
 		if err != nil {
 			return nil, err
 		}
@@ -387,7 +387,7 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error {
 	}
 
 	if !cConfig.DisableAgent {
-		hub, err := cwhub.NewHub(cConfig.Hub, nil, log.StandardLogger())
+		hub, err := cwhub.NewHub(cConfig.Hub, log.StandardLogger())
 		if err != nil {
 			return err
 		}

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/creack/pty v1.1.21 // indirect
 	github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
-	github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f
+	github.com/crowdsecurity/go-cs-lib v0.0.16
 	github.com/crowdsecurity/grokky v0.2.2
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1