single method interfaces

cmd/crowdsec-cli/clihub/hub.go

@@ -116,9 +116,9 @@ func (cli *cliHub) update(ctx context.Context, withContent bool) error {
 		return err
 	}
 
-	hubProvider := require.HubDownloader(ctx, cli.cfg())
+	indexProvider := require.HubDownloader(ctx, cli.cfg())
 
-	if err := hub.Update(ctx, hubProvider, withContent); err != nil {
+	if err := hub.Update(ctx, indexProvider, withContent); err != nil {
 		return fmt.Errorf("failed to update hub: %w", err)
 	}
 
@@ -173,11 +173,11 @@ func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force boo
 
 	plan := hubops.NewActionPlan(hub)
 
-	hubProvider := require.HubDownloader(ctx, cfg)
+	contentProvider := require.HubDownloader(ctx, cfg)
 
 	for _, itemType := range cwhub.ItemTypes {
 		for _, item := range hub.GetInstalledByType(itemType, true) {
-			plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force))
+			plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force))
 		}
 	}
 

cmd/crowdsec-cli/cliitem/item.go

@@ -78,7 +78,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun
 
 	plan := hubops.NewActionPlan(hub)
 
-	hubProvider := require.HubDownloader(ctx, cfg)
+	contentProvider := require.HubDownloader(ctx, cfg)
 
 	for _, name := range args {
 		item := hub.GetItem(cli.name, name)
@@ -93,7 +93,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun
 			continue
 		}
 
-		if err = plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil {
+		if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil {
 			return err
 		}
 
@@ -292,12 +292,12 @@ func (cli cliItem) newRemoveCmd() *cobra.Command {
 	return cmd
 }
 
-func (cli cliItem) upgradePlan(hub *cwhub.Hub, hubProvider cwhub.HubProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) {
+func (cli cliItem) upgradePlan(hub *cwhub.Hub, contentProvider cwhub.ContentProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) {
 	plan := hubops.NewActionPlan(hub)
 
 	if all {
 		for _, item := range hub.GetInstalledByType(cli.name, true) {
-			if err := plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil {
+			if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil {
 				return nil, err
 			}
 		}
@@ -315,7 +315,7 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, hubProvider cwhub.HubProvider, ar
 			return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
 		}
 
-		if err := plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil {
+		if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil {
 			return nil, err
 		}
 	}
@@ -331,9 +331,9 @@ func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun
 		return err
 	}
 
-	hubProvider := require.HubDownloader(ctx, cfg)
+	contentProvider := require.HubDownloader(ctx, cfg)
 
-	plan, err := cli.upgradePlan(hub, hubProvider, args, force, all)
+	plan, err := cli.upgradePlan(hub, contentProvider, args, force, all)
 	if err != nil {
 		return err
 	}
@@ -394,10 +394,10 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff
 		cfg.Cscli.PrometheusUrl = url
 	}
 
-	var hubProvider cwhub.HubProvider
+	var contentProvider cwhub.ContentProvider
 
 	if diff {
-		hubProvider = require.HubDownloader(ctx, cfg)
+		contentProvider = require.HubDownloader(ctx, cfg)
 	}
 
 	hub, err := require.Hub(cfg, log.StandardLogger())
@@ -412,7 +412,7 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff
 		}
 
 		if diff {
-			fmt.Println(cli.whyTainted(ctx, hub, hubProvider, item, rev))
+			fmt.Println(cli.whyTainted(ctx, hub, contentProvider, item, rev))
 
 			continue
 		}
@@ -502,7 +502,7 @@ func (cli cliItem) newListCmd() *cobra.Command {
 }
 
 // return the diff between the installed version and the latest version
-func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, hubProvider cwhub.HubProvider, reverse bool) (string, error) {
+func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, contentProvider cwhub.ContentProvider, reverse bool) (string, error) {
 	if !item.State.Installed {
 		return "", fmt.Errorf("'%s' is not installed", item.FQName())
 	}
@@ -513,7 +513,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, hubProvider c
 	}
 	defer os.Remove(dest.Name())
 
-	_, remoteURL, err := item.FetchContentTo(ctx, hubProvider, dest.Name())
+	_, remoteURL, err := item.FetchContentTo(ctx, contentProvider, dest.Name())
 	if err != nil {
 		return "", err
 	}
@@ -544,7 +544,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, hubProvider c
 	return fmt.Sprintf("%s", diff), nil
 }
 
-func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, hubProvider cwhub.HubProvider, item *cwhub.Item, reverse bool) string {
+func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, item *cwhub.Item, reverse bool) string {
 	if !item.State.Installed {
 		return fmt.Sprintf("# %s is not installed", item.FQName())
 	}
@@ -569,7 +569,7 @@ func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, hubProvider c
 			ret = append(ret, err.Error())
 		}
 
-		diff, err := cli.itemDiff(ctx, sub, hubProvider, reverse)
+		diff, err := cli.itemDiff(ctx, sub, contentProvider, reverse)
 		if err != nil {
 			ret = append(ret, err.Error())
 		}

cmd/crowdsec-cli/clisetup/setup.go

@@ -296,9 +296,9 @@ func (cli *cliSetup) install(ctx context.Context, yes bool, dryRun bool, fromFil
 
 	verbose := (cfg.Cscli.Output == "raw")
 
-	hubProvider := require.HubDownloader(ctx, cfg)
+	contentProvider := require.HubDownloader(ctx, cfg)
 
-	return setup.InstallHubItems(ctx, hub, hubProvider, input, yes, dryRun, verbose)
+	return setup.InstallHubItems(ctx, hub, contentProvider, input, yes, dryRun, verbose)
 }
 
 func (cli *cliSetup) validate(fromFile string) error {

cmd/crowdsec-cli/config_restore.go

@@ -23,7 +23,7 @@ func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error {
 		return err
 	}
 
-	hubProvider := require.HubDownloader(ctx, cfg)
+	contentProvider := require.HubDownloader(ctx, cfg)
 
 	for _, itype := range cwhub.ItemTypes {
 		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
@@ -55,7 +55,7 @@ func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error {
 
 			plan := hubops.NewActionPlan(hub)
 
-			if err = plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, false)); err != nil {
+			if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, false)); err != nil {
 				return err
 			}
 

pkg/cwhub/cwhub_test.go

@@ -61,14 +61,14 @@ func testHub(t *testing.T, update bool) *Hub {
 	require.NoError(t, err)
 
 	if update {
-		hubProvider := &Downloader{
+		indexProvider := &Downloader{
 			Branch:      "master",
 			URLTemplate: mockURLTemplate,
 			IndexPath:   ".index.json",
 		}
 
 		ctx := context.Background()
-		err := hub.Update(ctx, hubProvider, false)
+		err := hub.Update(ctx, indexProvider, false)
 		require.NoError(t, err)
 	}
 

pkg/cwhub/doc.go

@@ -87,7 +87,7 @@
 //
 // Some commands require an object to provide the hub index, or contents:
 //
-//	hubProvider := cwhub.Downloader{
+//	indexProvider := cwhub.Downloader{
 //		URLTemplate: "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s",
 //		Branch: "master",
 //		IndexPath: ".index.json",
@@ -98,7 +98,7 @@
 //
 // Before calling hub.Load(), you can update the index file by calling the Update() method:
 //
-//	err := hub.Update(context.Background(), hubProvider)
+//	err := hub.Update(context.Background(), indexProvider)
 //	if err != nil {
 //		return fmt.Errorf("unable to update hub index: %w", err)
 //	}

pkg/cwhub/download.go

@@ -12,13 +12,24 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/downloader"
 )
 
-// Downloader is used to retrieve index and items from a remote hub.
+// Downloader is used to retrieve index and items from a remote hub, with cache control.
 type Downloader struct {
 	Branch           string
 	URLTemplate      string
 	IndexPath        string
 }
 
+// IndexProvider retrieves and writes .index.json
+type IndexProvider interface {
+	FetchIndex(ctx context.Context, indexFile string, withContent bool, logger *logrus.Logger) (bool, error)
+}
+
+// ContentProvider retrieves and writes the YAML files with the item content.
+type ContentProvider interface {
+	FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error)
+}
+
+
 // urlTo builds the URL to download a file from the remote hub.
 func (d *Downloader) urlTo(remotePath string) (string, error) {
 	// the template must contain two string placeholders
@@ -47,7 +58,9 @@ func addURLParam(rawURL string, param string, value string) (string, error) {
 	return parsedURL.String(), nil
 }
 
-// FetchIndex downloads the index from the hub and returns the content.
+// FetchIndex downloads the index from the hub and writes it to the filesystem.
+// It uses a temporary file to avoid partial downloads, and won't overwrite the original
+// if it has not changed.
 func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withContent bool, logger *logrus.Logger) (bool, error) {
 	url, err := d.urlTo(d.IndexPath)
 	if err != nil {
@@ -79,7 +92,9 @@ func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withConten
 	return downloaded, nil
 }
 
-// FetchContent downloads the content to the specified path and checks the hash.
+// FetchContent downloads the content to the specified path, through a temporary file
+// to avoid partial downloads.
+// If the hash does not match, it will not overwrite and log a warning.
 func (d *Downloader) FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error) {
 	url, err := d.urlTo(remotePath)
 	if err != nil {

pkg/cwhub/download_test.go

@@ -0,0 +1,50 @@
+package cwhub
+
+import (
+	"context"
+	"io"
+	"os"
+	"testing"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFetchIndex(t *testing.T) {
+	ctx := context.Background()
+
+	mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("with_content") == "true" {
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(`Hi I'm an index with content`))
+		} else {
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(`Hi I'm a regular index`))
+		}
+	}))
+	defer mockServer.Close()
+
+	downloader := &Downloader{
+		Branch:      "main",
+		URLTemplate: mockServer.URL + "/%s/%s",
+		IndexPath:   "index.txt",
+	}
+
+	logger := logrus.New()
+	logger.Out = io.Discard
+
+	destPath := filepath.Join(t.TempDir(), "index.txt")
+	withContent := true
+
+	downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, logger)
+	require.NoError(t, err)
+	assert.True(t, downloaded)
+
+	content, err := os.ReadFile(destPath)
+	require.NoError(t, err)
+	assert.Equal(t, "Hi I'm an index with content", string(content))
+}

pkg/cwhub/fetch.go

@@ -50,7 +50,7 @@ func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error {
 
 // FetchContentTo writes the last version of the item's YAML file to the specified path.
 // Returns whether the file was downloaded, and the remote url for feedback purposes.
-func (i *Item) FetchContentTo(ctx context.Context, hubProvider HubProvider, destPath string) (bool, string, error) {
+func (i *Item) FetchContentTo(ctx context.Context, contentProvider ContentProvider, destPath string) (bool, string, error) {
 	wantHash := i.latestHash()
 	if wantHash == "" {
 		return false, "", fmt.Errorf("%s: latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again", i.FQName())
@@ -65,5 +65,5 @@ func (i *Item) FetchContentTo(ctx context.Context, hubProvider HubProvider, dest
 		return true, fmt.Sprintf("(embedded in %s)", i.hub.local.HubIndexFile), nil
 	}
 
-	return hubProvider.FetchContent(ctx, i.RemotePath, destPath, wantHash, i.hub.logger)
+	return contentProvider.FetchContent(ctx, i.RemotePath, destPath, wantHash, i.hub.logger)
 }

pkg/cwhub/hub.go

@@ -153,20 +153,15 @@ func (h *Hub) ItemStats() []string {
 	return ret
 }
 
-type HubProvider interface {
-	FetchIndex(ctx context.Context, indexFile string, withContent bool, logger *logrus.Logger) (bool, error)
-	FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error)
-}
-
 // Update downloads the latest version of the index and writes it to disk if it changed.
 // It cannot be called after Load() unless the hub is completely empty.
-func (h *Hub) Update(ctx context.Context, hubProvider HubProvider, withContent bool) error {
+func (h *Hub) Update(ctx context.Context, indexProvider IndexProvider, withContent bool) error {
 	if len(h.pathIndex) > 0 {
 		// if this happens, it's a bug.
 		return errors.New("cannot update hub after items have been loaded")
 	}
 
-	downloaded, err := hubProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger)
+	downloaded, err := indexProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger)
 	if err != nil {
 		return err
 	}