diff --git a/cmd/crowdsec-cli/clihub/hub.go b/cmd/crowdsec-cli/clihub/hub.go index 160293746d5..83a9a7cdb9e 100644 --- a/cmd/crowdsec-cli/clihub/hub.go +++ b/cmd/crowdsec-cli/clihub/hub.go @@ -116,9 +116,9 @@ func (cli *cliHub) update(ctx context.Context, withContent bool) error { return err } - hubProvider := require.HubDownloader(ctx, cli.cfg()) + indexProvider := require.HubDownloader(ctx, cli.cfg()) - if err := hub.Update(ctx, hubProvider, withContent); err != nil { + if err := hub.Update(ctx, indexProvider, withContent); err != nil { return fmt.Errorf("failed to update hub: %w", err) } @@ -173,11 +173,11 @@ func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force boo plan := hubops.NewActionPlan(hub) - hubProvider := require.HubDownloader(ctx, cfg) + contentProvider := require.HubDownloader(ctx, cfg) for _, itemType := range cwhub.ItemTypes { for _, item := range hub.GetInstalledByType(itemType, true) { - plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)) + plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)) } } diff --git a/cmd/crowdsec-cli/cliitem/item.go b/cmd/crowdsec-cli/cliitem/item.go index 55e39030f76..04a6735316a 100644 --- a/cmd/crowdsec-cli/cliitem/item.go +++ b/cmd/crowdsec-cli/cliitem/item.go @@ -78,7 +78,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun plan := hubops.NewActionPlan(hub) - hubProvider := require.HubDownloader(ctx, cfg) + contentProvider := require.HubDownloader(ctx, cfg) for _, name := range args { item := hub.GetItem(cli.name, name) @@ -93,7 +93,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun continue } - if err = plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil { + if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { return err } @@ -292,12 +292,12 @@ func (cli cliItem) newRemoveCmd() *cobra.Command { return cmd } -func (cli cliItem) upgradePlan(hub *cwhub.Hub, hubProvider cwhub.HubProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) { +func (cli cliItem) upgradePlan(hub *cwhub.Hub, contentProvider cwhub.ContentProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) { plan := hubops.NewActionPlan(hub) if all { for _, item := range hub.GetInstalledByType(cli.name, true) { - if err := plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil { + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { return nil, err } } @@ -315,7 +315,7 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, hubProvider cwhub.HubProvider, ar return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name) } - if err := plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, force)); err != nil { + if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil { return nil, err } } @@ -331,9 +331,9 @@ func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun return err } - hubProvider := require.HubDownloader(ctx, cfg) + contentProvider := require.HubDownloader(ctx, cfg) - plan, err := cli.upgradePlan(hub, hubProvider, args, force, all) + plan, err := cli.upgradePlan(hub, contentProvider, args, force, all) if err != nil { return err } @@ -394,10 +394,10 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff cfg.Cscli.PrometheusUrl = url } - var hubProvider cwhub.HubProvider + var contentProvider cwhub.ContentProvider if diff { - hubProvider = require.HubDownloader(ctx, cfg) + contentProvider = require.HubDownloader(ctx, cfg) } hub, err := require.Hub(cfg, log.StandardLogger()) @@ -412,7 +412,7 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff } if diff { - fmt.Println(cli.whyTainted(ctx, hub, hubProvider, item, rev)) + fmt.Println(cli.whyTainted(ctx, hub, contentProvider, item, rev)) continue } @@ -502,7 +502,7 @@ func (cli cliItem) newListCmd() *cobra.Command { } // return the diff between the installed version and the latest version -func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, hubProvider cwhub.HubProvider, reverse bool) (string, error) { +func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, contentProvider cwhub.ContentProvider, reverse bool) (string, error) { if !item.State.Installed { return "", fmt.Errorf("'%s' is not installed", item.FQName()) } @@ -513,7 +513,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, hubProvider c } defer os.Remove(dest.Name()) - _, remoteURL, err := item.FetchContentTo(ctx, hubProvider, dest.Name()) + _, remoteURL, err := item.FetchContentTo(ctx, contentProvider, dest.Name()) if err != nil { return "", err } @@ -544,7 +544,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, hubProvider c return fmt.Sprintf("%s", diff), nil } -func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, hubProvider cwhub.HubProvider, item *cwhub.Item, reverse bool) string { +func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, item *cwhub.Item, reverse bool) string { if !item.State.Installed { return fmt.Sprintf("# %s is not installed", item.FQName()) } @@ -569,7 +569,7 @@ func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, hubProvider c ret = append(ret, err.Error()) } - diff, err := cli.itemDiff(ctx, sub, hubProvider, reverse) + diff, err := cli.itemDiff(ctx, sub, contentProvider, reverse) if err != nil { ret = append(ret, err.Error()) } diff --git a/cmd/crowdsec-cli/clisetup/setup.go b/cmd/crowdsec-cli/clisetup/setup.go index a189e56e704..0de85e63548 100644 --- a/cmd/crowdsec-cli/clisetup/setup.go +++ b/cmd/crowdsec-cli/clisetup/setup.go @@ -296,9 +296,9 @@ func (cli *cliSetup) install(ctx context.Context, yes bool, dryRun bool, fromFil verbose := (cfg.Cscli.Output == "raw") - hubProvider := require.HubDownloader(ctx, cfg) + contentProvider := require.HubDownloader(ctx, cfg) - return setup.InstallHubItems(ctx, hub, hubProvider, input, yes, dryRun, verbose) + return setup.InstallHubItems(ctx, hub, contentProvider, input, yes, dryRun, verbose) } func (cli *cliSetup) validate(fromFile string) error { diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index e9221df9f9e..b5fbf36b2b4 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -23,7 +23,7 @@ func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error { return err } - hubProvider := require.HubDownloader(ctx, cfg) + contentProvider := require.HubDownloader(ctx, cfg) for _, itype := range cwhub.ItemTypes { itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) @@ -55,7 +55,7 @@ func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error { plan := hubops.NewActionPlan(hub) - if err = plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, false)); err != nil { + if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, false)); err != nil { return err } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 5e57a13a46d..94a1d6ef6fd 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -61,14 +61,14 @@ func testHub(t *testing.T, update bool) *Hub { require.NoError(t, err) if update { - hubProvider := &Downloader{ + indexProvider := &Downloader{ Branch: "master", URLTemplate: mockURLTemplate, IndexPath: ".index.json", } ctx := context.Background() - err := hub.Update(ctx, hubProvider, false) + err := hub.Update(ctx, indexProvider, false) require.NoError(t, err) } diff --git a/pkg/cwhub/doc.go b/pkg/cwhub/doc.go index f893e1fb1a9..b85d7634da4 100644 --- a/pkg/cwhub/doc.go +++ b/pkg/cwhub/doc.go @@ -87,7 +87,7 @@ // // Some commands require an object to provide the hub index, or contents: // -// hubProvider := cwhub.Downloader{ +// indexProvider := cwhub.Downloader{ // URLTemplate: "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s", // Branch: "master", // IndexPath: ".index.json", @@ -98,7 +98,7 @@ // // Before calling hub.Load(), you can update the index file by calling the Update() method: // -// err := hub.Update(context.Background(), hubProvider) +// err := hub.Update(context.Background(), indexProvider) // if err != nil { // return fmt.Errorf("unable to update hub index: %w", err) // } diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 0ed419d4f6d..2c97d74015f 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -12,13 +12,24 @@ import ( "github.com/crowdsecurity/go-cs-lib/downloader" ) -// Downloader is used to retrieve index and items from a remote hub. +// Downloader is used to retrieve index and items from a remote hub, with cache control. type Downloader struct { Branch string URLTemplate string IndexPath string } +// IndexProvider retrieves and writes .index.json +type IndexProvider interface { + FetchIndex(ctx context.Context, indexFile string, withContent bool, logger *logrus.Logger) (bool, error) +} + +// ContentProvider retrieves and writes the YAML files with the item content. +type ContentProvider interface { + FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error) +} + + // urlTo builds the URL to download a file from the remote hub. func (d *Downloader) urlTo(remotePath string) (string, error) { // the template must contain two string placeholders @@ -47,7 +58,9 @@ func addURLParam(rawURL string, param string, value string) (string, error) { return parsedURL.String(), nil } -// FetchIndex downloads the index from the hub and returns the content. +// FetchIndex downloads the index from the hub and writes it to the filesystem. +// It uses a temporary file to avoid partial downloads, and won't overwrite the original +// if it has not changed. func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withContent bool, logger *logrus.Logger) (bool, error) { url, err := d.urlTo(d.IndexPath) if err != nil { @@ -79,7 +92,9 @@ func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withConten return downloaded, nil } -// FetchContent downloads the content to the specified path and checks the hash. +// FetchContent downloads the content to the specified path, through a temporary file +// to avoid partial downloads. +// If the hash does not match, it will not overwrite and log a warning. func (d *Downloader) FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error) { url, err := d.urlTo(remotePath) if err != nil { diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go new file mode 100644 index 00000000000..edfed469157 --- /dev/null +++ b/pkg/cwhub/download_test.go @@ -0,0 +1,50 @@ +package cwhub + +import ( + "context" + "io" + "os" + "testing" + "net/http" + "net/http/httptest" + "path/filepath" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetchIndex(t *testing.T) { + ctx := context.Background() + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("with_content") == "true" { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`Hi I'm an index with content`)) + } else { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`Hi I'm a regular index`)) + } + })) + defer mockServer.Close() + + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", + IndexPath: "index.txt", + } + + logger := logrus.New() + logger.Out = io.Discard + + destPath := filepath.Join(t.TempDir(), "index.txt") + withContent := true + + downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, logger) + require.NoError(t, err) + assert.True(t, downloaded) + + content, err := os.ReadFile(destPath) + require.NoError(t, err) + assert.Equal(t, "Hi I'm an index with content", string(content)) +} diff --git a/pkg/cwhub/fetch.go b/pkg/cwhub/fetch.go index 138e0e5e6a2..e8f0b395325 100644 --- a/pkg/cwhub/fetch.go +++ b/pkg/cwhub/fetch.go @@ -50,7 +50,7 @@ func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error { // FetchContentTo writes the last version of the item's YAML file to the specified path. // Returns whether the file was downloaded, and the remote url for feedback purposes. -func (i *Item) FetchContentTo(ctx context.Context, hubProvider HubProvider, destPath string) (bool, string, error) { +func (i *Item) FetchContentTo(ctx context.Context, contentProvider ContentProvider, destPath string) (bool, string, error) { wantHash := i.latestHash() if wantHash == "" { return false, "", fmt.Errorf("%s: latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again", i.FQName()) @@ -65,5 +65,5 @@ func (i *Item) FetchContentTo(ctx context.Context, hubProvider HubProvider, dest return true, fmt.Sprintf("(embedded in %s)", i.hub.local.HubIndexFile), nil } - return hubProvider.FetchContent(ctx, i.RemotePath, destPath, wantHash, i.hub.logger) + return contentProvider.FetchContent(ctx, i.RemotePath, destPath, wantHash, i.hub.logger) } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index ce56a780815..5e0eda9b3fa 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -153,20 +153,15 @@ func (h *Hub) ItemStats() []string { return ret } -type HubProvider interface { - FetchIndex(ctx context.Context, indexFile string, withContent bool, logger *logrus.Logger) (bool, error) - FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error) -} - // Update downloads the latest version of the index and writes it to disk if it changed. // It cannot be called after Load() unless the hub is completely empty. -func (h *Hub) Update(ctx context.Context, hubProvider HubProvider, withContent bool) error { +func (h *Hub) Update(ctx context.Context, indexProvider IndexProvider, withContent bool) error { if len(h.pathIndex) > 0 { // if this happens, it's a bug. return errors.New("cannot update hub after items have been loaded") } - downloaded, err := hubProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger) + downloaded, err := indexProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger) if err != nil { return err } diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index dbde5a8a623..c2b949b7cdf 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -19,13 +19,13 @@ func TestInitHubUpdate(t *testing.T) { ctx := context.Background() - hubProvider := &Downloader{ + indexProvider := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", IndexPath: ".index.json", } - err = hub.Update(ctx, hubProvider, false) + err = hub.Update(ctx, indexProvider, false) require.NoError(t, err) err = hub.Load() @@ -53,25 +53,25 @@ func TestUpdateIndex(t *testing.T) { ctx := context.Background() - hubProvider := &Downloader{ + indexProvider := &Downloader{ URLTemplate: "x", Branch: "", IndexPath: "", } - err = hub.Update(ctx, hubProvider, false) + err = hub.Update(ctx, indexProvider, false) cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") // bad domain fmt.Println("Test 'bad domain'") - hubProvider = &Downloader{ + indexProvider = &Downloader{ URLTemplate: "https://baddomain/crowdsecurity/%s/%s", Branch: "master", IndexPath: ".index.json", } - err = hub.Update(ctx, hubProvider, false) + err = hub.Update(ctx, indexProvider, false) require.NoError(t, err) // XXX: this is not failing // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") @@ -79,7 +79,7 @@ func TestUpdateIndex(t *testing.T) { // bad target path fmt.Println("Test 'bad target path'") - hubProvider = &Downloader{ + indexProvider = &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", IndexPath: ".index.json", @@ -87,6 +87,6 @@ func TestUpdateIndex(t *testing.T) { hub.local.HubIndexFile = "/does/not/exist/index.json" - err = hub.Update(ctx, hubProvider, false) + err = hub.Update(ctx, indexProvider, false) cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:") } diff --git a/pkg/hubops/download.go b/pkg/hubops/download.go index 6ed173ac620..49895963468 100644 --- a/pkg/hubops/download.go +++ b/pkg/hubops/download.go @@ -28,11 +28,11 @@ import ( type DownloadCommand struct { Item *cwhub.Item Force bool - hubProvider cwhub.HubProvider + contentProvider cwhub.ContentProvider } -func NewDownloadCommand(item *cwhub.Item, hubProvider cwhub.HubProvider, force bool) *DownloadCommand { - return &DownloadCommand{Item: item, Force: force, hubProvider: hubProvider} +func NewDownloadCommand(item *cwhub.Item, contentProvider cwhub.ContentProvider, force bool) *DownloadCommand { + return &DownloadCommand{Item: item, Force: force, contentProvider: contentProvider} } func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) { @@ -61,7 +61,7 @@ func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) { } for sub := range i.LatestDependencies().SubItems(plan.hub) { - if err := plan.AddCommand(NewDownloadCommand(sub, c.hubProvider, c.Force)); err != nil { + if err := plan.AddCommand(NewDownloadCommand(sub, c.contentProvider, c.Force)); err != nil { return false, err } @@ -159,7 +159,7 @@ func (c *DownloadCommand) Run(ctx context.Context, plan *ActionPlan) error { return err } - downloaded, _, err := i.FetchContentTo(ctx, c.hubProvider, finalPath) + downloaded, _, err := i.FetchContentTo(ctx, c.contentProvider, finalPath) if err != nil { return fmt.Errorf("%s: %w", i.FQName(), err) } diff --git a/pkg/setup/install.go b/pkg/setup/install.go index d90b6177e00..d1d3dbc4262 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -48,7 +48,7 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) { } // InstallHubItems installs the objects recommended in a setup file. -func InstallHubItems(ctx context.Context, hub *cwhub.Hub, hubProvider cwhub.HubProvider, input []byte, yes, dryRun, verbose bool) error { +func InstallHubItems(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, input []byte, yes, dryRun, verbose bool) error { setupEnvelope, err := decodeSetup(input, false) if err != nil { return err @@ -71,7 +71,7 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, hubProvider cwhub.HubP return fmt.Errorf("collection %s not found", collection) } - plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } @@ -83,7 +83,7 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, hubProvider cwhub.HubP return fmt.Errorf("parser %s not found", parser) } - plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } @@ -95,7 +95,7 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, hubProvider cwhub.HubP return fmt.Errorf("scenario %s not found", scenario) } - plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) } @@ -107,7 +107,7 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, hubProvider cwhub.HubP return fmt.Errorf("postoverflow %s not found", postoverflow) } - plan.AddCommand(hubops.NewDownloadCommand(item, hubProvider, forceAction)) + plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction)) if !downloadOnly { plan.AddCommand(hubops.NewEnableCommand(item, forceAction)) }