Merge branch 'master' into windows-evtx-replay

.github/codecov-ignore-generated.sh → .github/generate-codecov-yml.sh

@@ -2,11 +2,14 @@
 
 # Run this from the repository root:
 #
-# .github/codecov-ignore-generated.sh >> .github/codecov.yml
+# .github/generate-codecov-yml.sh >> .github/codecov.yml
 
 cat <<EOT
 # we measure coverage but don't enforce it
 # https://docs.codecov.com/docs/codecov-yaml
+codecov:
+  require_ci_to_pass: false
+
 coverage:
   status:
     patch:

.github/workflows/bats-sqlite-coverage.yml

@@ -43,6 +43,10 @@ jobs:
       run: |
         make clean bats-build bats-fixture BUILD_STATIC=1
 
+    - name: Generate codecov configuration
+      run: |
+          .github/generate-codecov-yml.sh >> .github/codecov.yml
+
     - name: "Run tests"
       run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
 
@@ -79,10 +83,6 @@ jobs:
       run: for file in $(find ./test/local/var/log -type f); do echo ">>>>> $file"; cat $file; echo; done
       if: ${{ always() }}
 
-    - name: Ignore-list of generated files for codecov
-      run: |
-          .github/codecov-ignore-generated.sh >> .github/codecov.yml
-
     - name: Upload bats coverage to codecov
       uses: codecov/codecov-action@v4
       with:

.github/workflows/go-tests-windows.yml

@@ -40,17 +40,17 @@ jobs:
       run: |
         make build BUILD_RE2_WASM=1
 
+    - name: Generate codecov configuration
+      run: |
+          .github/generate-codecov-yml.sh >> .github/codecov.yml
+
     - name: Run tests
       run: |
         go install github.com/kyoh86/[email protected]
         go test -tags expr_debug -coverprofile coverage.out -covermode=atomic ./... > out.txt
         if(!$?) { cat out.txt | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter; Exit 1 }
         cat out.txt | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
-    - name: Ignore-list of generated files for codecov
-      run: |
-          .github/codecov-ignore-generated.sh >> .github/codecov.yml
-
     - name: Upload unit coverage to Codecov
       uses: codecov/codecov-action@v4
       with:

.github/workflows/go-tests.yml

@@ -128,10 +128,21 @@ jobs:
       with:
         go-version: "1.22"
 
-    - name: Run "make generate" and check for changes
+    - name: Run "go generate" and check for changes
       run: |
         set -e
-        make generate 2>/dev/null
+        # ensure the version of 'protoc' matches the one that generated the files
+        PROTOBUF_VERSION="21.12"
+        # don't pollute the repo
+        pushd $HOME
+        curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
+        unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d $HOME/.protoc
+        popd
+        export PATH="$HOME/.protoc/bin:$PATH"
+        go install google.golang.org/protobuf/cmd/[email protected]
+        go install google.golang.org/grpc/cmd/[email protected]
+        go generate ./...
+        protoc --version
         if [[ $(git status --porcelain) ]]; then
           echo "Error: Uncommitted changes found after running 'make generate'. Please commit all generated code."
           git diff
@@ -145,6 +156,10 @@ jobs:
           aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
           aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
 
+    - name: Generate codecov configuration
+      run: |
+          .github/generate-codecov-yml.sh >> .github/codecov.yml
+
     - name: Build and run tests, static
       run: |
         sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
@@ -165,10 +180,6 @@ jobs:
         set -o pipefail
         make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
-    - name: Ignore-list of generated files for codecov
-      run: |
-          .github/codecov-ignore-generated.sh >> .github/codecov.yml
-
     - name: Upload unit coverage to Codecov
       uses: codecov/codecov-action@v4
       with:

.golangci.yml

@@ -321,7 +321,7 @@ issues:
     # `err` is often shadowed, we may continue to do it
     - linters:
         - govet
-      text: "shadow: declaration of \"err\" shadows declaration"
+      text: "shadow: declaration of \"(err|ctx)\" shadows declaration"
 
     - linters:
         - errcheck

Makefile

@@ -263,12 +263,6 @@ cscli:  ## Build cscli
 crowdsec:  ## Build crowdsec
 	@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
 
-.PHONY: generate
-generate:  ## Generate code for the database and APIs
-	$(GO) generate ./pkg/database/ent
-	$(GO) generate ./pkg/models
-	$(GO) generate ./pkg/modelscapi
-
 .PHONY: testclean
 testclean: bats-clean  ## Remove test artifacts
 	@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)

cmd/crowdsec-cli/clinotifications/notifications.go

@@ -275,7 +275,8 @@ func (cli cliNotifications) newTestCmd() *cobra.Command {
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
 		ValidArgsFunction: cli.notificationConfigFilter,
-		PreRunE: func(_ *cobra.Command, args []string) error {
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
 			cfg := cli.cfg()
 			pconfigs, err := cli.getPluginConfigs()
 			if err != nil {
@@ -286,7 +287,7 @@ func (cli cliNotifications) newTestCmd() *cobra.Command {
 				return fmt.Errorf("plugin name: '%s' does not exist", args[0])
 			}
 			// Create a single profile with plugin name as notification name
-			return pluginBroker.Init(cfg.PluginConfig, []*csconfig.ProfileCfg{
+			return pluginBroker.Init(ctx, cfg.PluginConfig, []*csconfig.ProfileCfg{
 				{
 					Notifications: []string{
 						pcfg.Name,
@@ -377,12 +378,13 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
 
 			return nil
 		},
-		RunE: func(_ *cobra.Command, _ []string) error {
+		RunE: func(cmd *cobra.Command, _ []string) error {
 			var (
 				pluginBroker csplugin.PluginBroker
 				pluginTomb   tomb.Tomb
 			)
 
+			ctx := cmd.Context()
 			cfg := cli.cfg()
 
 			if alertOverride != "" {
@@ -391,7 +393,7 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
 				}
 			}
 
-			err := pluginBroker.Init(cfg.PluginConfig, cfg.API.Server.Profiles, cfg.ConfigPaths)
+			err := pluginBroker.Init(ctx, cfg.PluginConfig, cfg.API.Server.Profiles, cfg.ConfigPaths)
 			if err != nil {
 				return fmt.Errorf("can't initialize plugins: %w", err)
 			}

cmd/crowdsec-cli/clipapi/papi.go

@@ -127,7 +127,7 @@ func (cli *cliPapi) sync(ctx context.Context, out io.Writer, db *database.Client
 		return fmt.Errorf("unable to initialize API client: %w", err)
 	}
 
-	t.Go(apic.Push)
+	t.Go(func() error { return apic.Push(ctx) })
 
 	papi, err := apiserver.NewPAPI(apic, db, cfg.API.Server.ConsoleConfig, log.GetLevel())
 	if err != nil {

cmd/crowdsec/api.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"runtime"
@@ -14,12 +15,12 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
-func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
+func initAPIServer(ctx context.Context, cConfig *csconfig.Config) (*apiserver.APIServer, error) {
 	if cConfig.API.Server.OnlineClient == nil || cConfig.API.Server.OnlineClient.Credentials == nil {
 		log.Info("push and pull to Central API disabled")
 	}
 
-	apiServer, err := apiserver.NewServer(cConfig.API.Server)
+	apiServer, err := apiserver.NewServer(ctx, cConfig.API.Server)
 	if err != nil {
 		return nil, fmt.Errorf("unable to run local API: %w", err)
 	}
@@ -39,7 +40,7 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
 			return nil, errors.New("plugins are enabled, but config_paths.plugin_dir is not defined")
 		}
 
-		err = pluginBroker.Init(cConfig.PluginConfig, cConfig.API.Server.Profiles, cConfig.ConfigPaths)
+		err = pluginBroker.Init(ctx, cConfig.PluginConfig, cConfig.API.Server.Profiles, cConfig.ConfigPaths)
 		if err != nil {
 			return nil, fmt.Errorf("unable to run plugin broker: %w", err)
 		}
@@ -58,11 +59,14 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
 
 func serveAPIServer(apiServer *apiserver.APIServer) {
 	apiReady := make(chan bool, 1)
+
 	apiTomb.Go(func() error {
 		defer trace.CatchPanic("crowdsec/serveAPIServer")
+
 		go func() {
 			defer trace.CatchPanic("crowdsec/runAPIServer")
 			log.Debugf("serving API after %s ms", time.Since(crowdsecT0))
+
 			if err := apiServer.Run(apiReady); err != nil {
 				log.Fatal(err)
 			}
@@ -76,6 +80,7 @@ func serveAPIServer(apiServer *apiserver.APIServer) {
 		<-apiTomb.Dying() // lock until go routine is dying
 		pluginTomb.Kill(nil)
 		log.Infof("serve: shutting down api server")
+
 		return apiServer.Shutdown()
 	})
 	<-apiReady
@@ -87,5 +92,6 @@ func hasPlugins(profiles []*csconfig.ProfileCfg) bool {
 			return true
 		}
 	}
+
 	return false
 }

cmd/crowdsec/crowdsec.go

@@ -169,7 +169,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H
 
 	log.Info("Starting processing data")
 
-	if err := acquisition.StartAcquisition(dataSources, inputLineChan, &acquisTomb); err != nil {
+	if err := acquisition.StartAcquisition(context.TODO(), dataSources, inputLineChan, &acquisTomb); err != nil {
 		return fmt.Errorf("starting acquisition error: %w", err)
 	}
 

cmd/crowdsec/serve.go

@@ -52,6 +52,8 @@ func debugHandler(sig os.Signal, cConfig *csconfig.Config) error {
 func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 	var tmpFile string
 
+	ctx := context.TODO()
+
 	// re-initialize tombs
 	acquisTomb = tomb.Tomb{}
 	parsersTomb = tomb.Tomb{}
@@ -74,7 +76,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 			cConfig.API.Server.OnlineClient = nil
 		}
 
-		apiServer, err := initAPIServer(cConfig)
+		apiServer, err := initAPIServer(ctx, cConfig)
 		if err != nil {
 			return nil, fmt.Errorf("unable to init api server: %w", err)
 		}
@@ -88,7 +90,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 			return nil, err
 		}
 
-		if err := hub.Load(); err != nil {
+		if err = hub.Load(); err != nil {
 			return nil, err
 		}
 
@@ -374,7 +376,7 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error {
 			cConfig.API.Server.OnlineClient = nil
 		}
 
-		apiServer, err := initAPIServer(cConfig)
+		apiServer, err := initAPIServer(ctx, cConfig)
 		if err != nil {
 			return fmt.Errorf("api server init: %w", err)
 		}
@@ -390,7 +392,7 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error {
 			return err
 		}
 
-		if err := hub.Load(); err != nil {
+		if err = hub.Load(); err != nil {
 			return err
 		}
 

cmd/notification-dummy/main.go

@@ -9,6 +9,7 @@ import (
 	plugin "github.com/hashicorp/go-plugin"
 	"gopkg.in/yaml.v3"
 
+	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
 	"github.com/crowdsecurity/crowdsec/pkg/protobufs"
 )
 
@@ -19,6 +20,7 @@ type PluginConfig struct {
 }
 
 type DummyPlugin struct {
+	protobufs.UnimplementedNotifierServer
 	PluginConfigByName map[string]PluginConfig
 }
 
@@ -84,7 +86,7 @@ func main() {
 	plugin.Serve(&plugin.ServeConfig{
 		HandshakeConfig: handshake,
 		Plugins: map[string]plugin.Plugin{
-			"dummy": &protobufs.NotifierPlugin{
+			"dummy": &csplugin.NotifierPlugin{
 				Impl: sp,
 			},
 		},

cmd/notification-email/main.go

@@ -12,6 +12,7 @@ import (
 	mail "github.com/xhit/go-simple-mail/v2"
 	"gopkg.in/yaml.v3"
 
+	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
 	"github.com/crowdsecurity/crowdsec/pkg/protobufs"
 )
 
@@ -55,6 +56,7 @@ type PluginConfig struct {
 }
 
 type EmailPlugin struct {
+	protobufs.UnimplementedNotifierServer
 	ConfigByName map[string]PluginConfig
 }
 
@@ -170,7 +172,7 @@ func main() {
 	plugin.Serve(&plugin.ServeConfig{
 		HandshakeConfig: handshake,
 		Plugins: map[string]plugin.Plugin{
-			"email": &protobufs.NotifierPlugin{
+			"email": &csplugin.NotifierPlugin{
 				Impl: &EmailPlugin{ConfigByName: make(map[string]PluginConfig)},
 			},
 		},

cmd/notification-file/main.go

@@ -15,6 +15,7 @@ import (
 	plugin "github.com/hashicorp/go-plugin"
 	"gopkg.in/yaml.v3"
 
+	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
 	"github.com/crowdsecurity/crowdsec/pkg/protobufs"
 )
 
@@ -52,6 +53,7 @@ type LogRotate struct {
 }
 
 type FilePlugin struct {
+	protobufs.UnimplementedNotifierServer
 	PluginConfigByName map[string]PluginConfig
 }
 
@@ -241,7 +243,7 @@ func main() {
 	plugin.Serve(&plugin.ServeConfig{
 		HandshakeConfig: handshake,
 		Plugins: map[string]plugin.Plugin{
-			"file": &protobufs.NotifierPlugin{
+			"file": &csplugin.NotifierPlugin{
 				Impl: sp,
 			},
 		},

cmd/notification-http/main.go

@@ -16,6 +16,7 @@ import (
 	plugin "github.com/hashicorp/go-plugin"
 	"gopkg.in/yaml.v3"
 
+	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
 	"github.com/crowdsecurity/crowdsec/pkg/protobufs"
 )
 
@@ -34,6 +35,7 @@ type PluginConfig struct {
 }
 
 type HTTPPlugin struct {
+	protobufs.UnimplementedNotifierServer
 	PluginConfigByName map[string]PluginConfig
 }
 
@@ -190,7 +192,7 @@ func main() {
 	plugin.Serve(&plugin.ServeConfig{
 		HandshakeConfig: handshake,
 		Plugins: map[string]plugin.Plugin{
-			"http": &protobufs.NotifierPlugin{
+			"http": &csplugin.NotifierPlugin{
 				Impl: sp,
 			},
 		},