Merge branch 'master' into feat/cscli-network-flag

crowdsecurity · Dec 18, 2024 · beee1a2 · beee1a2
2 parents df6d51c + b1e2b95
commit beee1a2
Show file tree

Hide file tree

Showing 486 changed files with 16,486 additions and 8,206 deletions.
diff --git a/.github/codecov-ignore-generated.sh b/.github/codecov-ignore-generated.sh
diff --git a/.github/codecov.yml b/.github/codecov.yml
@@ -1,5 +1,8 @@
 # we measure coverage but don't enforce it
 # https://docs.codecov.com/docs/codecov-yaml
+codecov:
+  require_ci_to_pass: false
+
 coverage:
   status:
     patch:
@@ -10,7 +13,7 @@ coverage:
         target: 0%
 
 # if a directory is ignored, there is no way to un-ignore files like pkg/models/helpers.go
-# so we make a full list, manually updated - but it could be generated right before running codecov
+# so we make a full list
 ignore:
   - "./pkg/modelscapi/success_response.go"
   - "./pkg/modelscapi/get_decisions_stream_response_deleted.go"
@@ -41,17 +44,26 @@ ignore:
   - "./pkg/modelscapi/enroll_request.go"
   - "./pkg/modelscapi/register_request.go"
   - "./pkg/modelscapi/add_signals_request_item_source.go"
+  - "./pkg/models/success_response.go"
+  - "./pkg/models/hub_items.go"
   - "./pkg/models/alert.go"
   - "./pkg/models/metrics_bouncer_info.go"
   - "./pkg/models/add_signals_request_item.go"
+  - "./pkg/models/metrics_meta.go"
+  - "./pkg/models/metrics_detail_item.go"
   - "./pkg/models/add_signals_request_item_decisions_item.go"
+  - "./pkg/models/hub_item.go"
   - "./pkg/models/get_alerts_response.go"
+  - "./pkg/models/metrics_labels.go"
   - "./pkg/models/watcher_auth_request.go"
   - "./pkg/models/add_alerts_request.go"
   - "./pkg/models/event.go"
   - "./pkg/models/decisions_delete_request_item.go"
   - "./pkg/models/meta.go"
+  - "./pkg/models/detailed_metrics.go"
   - "./pkg/models/delete_alerts_response.go"
+  - "./pkg/models/remediation_components_metrics.go"
+  - "./pkg/models/console_options.go"
   - "./pkg/models/topx_response.go"
   - "./pkg/models/add_signals_request.go"
   - "./pkg/models/delete_decision_response.go"
@@ -60,24 +72,34 @@ ignore:
   - "./pkg/models/source.go"
   - "./pkg/models/decisions_stream_response.go"
   - "./pkg/models/error_response.go"
+  - "./pkg/models/all_metrics.go"
+  - "./pkg/models/o_sversion.go"
   - "./pkg/models/decision.go"
   - "./pkg/models/decisions_delete_request.go"
   - "./pkg/models/flush_decision_response.go"
   - "./pkg/models/watcher_auth_response.go"
+  - "./pkg/models/lapi_metrics.go"
   - "./pkg/models/watcher_registration_request.go"
   - "./pkg/models/metrics_agent_info.go"
+  - "./pkg/models/log_processors_metrics.go"
   - "./pkg/models/add_signals_request_item_source.go"
+  - "./pkg/models/base_metrics.go"
   - "./pkg/models/add_alerts_response.go"
   - "./pkg/models/metrics.go"
   - "./pkg/protobufs/notifier.pb.go"
+  - "./pkg/protobufs/notifier_grpc.pb.go"
+  - "./pkg/database/ent/metric_update.go"
   - "./pkg/database/ent/machine_delete.go"
   - "./pkg/database/ent/decision_query.go"
   - "./pkg/database/ent/meta_query.go"
+  - "./pkg/database/ent/metric/where.go"
+  - "./pkg/database/ent/metric/metric.go"
   - "./pkg/database/ent/machine_create.go"
   - "./pkg/database/ent/alert.go"
   - "./pkg/database/ent/event_update.go"
   - "./pkg/database/ent/alert_create.go"
   - "./pkg/database/ent/alert_query.go"
+  - "./pkg/database/ent/metric_delete.go"
   - "./pkg/database/ent/lock_create.go"
   - "./pkg/database/ent/bouncer_update.go"
   - "./pkg/database/ent/meta_update.go"
@@ -92,6 +114,7 @@ ignore:
   - "./pkg/database/ent/migrate/migrate.go"
   - "./pkg/database/ent/migrate/schema.go"
   - "./pkg/database/ent/configitem.go"
+  - "./pkg/database/ent/metric_query.go"
   - "./pkg/database/ent/event.go"
   - "./pkg/database/ent/event_query.go"
   - "./pkg/database/ent/lock_update.go"
@@ -111,6 +134,7 @@ ignore:
   - "./pkg/database/ent/bouncer/bouncer.go"
   - "./pkg/database/ent/bouncer/where.go"
   - "./pkg/database/ent/hook/hook.go"
+  - "./pkg/database/ent/metric.go"
   - "./pkg/database/ent/configitem_create.go"
   - "./pkg/database/ent/configitem_delete.go"
   - "./pkg/database/ent/tx.go"
@@ -120,6 +144,7 @@ ignore:
   - "./pkg/database/ent/machine/where.go"
   - "./pkg/database/ent/machine/machine.go"
   - "./pkg/database/ent/event_create.go"
+  - "./pkg/database/ent/metric_create.go"
   - "./pkg/database/ent/decision/where.go"
   - "./pkg/database/ent/decision/decision.go"
   - "./pkg/database/ent/enttest/enttest.go"

diff --git a/.github/generate-codecov-yml.sh b/.github/generate-codecov-yml.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Run this from the repository root:
+#
+# .github/generate-codecov-yml.sh >> .github/codecov.yml
+
+cat <<EOT
+# we measure coverage but don't enforce it
+# https://docs.codecov.com/docs/codecov-yaml
+codecov:
+  require_ci_to_pass: false
+
+coverage:
+  status:
+    patch:
+      default:
+        target: 0%
+    project:
+      default:
+        target: 0%
+
+# if a directory is ignored, there is no way to un-ignore files like pkg/models/helpers.go
+# so we make a full list
+ignore:
+EOT
+
+find . -name "*.go" | while read -r file; do
+    if head -n 1 "$file" | grep -q "Code generated by"; then
+        echo "  - \"$file\""
+    fi
+done
diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml
@@ -33,7 +33,7 @@ jobs:
     - name: "Set up Go"
       uses: actions/setup-go@v5
       with:
-        go-version: "1.22.4"
+        go-version: "1.23"
 
     - name: "Install bats dependencies"
       env:

diff --git a/.github/workflows/bats-mysql.yml b/.github/workflows/bats-mysql.yml
@@ -36,7 +36,7 @@ jobs:
     - name: "Set up Go"
       uses: actions/setup-go@v5
       with:
-        go-version: "1.22.4"
+        go-version: "1.23"
 
     - name: "Install bats dependencies"
       env:

diff --git a/.github/workflows/bats-postgres.yml b/.github/workflows/bats-postgres.yml
@@ -45,7 +45,7 @@ jobs:
     - name: "Set up Go"
       uses: actions/setup-go@v5
       with:
-        go-version: "1.22.4"
+        go-version: "1.23"
 
     - name: "Install bats dependencies"
       env:

diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml
@@ -31,7 +31,7 @@ jobs:
     - name: "Set up Go"
       uses: actions/setup-go@v5
       with:
-        go-version: "1.22.4"
+        go-version: "1.23"
 
     - name: "Install bats dependencies"
       env:
@@ -43,6 +43,10 @@ jobs:
       run: |
         make clean bats-build bats-fixture BUILD_STATIC=1
 
+    - name: Generate codecov configuration
+      run: |
+          .github/generate-codecov-yml.sh >> .github/codecov.yml
+
     - name: "Run tests"
       run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
 

diff --git a/.github/workflows/ci-windows-build-msi.yml b/.github/workflows/ci-windows-build-msi.yml
@@ -35,7 +35,7 @@ jobs:
     - name: "Set up Go"
       uses: actions/setup-go@v5
       with:
-        go-version: "1.22.4"
+        go-version: "1.23"
 
     - name: Build
       run: make windows_installer BUILD_RE2_WASM=1

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -52,7 +52,7 @@ jobs:
     - name: "Set up Go"
       uses: actions/setup-go@v5
       with:
-        go-version: "1.22.4"
+        go-version: "1.23"
         cache-dependency-path: "**/go.sum"
 
     # Initializes the CodeQL tools for scanning.

diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml
@@ -35,10 +35,10 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:
-          config: .github/buildkit.toml
+          buildkitd-config: .github/buildkit.toml
 
       - name: "Build image"
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile${{ matrix.flavor == 'debian' && '.debian' || '' }}
@@ -53,23 +53,12 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.x"
-
-      - name: "Install pipenv"
-        run: |
-          cd docker/test
-          python -m pip install --upgrade pipenv wheel
-
-      - name: "Cache virtualenvs"
-        id: cache-pipenv
-        uses: actions/cache@v4
-        with:
-          path: ~/.local/share/virtualenvs
-          key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
+          cache: 'pipenv'
 
       - name: "Install dependencies"
-        if: steps.cache-pipenv.outputs.cache-hit != 'true'
         run: |
           cd docker/test
+          python -m pip install --upgrade pipenv wheel
           pipenv install --deploy
 
       - name: "Create Docker network"

diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml
@@ -34,16 +34,20 @@ jobs:
     - name: "Set up Go"
       uses: actions/setup-go@v5
       with:
-        go-version: "1.22.4"
+        go-version: "1.23"
 
     - name: Build
       run: |
         make build BUILD_RE2_WASM=1
 
+    - name: Generate codecov configuration
+      run: |
+          .github/generate-codecov-yml.sh >> .github/codecov.yml
+
     - name: Run tests
       run: |
         go install github.com/kyoh86/[email protected]
-        go test -coverprofile coverage.out -covermode=atomic ./... > out.txt
+        go test -tags expr_debug -coverprofile coverage.out -covermode=atomic ./... > out.txt
         if(!$?) { cat out.txt | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter; Exit 1 }
         cat out.txt | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
@@ -55,12 +59,8 @@ jobs:
         token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v4
+      uses: golangci/golangci-lint-action@v6
       with:
-        version: v1.59
+        version: v1.62
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false
-        # the cache is already managed above, enabling it here
-        # gives errors when extracting
-        skip-pkg-cache: true
-        skip-build-cache: true
diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml
@@ -126,13 +126,40 @@ jobs:
     - name: "Set up Go"
       uses: actions/setup-go@v5
       with:
-        go-version: "1.22.4"
+        go-version: "1.23"
+
+    - name: Run "go generate" and check for changes
+      run: |
+        set -e
+        # ensure the version of 'protoc' matches the one that generated the files
+        PROTOBUF_VERSION="21.12"
+        # don't pollute the repo
+        pushd $HOME
+        curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
+        unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d $HOME/.protoc
+        popd
+        export PATH="$HOME/.protoc/bin:$PATH"
+        go install google.golang.org/protobuf/cmd/[email protected]
+        go install google.golang.org/grpc/cmd/[email protected]
+        go generate ./...
+        protoc --version
+        if [[ $(git status --porcelain) ]]; then
+          echo "Error: Uncommitted changes found after running 'make generate'. Please commit all generated code."
+          git diff
+          exit 1
+        else
+          echo "No changes detected after running 'make generate'."
+        fi
 
     - name: Create localstack streams
       run: |
           aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
           aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
 
+    - name: Generate codecov configuration
+      run: |
+          .github/generate-codecov-yml.sh >> .github/codecov.yml
+
     - name: Build and run tests, static
       run: |
         sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
@@ -142,6 +169,11 @@ jobs:
         make build BUILD_STATIC=1
         make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
+    # check if some component stubs are missing
+    - name: "Build profile: minimal"
+      run: |
+        make build BUILD_PROFILE=minimal
+
     - name: Run tests again, dynamic
       run: |
         make clean build
@@ -156,12 +188,8 @@ jobs:
         token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v4
+      uses: golangci/golangci-lint-action@v6
       with:
-        version: v1.59
+        version: v1.62
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false
-        # the cache is already managed above, enabling it here
-        # gives errors when extracting
-        skip-pkg-cache: true
-        skip-build-cache: true
diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml
@@ -47,7 +47,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:
-          config: .github/buildkit.toml
+          buildkitd-config: .github/buildkit.toml
 
       - name: Login to DockerHub
         uses: docker/login-action@v3
@@ -93,7 +93,7 @@ jobs:
 
       - name: Build and push image (slim)
         if: ${{ inputs.slim }}
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}
@@ -109,7 +109,7 @@ jobs:
             BUILD_VERSION=${{ inputs.crowdsec_version }}
 
       - name: Build and push image (full)
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}

diff --git a/.github/workflows/publish-tarball-release.yml b/.github/workflows/publish-tarball-release.yml
@@ -25,7 +25,7 @@ jobs:
       - name: "Set up Go"
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22.4"
+          go-version: "1.23"
 
       - name: Build the binaries
         run: |