feat(metrics): Implement pushUsageMetrics method

crowdsecurity · Dec 26, 2024 · f0dace5 · f0dace5
1 parent 633701d
commit f0dace5
Show file tree

Hide file tree

Showing 15 changed files with 595 additions and 48 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -22,14 +22,15 @@ As far as possible, we try to adhere to [Symfony guidelines](https://symfony.com
 
 ### Added
 
+- Add `LapiRemediation::pushUsageMetrics` method to push usage metrics to LAPI
 - Add `bouncing_level` configuration to cap maximum remediation level
 - Add `AbstractRemediation::resetRemediationOriginCount` method to reset origin count cache item for a remediation
 
 ### Changed
 
 - For `lists` origin, store also the list name (scenario) in origins count cache item
 - **Breaking change**: Rename `AbstractRemediation::updateRemediationOriginCount` method to 
-  `incrementRemediationOriginCount` and add a `$remediation` parameter.
+  `updateRemediationOriginCount` with new `$remediation` and `$delta` parameters.
 - **Breaking change**: Store `clean` as origin in cache for `bypass` remediation even if 
   original retrieved remediation was not a bypass (unhandled or capped remediation ). If `bypass` is the result of 
   AppSec remediation, we continue to store `clean_appsec` as origin in cache.

diff --git a/composer.json b/composer.json
@@ -47,7 +47,7 @@
         "symfony/cache": "^5.4.11|| ^6.0.11",
         "crowdsec/common": "^2.3.2",
         "crowdsec/capi-client": "^3.2.0",
-        "crowdsec/lapi-client": "^3.3.0",
+        "crowdsec/lapi-client": "^4.0.0",
         "monolog/monolog": "^1.17 || ^2.1",
         "mlocati/ip-lib": "^1.18",
         "geoip2/geoip2": "^2.13.0"

diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md
@@ -67,11 +67,13 @@ This kind of action is called a remediation and can be:
     - Use the cached decisions for CAPI and for LAPI in stream mode
     - For LAPI in live mode, call LAPI if there is no cached decision
     - Use customizable remediation priorities
-  - Determine AppSec (LAPI) remediation for a given request 
+  - Determine AppSec (LAPI) remediation for a given request
+
+- CrowdSec metrics
+  - Push usage metrics to LAPI
 
 - Overridable cache handler (built-in support for `Redis`, `Memcached` and `PhpFiles` caches)
 
-
 - Large PHP matrix compatibility: from 7.2 to 8.4
 
 
@@ -341,6 +343,16 @@ The `$rawBody` parameter is optional and must be used if the forwarded request c
 
 Please see the [CrowdSec AppSec documentation](https://docs.crowdsec.net/docs/appsec/intro) for more details.
 
+##### Push usage metrics to LAPI
+
+To push usage metrics to LAPI, you can do the following call:
+
+```php
+    $remediationEngine->pushUsageMetrics($bouncerName, $bouncerVersion, $bouncerType);
+```
+
+Metrics are retrieved from the cache and sent to LAPI.
+
 
 #### Example scripts
 
@@ -437,6 +449,20 @@ php tests/scripts/get-remediation-appsec.php <APPSEC_URL> <IP> <URI> <HOST> <VER
 php tests/scripts/get-appsec-remediation http://crowdsec:7422  172.0.0.24 /login example.com POST c580eb*********de541 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0' '{"Content-Type":"application/x-www-form-urlencoded","Accept-Language":"en-US,en;q=0.5"}' 'class.module.classLoader.resources.'
 ```
 
+##### Push usage metrics to LAPI
+
+###### Command usage
+
+```php
+php tests/scripts/push-lapi-usage-metrics.php <BOUNCER_KEY> <LAPI_URL>
+```
+
+###### Example usage
+
+```bash
+  php tests/scripts/push-lapi-usage-metrics.php 68c2b479830c89bfd48926f9d764da39  https://crowdsec:8080
+```
+
 
 ## CAPI remediation engine configurations
 
@@ -744,10 +770,11 @@ the origin. When the retrieved remediation is a `bypass` (i.e. no active decisio
 $originsCount = $remediation->getOriginsCount();
 
 /*$originsCount = [
-    'appsec' => 6,
-    'clean' => 150,
-    'clean_appsec' => 2, 
-    'capi' => 28,
-    'lists' => 16,
+    'appsec' => ['ban' => 10],
+    'clean' => ['bypass' =>150],
+    'clean_appsec' => ['bypass' =>2], 
+    'CAPI' => ['ban' => 28],
+    'cscli' => ['ban' => 5, 'captcha' => 3],
+    'lists:tor' => ['custom' => 16],
 ]*/
 ```
diff --git a/src/AbstractRemediation.php b/src/AbstractRemediation.php
@@ -99,25 +99,27 @@ public function getOriginsCount(): array
      * @throws CacheException
      * @throws InvalidArgumentException
      */
-    public function incrementRemediationOriginCount(string $origin, string $remediation): int
+    public function updateRemediationOriginCount(string $origin, string $remediation, int $delta = 1): int
     {
         $cacheOriginCount = $this->getOriginsCountItem();
         $count = isset($cacheOriginCount[$origin][$remediation]) ?
             (int) $cacheOriginCount[$origin][$remediation] :
             0;
 
+        $count += $delta;
+        $finalCount = max(0, $count);
         $this->cacheStorage->upsertItem(
             AbstractCache::ORIGINS_COUNT,
             [
                 $origin => [
-                    $remediation => ++$count,
+                    $remediation => $finalCount,
                 ],
             ],
             0,
             [AbstractCache::ORIGINS_COUNT]
         );
 
-        return $count;
+        return $finalCount;
     }
 
     /**
@@ -311,7 +313,7 @@ protected function processCachedDecisions(array $cacheDecisions): string
         $remediation = !empty($remediationData[self::INDEX_REM]) ? (string) $remediationData[self::INDEX_REM] :
             Constants::REMEDIATION_BYPASS;
         if ($origin) {
-            $this->incrementRemediationOriginCount($origin, $remediation);
+            $this->updateRemediationOriginCount($origin, $remediation);
         }
 
         return $remediation;

diff --git a/src/CacheStorage/AbstractCache.php b/src/CacheStorage/AbstractCache.php
@@ -42,6 +42,8 @@ abstract class AbstractCache
     public const INDEX_ORIGIN = 3;
     /** @var string The cache key prefix for a IPV4 range bucket */
     public const IPV4_BUCKET_KEY = 'range_bucket_ipv4';
+    /** @var string Internal name for last metrics sent timestamp */
+    public const LAST_METRICS_SENT = 'last_metrics_sent';
     /** @var string Internal name for last pull */
     public const LAST_PULL = 'last_pull';
     /** @var string Internal name for list */

diff --git a/src/CapiRemediation.php b/src/CapiRemediation.php
@@ -56,7 +56,7 @@ public function getIpRemediation(string $ip): string
                 'ip' => $ip,
             ]);
             $remediation = Constants::REMEDIATION_BYPASS;
-            $this->incrementRemediationOriginCount(AbstractCache::CLEAN, $remediation);
+            $this->updateRemediationOriginCount(AbstractCache::CLEAN, $remediation);
 
             // As CAPI is always in stream_mode, we do not store this bypass
             return $remediation;

diff --git a/src/LapiRemediation.php b/src/LapiRemediation.php
@@ -6,6 +6,7 @@
 
 use CrowdSec\LapiClient\Bouncer;
 use CrowdSec\LapiClient\ClientException;
+use CrowdSec\LapiClient\Constants as LapiConstants;
 use CrowdSec\LapiClient\TimeoutException;
 use CrowdSec\RemediationEngine\CacheStorage\AbstractCache;
 use CrowdSec\RemediationEngine\CacheStorage\CacheStorageException;
@@ -86,7 +87,7 @@ public function getAppSecRemediation(array $headers, string $rawBody = ''): stri
         }
         $rawRemediation = $this->parseAppSecDecision($rawAppSecDecision);
         if (Constants::REMEDIATION_BYPASS === $rawRemediation) {
-            $this->incrementRemediationOriginCount(AbstractCache::CLEAN_APPSEC, $rawRemediation);
+            $this->updateRemediationOriginCount(AbstractCache::CLEAN_APPSEC, $rawRemediation);
 
             return $rawRemediation;
         }
@@ -105,6 +106,13 @@ public function getClient(): Bouncer
     }
 
     /**
+     * Retrieve the remediation for a given IP.
+     *
+     * It will first check the cache for the IP decisions.
+     * If no decisions are found, it will call LAPI to get the decisions.
+     * The decisions are then stored in the cache.
+     * The remediation is then processed and returned.
+     *
      * @throws CacheStorageException
      * @throws InvalidArgumentException
      * @throws RemediationException
@@ -123,12 +131,12 @@ public function getIpRemediation(string $ip): string
             // In stream_mode, we do not store this bypass, and we do not call LAPI directly
             if ($this->getConfig('stream_mode')) {
                 $remediation = Constants::REMEDIATION_BYPASS;
-                $this->incrementRemediationOriginCount(AbstractCache::CLEAN, $remediation);
+                $this->updateRemediationOriginCount(AbstractCache::CLEAN, $remediation);
 
                 return $remediation;
             }
             // In live mode, ask LAPI (Retrieve Ip AND Range scoped decisions)
-            $this->storeFirstCall();
+            $this->storeFirstCall(time());
             $rawIpDecisions = $this->client->getFilteredDecisions(['ip' => $ip]);
             $ipDecisions = $this->convertRawDecisionsToDecisions($rawIpDecisions);
             // IPV6 range scoped decisions are not yet stored in cache, so we store it as IP scoped decisions
@@ -160,6 +168,74 @@ public function getIpRemediation(string $ip): string
     }
 
     /**
+     * Push usage metrics to LAPI.
+     *
+     * The metrics are built from the cache and then sent to LAPI.
+     * The cache is then updated to reflect the metrics sent.
+     * Returns the metrics items sent to LAPI.
+     *
+     * @throws CacheException
+     * @throws InvalidArgumentException
+     */
+    public function pushUsageMetrics(
+        string $bouncerName,
+        string $bouncerVersion,
+        string $bouncerType = LapiConstants::METRICS_TYPE
+    ): array {
+        $cacheConfigItem = $this->cacheStorage->getItem(AbstractCache::CONFIG);
+        $cacheConfig = $cacheConfigItem->isHit() ? $cacheConfigItem->get() : [];
+        $start = $cacheConfig[AbstractCache::FIRST_LAPI_CALL] ?? 0;
+        $now = time();
+        $lastSent = $cacheConfig[AbstractCache::LAST_METRICS_SENT] ?? $start;
+
+        $originsCount = $this->getOriginsCount();
+        $build = $this->buildMetricsItems($originsCount);
+        $metricsItems = $build['items'] ?? [];
+        $originsToUpdate = $build['origins'] ?? [];
+        if (empty($metricsItems)) {
+            $this->logger->info('No metrics to send', [
+                'type' => 'LAPI_REM_NO_METRICS',
+            ]);
+
+            return [];
+        }
+
+        $properties = [
+            'name' => $bouncerName,
+            'type' => $bouncerType,
+            'version' => $bouncerVersion,
+            'utc_startup_timestamp' => $start,
+        ];
+        $meta = [
+            'window_size_seconds' => max(0, $now - $lastSent),
+            'utc_now_timestamp' => $now,
+        ];
+
+        $metrics = $this->client->buildUsageMetrics($properties, $meta, $metricsItems);
+
+        $this->client->pushUsageMetrics($metrics);
+
+        // Decrement the count of each origin/remediation
+        foreach ($originsToUpdate as $origin => $remediationCount) {
+            foreach ($remediationCount as $remediation => $delta) {
+                // We update the count of each origin/remediation, one by one
+                // because we want to handle the case where an origin/remediation/count has been updated
+                // between the time we get the count and the time we update it
+                $this->updateRemediationOriginCount($origin, $remediation, $delta);
+            }
+        }
+
+        $this->storeMetricsLastSent($now);
+
+        return $metrics;
+    }
+
+    /**
+     * Refresh the decisions from LAPI.
+     *
+     * This method is only available in stream mode.
+     * Depending on the warmup status, it will either process a startup or a regular refresh.
+     *
      * @SuppressWarnings(PHPMD.BooleanArgumentFlag)
      *
      * @throws CacheException
@@ -189,6 +265,46 @@ public function refreshDecisions(): array
         return $this->getStreamDecisions(false, $filter);
     }
 
+    private function buildMetricsItems(array $originsCount): array
+    {
+        $metricsItems = [];
+        $processed = 0;
+        $originsToUpdate = [];
+        foreach ($originsCount as $origin => $remediationCount) {
+            foreach ($remediationCount as $remediation => $count) {
+                if ($count <= 0) {
+                    continue;
+                }
+                // Count all processed metrics, even clean ones
+                $processed += $count;
+                // Prepare data to update origins count item after processing
+                $originsToUpdate[$origin][$remediation] = -$count;
+                if (Constants::REMEDIATION_BYPASS === $remediation) {
+                    continue;
+                }
+                // Create "dropped" metrics
+                $metricsItems[] = [
+                    'name' => 'dropped',
+                    'value' => $count,
+                    'unit' => 'request',
+                    'labels' => [
+                        'origin' => $origin,
+                        'remediation' => $remediation,
+                    ],
+                ];
+            }
+        }
+        if ($processed > 0) {
+            $metricsItems[] = [
+                'name' => 'processed',
+                'value' => $processed,
+                'unit' => 'request',
+            ];
+        }
+
+        return ['items' => $metricsItems, 'origins' => $originsToUpdate];
+    }
+
     /**
      * Process and validate input configurations.
      */
@@ -235,6 +351,7 @@ private function getScopes(): array
      */
     private function getStreamDecisions(bool $startup = false, array $filter = []): array
     {
+        $this->storeFirstCall(time());
         $rawDecisions = $this->client->getStreamDecisions($startup, $filter);
         $newDecisions = $this->convertRawDecisionsToDecisions($rawDecisions[self::CS_NEW] ?? []);
         $deletedDecisions = $this->convertRawDecisionsToDecisions($rawDecisions[self::CS_DEL] ?? []);
@@ -299,19 +416,41 @@ private function parseAppSecDecision(array $rawAppSecDecision): string
      * @throws CacheException
      * @throws InvalidArgumentException
      */
-    private function storeFirstCall(): void
+    private function storeFirstCall(int $timestamp): void
     {
         $firstCall = $this->getFirstCall();
         if (0 !== $firstCall) {
             return;
         }
-        $time = time();
-        $content = [AbstractCache::FIRST_LAPI_CALL => $time];
+        $content = [AbstractCache::FIRST_LAPI_CALL => $timestamp];
         $this->logger->info(
             'Flag LAPI first call',
             [
                 'type' => 'LAPI_REM_CACHE_FIRST_CALL',
-                'time' => $time,
+                'time' => $timestamp,
+            ]
+        );
+
+        $this->cacheStorage->upsertItem(
+            AbstractCache::CONFIG,
+            $content,
+            0,
+            [AbstractCache::CONFIG]
+        );
+    }
+
+    /**
+     * @throws CacheException
+     * @throws InvalidArgumentException
+     */
+    private function storeMetricsLastSent(int $timestamp): void
+    {
+        $content = [AbstractCache::LAST_METRICS_SENT => $timestamp];
+        $this->logger->debug(
+            'Flag metrics last sent',
+            [
+                'type' => 'LAPI_REM_CACHE_METRICS_LAST_SENT',
+                'time' => $timestamp,
             ]
         );
 
@@ -372,21 +511,9 @@ private function warmUp(array $filter): array
         $result = $this->getStreamDecisions(true, $filter);
         // Store the fact that the cache has been warmed up.
         $this->logger->info('Flag cache warmup', ['type' => 'LAPI_REM_CACHE_WARMUP']);
-        $content = [AbstractCache::WARMUP => true];
-        if (0 === $this->getFirstCall()) {
-            $time = time();
-            $content[AbstractCache::FIRST_LAPI_CALL] = $time;
-            $this->logger->info(
-                'Flag LAPI first call',
-                [
-                    'type' => 'LAPI_REM_CACHE_FIRST_CALL',
-                    'time' => $time,
-                ]
-            );
-        }
         $this->cacheStorage->upsertItem(
             AbstractCache::CONFIG,
-            $content,
+            [AbstractCache::WARMUP => true],
             0,
             [AbstractCache::CONFIG]
         );