From 567cfe1ca7a8784b4417d4afba21c43bde1ebe52 Mon Sep 17 00:00:00 2001
From: Andreas Braun <alcaeus@users.noreply.github.com>
Date: Mon, 30 Oct 2023 09:06:50 +0100
Subject: [PATCH] PHPLIB-1187: Run benchmark on Evergreen (#1185)

* Run benchmark on evergreen

* Add evergreen report to benchmark

* Only run multiple revs for fast benchmarks

* Aim to speed up ParallelMultiFile benchmarks

* Skip AMP worker benchmarks in CI
---
 .evergreen/config/functions.yml               |  13 ++
 .evergreen/config/php.ini                     |   1 +
 .evergreen/config/test-tasks.yml              |  12 ++
 .evergreen/config/test-variants.yml           |  15 +++
 benchmark/phpbench.json.dist                  |   3 +-
 benchmark/src/BSON/DocumentBench.php          |   2 +
 benchmark/src/BSON/PackedArrayBench.php       |   2 +
 .../ParallelMultiFileExportBench.php          |  36 +++---
 .../ParallelMultiFileImportBench.php          |  36 ++----
 benchmark/src/DriverBench/SingleDocBench.php  |   3 -
 benchmark/src/Extension/EvergreenReport.php   | 112 ++++++++++++++++++
 benchmark/src/Extension/MongoDBExtension.php  |  10 ++
 12 files changed, 191 insertions(+), 54 deletions(-)
 create mode 100644 benchmark/src/Extension/EvergreenReport.php

diff --git a/.evergreen/config/functions.yml b/.evergreen/config/functions.yml
index 4df8d3edf..9190a1f8f 100644
--- a/.evergreen/config/functions.yml
+++ b/.evergreen/config/functions.yml
@@ -482,3 +482,16 @@ functions:
         binary: bash
         args:
           - .evergreen/compile-extension.sh
+
+  # Run benchmarks. The filter skips the benchAmpWorkers subjects as they fail due to socket exceptions
+  "run benchmark":
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: "src/benchmark"
+        script: |
+          ${PREPARE_SHELL}
+          export PATH="${PHP_PATH}/bin:$PATH"
+          
+          php ../composer.phar install --no-suggest
+          vendor/bin/phpbench run --report=env --report=evergreen --report=aggregate --output html --filter='bench(?!AmpWorkers)'
diff --git a/.evergreen/config/php.ini b/.evergreen/config/php.ini
index 45969d065..0b1af3129 100644
--- a/.evergreen/config/php.ini
+++ b/.evergreen/config/php.ini
@@ -1 +1,2 @@
 extension=mongodb.so
+memory_limit=-1
diff --git a/.evergreen/config/test-tasks.yml b/.evergreen/config/test-tasks.yml
index f71942113..a3d5feff1 100644
--- a/.evergreen/config/test-tasks.yml
+++ b/.evergreen/config/test-tasks.yml
@@ -20,3 +20,15 @@ tasks:
     commands:
       - func: "bootstrap mongohoused"
       - func: "run atlas data lake test"
+
+  - name: "run-benchmark"
+    exec_timeout_secs: 3600
+    commands:
+      - func: "bootstrap mongo-orchestration"
+        vars:
+          TOPOLOGY: "server"
+          MONGODB_VERSION: "v6.0-perf"
+      - func: "run benchmark"
+      - command: perf.send
+        params:
+          file: src/benchmark/.phpbench/results.json
diff --git a/.evergreen/config/test-variants.yml b/.evergreen/config/test-variants.yml
index bc7d8f1e5..34875e83f 100644
--- a/.evergreen/config/test-variants.yml
+++ b/.evergreen/config/test-variants.yml
@@ -114,3 +114,18 @@ buildvariants:
     tasks:
       - "test_atlas_task_group"
       - ".csfle"
+
+  # Run benchmarks
+  - name: benchmark-rhel90
+    tags: ["benchmark", "rhel", "x64"]
+    display_name: "Benchmark: RHEL 9.0, MongoDB 6.0"
+    run_on: rhel90-dbx-perf-large
+    expansions:
+      FETCH_BUILD_VARIANT: "build-rhel90"
+      FETCH_BUILD_TASK: "build-php-8.2"
+      PHP_VERSION: "8.2"
+    depends_on:
+      - variant: "build-rhel90"
+        name: "build-php-8.2"
+    tasks:
+      - "run-benchmark"
diff --git a/benchmark/phpbench.json.dist b/benchmark/phpbench.json.dist
index f3acbd04c..8d00a7db5 100644
--- a/benchmark/phpbench.json.dist
+++ b/benchmark/phpbench.json.dist
@@ -6,6 +6,5 @@
     "runner.file_pattern": "*Bench.php",
     "runner.path": "src",
     "runner.php_config": { "memory_limit": "1G" },
-    "runner.iterations": 3,
-    "runner.revs": 10
+    "runner.iterations": 3
 }
diff --git a/benchmark/src/BSON/DocumentBench.php b/benchmark/src/BSON/DocumentBench.php
index 143d9449f..1991cd614 100644
--- a/benchmark/src/BSON/DocumentBench.php
+++ b/benchmark/src/BSON/DocumentBench.php
@@ -5,6 +5,7 @@
 use MongoDB\Benchmark\Fixtures\Data;
 use MongoDB\BSON\Document;
 use PhpBench\Attributes\BeforeMethods;
+use PhpBench\Attributes\Revs;
 use PhpBench\Attributes\Warmup;
 use stdClass;
 
@@ -12,6 +13,7 @@
 use function iterator_to_array;
 
 #[BeforeMethods('prepareData')]
+#[Revs(10)]
 #[Warmup(1)]
 final class DocumentBench
 {
diff --git a/benchmark/src/BSON/PackedArrayBench.php b/benchmark/src/BSON/PackedArrayBench.php
index 37898fa32..5f8bd6284 100644
--- a/benchmark/src/BSON/PackedArrayBench.php
+++ b/benchmark/src/BSON/PackedArrayBench.php
@@ -5,12 +5,14 @@
 use MongoDB\Benchmark\Fixtures\Data;
 use MongoDB\BSON\PackedArray;
 use PhpBench\Attributes\BeforeMethods;
+use PhpBench\Attributes\Revs;
 use PhpBench\Attributes\Warmup;
 
 use function array_values;
 use function iterator_to_array;
 
 #[BeforeMethods('prepareData')]
+#[Revs(10)]
 #[Warmup(1)]
 final class PackedArrayBench
 {
diff --git a/benchmark/src/DriverBench/ParallelMultiFileExportBench.php b/benchmark/src/DriverBench/ParallelMultiFileExportBench.php
index e6aaabba4..fb02c2535 100644
--- a/benchmark/src/DriverBench/ParallelMultiFileExportBench.php
+++ b/benchmark/src/DriverBench/ParallelMultiFileExportBench.php
@@ -15,7 +15,6 @@
 use PhpBench\Attributes\BeforeClassMethods;
 use PhpBench\Attributes\Iterations;
 use PhpBench\Attributes\ParamProviders;
-use PhpBench\Attributes\Revs;
 use RuntimeException;
 
 use function array_chunk;
@@ -44,7 +43,6 @@
 #[AfterClassMethods('afterClass')]
 #[AfterMethods('afterIteration')]
 #[Iterations(1)]
-#[Revs(1)]
 final class ParallelMultiFileExportBench
 {
     public static function beforeClass(): void
@@ -74,15 +72,15 @@ public function afterIteration(): void
      * Using a single thread to export multiple files.
      * By executing a single Find command for multiple files, we can reduce the number of roundtrips to the server.
      *
-     * @param array{chunk:int} $params
+     * @param array{chunkSize:int} $params
      */
     #[ParamProviders(['provideChunkParams'])]
     public function benchSequential(array $params): void
     {
-        foreach (array_chunk(self::getFileNames(), $params['chunk']) as $i => $files) {
+        foreach (array_chunk(self::getFileNames(), $params['chunkSize']) as $i => $files) {
             self::exportFile($files, [], [
-                'limit' => 5_000 * $params['chunk'],
-                'skip' => 5_000 * $params['chunk'] * $i,
+                'limit' => 5_000 * $params['chunkSize'],
+                'skip' => 5_000 * $params['chunkSize'] * $i,
             ]);
         }
     }
@@ -103,12 +101,12 @@ public function benchFork(array $params): void
         Utils::reset();
 
         // Create a child process for each chunk of files
-        foreach (array_chunk(self::getFileNames(), $params['chunk']) as $i => $files) {
+        foreach (array_chunk(self::getFileNames(), $params['chunkSize']) as $i => $files) {
             $pid = pcntl_fork();
             if ($pid === 0) {
                 self::exportFile($files, [], [
-                    'limit' => 5_000 * $params['chunk'],
-                    'skip' => 5_000 * $params['chunk'] * $i,
+                    'limit' => 5_000 * $params['chunkSize'],
+                    'skip' => 5_000 * $params['chunkSize'] * $i,
                 ]);
 
                 // Exit the child process
@@ -133,21 +131,21 @@ public function benchFork(array $params): void
     /**
      * Using amphp/parallel with worker pool
      *
-     * @param array{chunk:int} $params
+     * @param array{chunkSize:int} $params
      */
     #[ParamProviders(['provideChunkParams'])]
     public function benchAmpWorkers(array $params): void
     {
-        $workerPool = new ContextWorkerPool(ceil(100 / $params['chunk']), new ContextWorkerFactory());
+        $workerPool = new ContextWorkerPool(ceil(100 / $params['chunkSize']), new ContextWorkerFactory());
 
         $futures = [];
-        foreach (array_chunk(self::getFileNames(), $params['chunk']) as $i => $files) {
+        foreach (array_chunk(self::getFileNames(), $params['chunkSize']) as $i => $files) {
             $futures[] = $workerPool->submit(
                 new ExportFileTask(
                     files: $files,
                     options: [
-                        'limit' => 5_000 * $params['chunk'],
-                        'skip' => 5_000 * $params['chunk'] * $i,
+                        'limit' => 5_000 * $params['chunkSize'],
+                        'skip' => 5_000 * $params['chunkSize'] * $i,
                     ],
                 ),
             )->getFuture();
@@ -160,13 +158,9 @@ public function benchAmpWorkers(array $params): void
 
     public static function provideChunkParams(): Generator
     {
-        yield 'by 1' => ['chunk' => 1];
-        yield 'by 2' => ['chunk' => 2];
-        yield 'by 4' => ['chunk' => 4];
-        yield 'by 8' => ['chunk' => 8];
-        yield 'by 13' => ['chunk' => 13];
-        yield 'by 20' => ['chunk' => 20];
-        yield 'by 100' => ['chunk' => 100];
+        yield '100 chunks' => ['chunkSize' => 1];
+        yield '25 chunks' => ['chunkSize' => 4];
+        yield '10 chunks' => ['chunkSize' => 10];
     }
 
     /**
diff --git a/benchmark/src/DriverBench/ParallelMultiFileImportBench.php b/benchmark/src/DriverBench/ParallelMultiFileImportBench.php
index cba5fcbd5..2e2159f8b 100644
--- a/benchmark/src/DriverBench/ParallelMultiFileImportBench.php
+++ b/benchmark/src/DriverBench/ParallelMultiFileImportBench.php
@@ -16,7 +16,6 @@
 use PhpBench\Attributes\BeforeMethods;
 use PhpBench\Attributes\Iterations;
 use PhpBench\Attributes\ParamProviders;
-use PhpBench\Attributes\Revs;
 use RuntimeException;
 
 use function array_chunk;
@@ -47,7 +46,6 @@
 #[AfterClassMethods('afterClass')]
 #[BeforeMethods('beforeIteration')]
 #[Iterations(1)]
-#[Revs(1)]
 final class ParallelMultiFileImportBench
 {
     public static function beforeClass(): void
@@ -73,20 +71,6 @@ public function beforeIteration(): void
         $database->createCollection(Utils::getCollectionName());
     }
 
-    /**
-     * Using Driver's BulkWrite in a single thread.
-     * The number of files to import in each iteration is controlled by the "chunk" parameter.
-     *
-     * @param array{chunk:int} $params
-     */
-    #[ParamProviders(['provideChunkParams'])]
-    public function benchBulkWrite(array $params): void
-    {
-        foreach (array_chunk(self::getFileNames(), $params['chunk']) as $files) {
-            self::importFile($files);
-        }
-    }
-
     /**
      * Using library's Collection::insertMany in a single thread
      */
@@ -116,7 +100,7 @@ public function benchInsertMany(): void
      * Using multiple forked threads. The number of threads is controlled by the "chunk" parameter,
      * which is the number of files to import in each thread.
      *
-     * @param array{chunk:int} $params
+     * @param array{chunkSize:int} $params
      */
     #[ParamProviders(['provideChunkParams'])]
     public function benchFork(array $params): void
@@ -128,7 +112,7 @@ public function benchFork(array $params): void
         // of a new libmongoc client.
         Utils::reset();
 
-        foreach (array_chunk(self::getFileNames(), $params['chunk']) as $files) {
+        foreach (array_chunk(self::getFileNames(), $params['chunkSize']) as $files) {
             $pid = pcntl_fork();
             if ($pid === 0) {
                 self::importFile($files);
@@ -155,16 +139,16 @@ public function benchFork(array $params): void
     /**
      * Using amphp/parallel with worker pool
      *
-     * @param array{processes:int} $params
+     * @param array{chunkSize:int} $params
      */
     #[ParamProviders(['provideChunkParams'])]
     public function benchAmpWorkers(array $params): void
     {
-        $workerPool = new ContextWorkerPool(ceil(100 / $params['chunk']), new ContextWorkerFactory());
+        $workerPool = new ContextWorkerPool(ceil(100 / $params['chunkSize']), new ContextWorkerFactory());
 
         $futures = array_map(
             fn ($files) => $workerPool->submit(new ImportFileTask($files))->getFuture(),
-            array_chunk(self::getFileNames(), $params['chunk']),
+            array_chunk(self::getFileNames(), $params['chunkSize']),
         );
 
         foreach (Future::iterate($futures) as $future) {
@@ -176,13 +160,9 @@ public function benchAmpWorkers(array $params): void
 
     public function provideChunkParams(): Generator
     {
-        yield 'by 1' => ['chunk' => 1];
-        yield 'by 2' => ['chunk' => 2];
-        yield 'by 4' => ['chunk' => 4];
-        yield 'by 8' => ['chunk' => 8];
-        yield 'by 13' => ['chunk' => 13];
-        yield 'by 20' => ['chunk' => 20];
-        yield 'by 100' => ['chunk' => 100];
+        yield '100 chunks' => ['chunkSize' => 1];
+        yield '25 chunks' => ['chunkSize' => 4];
+        yield '10 chunks' => ['chunkSize' => 10];
     }
 
     /**
diff --git a/benchmark/src/DriverBench/SingleDocBench.php b/benchmark/src/DriverBench/SingleDocBench.php
index 3e95019ce..ee49eb002 100644
--- a/benchmark/src/DriverBench/SingleDocBench.php
+++ b/benchmark/src/DriverBench/SingleDocBench.php
@@ -9,7 +9,6 @@
 use MongoDB\Driver\Command;
 use PhpBench\Attributes\BeforeMethods;
 use PhpBench\Attributes\ParamProviders;
-use PhpBench\Attributes\Revs;
 
 use function array_map;
 use function file_get_contents;
@@ -45,7 +44,6 @@ public function benchRunCommand(): void
      */
     #[BeforeMethods('beforeFindOneById')]
     #[ParamProviders('provideFindOneByIdParams')]
-    #[Revs(1)]
     public function benchFindOneById(array $params): void
     {
         $collection = Utils::getCollection();
@@ -79,7 +77,6 @@ public static function provideFindOneByIdParams(): Generator
      * @param array{document: object|array, repeat: int, options?: array} $params
      */
     #[ParamProviders('provideInsertOneParams')]
-    #[Revs(1)]
     public function benchInsertOne(array $params): void
     {
         $collection = Utils::getCollection();
diff --git a/benchmark/src/Extension/EvergreenReport.php b/benchmark/src/Extension/EvergreenReport.php
new file mode 100644
index 000000000..40ea06113
--- /dev/null
+++ b/benchmark/src/Extension/EvergreenReport.php
@@ -0,0 +1,112 @@
+<?php
+
+namespace MongoDB\Benchmark\Extension;
+
+use PhpBench\Compat\SymfonyOptionsResolverCompat;
+use PhpBench\Model\Suite;
+use PhpBench\Model\SuiteCollection;
+use PhpBench\Path\Path;
+use PhpBench\Registry\Config;
+use PhpBench\Report\GeneratorInterface;
+use PhpBench\Report\Model\Reports;
+use RuntimeException;
+use Symfony\Component\OptionsResolver\OptionsResolver;
+
+use function assert;
+use function date;
+use function dirname;
+use function file_exists;
+use function file_put_contents;
+use function json_encode;
+use function mkdir;
+use function sprintf;
+
+use const DATE_ATOM;
+use const JSON_PRETTY_PRINT;
+
+class EvergreenReport implements GeneratorInterface
+{
+    private const PARAM_PATH = 'path';
+
+    public function __construct(private string $cwd)
+    {
+    }
+
+    public function configure(OptionsResolver $options): void
+    {
+        $options->setDefaults([self::PARAM_PATH => '.phpbench/results.json']);
+        $options->setAllowedTypes(self::PARAM_PATH, ['string']);
+
+        SymfonyOptionsResolverCompat::setInfos($options, [self::PARAM_PATH => 'Path to output file']);
+    }
+
+    public function generate(SuiteCollection $collection, Config $config): Reports
+    {
+        $tests = [];
+
+        foreach ($collection as $suite) {
+            assert($suite instanceof Suite);
+            foreach ($suite as $benchmark) {
+                foreach ($benchmark as $subject) {
+                    foreach ($subject->getVariants() as $variant) {
+                        $stats = $variant->getStats()->getStats();
+                        $name = sprintf('%s::%s', $benchmark->getName(), $subject->getName());
+                        if ($variant->getParameterSet()->getName()) {
+                            $name .= '#' . $variant->getParameterSet()->getName();
+                        }
+
+                        $tests[] = [
+                            'info' => ['test_name' => $name],
+                            'created_at' => date(DATE_ATOM),
+                            'completed_at' => date(DATE_ATOM),
+                            'metrics' => [
+                                [
+                                    'name' => 'Avg. Time',
+                                    'type' => 'MEAN',
+                                    'value' => $stats['mean'],
+                                ],
+                                [
+                                    'name' => 'Min. Time',
+                                    'type' => 'MIN',
+                                    'value' => $stats['min'],
+                                ],
+                                [
+                                    'name' => 'Max. Time',
+                                    'type' => 'MAX',
+                                    'value' => $stats['max'],
+                                ],
+                                [
+                                    'name' => 'Std. Deviation',
+                                    'type' => 'STANDARD_DEVIATION',
+                                    'value' => $stats['stdev'],
+                                ],
+                            ],
+                        ];
+                    }
+                }
+            }
+        }
+
+        $outputPath = Path::makeAbsolute($config[self::PARAM_PATH], $this->cwd);
+        $outputDir = dirname($outputPath);
+
+        if (! file_exists($outputDir)) {
+            if (! @mkdir($outputDir, 0777, true)) {
+                throw new RuntimeException(sprintf(
+                    'Could not create directory "%s"',
+                    $outputDir,
+                ));
+            }
+        }
+
+        if (false === file_put_contents($outputPath, json_encode($tests, JSON_PRETTY_PRINT) . "\n")) {
+            throw new RuntimeException(sprintf(
+                'Could not write report to file "%s"',
+                $outputPath,
+            ));
+        }
+
+        // Return an empty report to not confuse the report renderer
+        return Reports::empty();
+    }
+}
diff --git a/benchmark/src/Extension/MongoDBExtension.php b/benchmark/src/Extension/MongoDBExtension.php
index e57226244..ddddb627c 100644
--- a/benchmark/src/Extension/MongoDBExtension.php
+++ b/benchmark/src/Extension/MongoDBExtension.php
@@ -4,6 +4,8 @@
 
 use PhpBench\DependencyInjection\Container;
 use PhpBench\DependencyInjection\ExtensionInterface;
+use PhpBench\Extension\CoreExtension;
+use PhpBench\Extension\ReportExtension;
 use PhpBench\Extension\RunnerExtension;
 use Symfony\Component\OptionsResolver\OptionsResolver;
 
@@ -16,6 +18,14 @@ public function load(Container $container): void
             fn (Container $container) => new EnvironmentProvider(),
             [RunnerExtension::TAG_ENV_PROVIDER => ['name' => 'mongodb']],
         );
+
+        $container->register(
+            EvergreenReport::class,
+            fn (Container $container) => new EvergreenReport(
+                $container->getParameter(CoreExtension::PARAM_WORKING_DIR),
+            ),
+            [ReportExtension::TAG_REPORT_GENERATOR => ['name' => 'evergreen']],
+        );
     }
 
     public function configure(OptionsResolver $resolver): void