From 31a4539712cd911d2173848d112a9444e9e8fd5c Mon Sep 17 00:00:00 2001 From: "panxuchen.pxc" Date: Wed, 10 Jul 2024 20:40:24 +0800 Subject: [PATCH 1/8] add benchmark --- .github/workflows/benchmark.yml | 42 +++++++++++++++++++++ examples/distributed_simulation/bench.py | 48 ++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 examples/distributed_simulation/bench.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..0b981fa13 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,42 @@ +name: AgentScope Benchmark + +on: [push, pull_request] + +jobs: + bench: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: True + matrix: + os: [ubuntu-latest] + env: + OS: ${{ matrix.os }} + PYTHON: '3.9' + steps: + - uses: actions/checkout@master + - name: Setup Python + uses: actions/setup-python@master + with: + python-version: 3.9 + - name: Install AgentScope + run: | + pip install -q -e .[distribute] + pip install pytest pytest-benchmark + - name: Run bench + run: | + cd examples/distributed_simulation + pytest bench.py --benchmark-json output.json + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Python Benchmark with pytest-benchmark + tool: 'pytest' + output-file-path: examples/distributed_simulation/output.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '150%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@pan-x-c' \ No newline at end of file diff --git a/examples/distributed_simulation/bench.py b/examples/distributed_simulation/bench.py new file mode 100644 index 000000000..80dae65a7 --- /dev/null +++ b/examples/distributed_simulation/bench.py @@ -0,0 +1,48 @@ +"""Simulation benchmark + +Please install pytest and pytest-benchmark to run this benchmark. +""" + +import pytest # pylint: disable=W0611 +from participant import Moderator, RandomParticipant, LLMParticipant +from main import run_main_process +import agentscope +from agentscope.server import RpcAgentServerLauncher + + +def test_simulation(benchmark): + """A single benchmark for the simulation""" + base_port = 23300 + par_server_num = 4 + mod_server_num = 1 + agentscope.init( + project="simulation", + name="server", + save_code=False, + save_api_invoke=False, + model_configs="configs/model_configs.json", + use_monitor=False, + ) + launchers = [] + for i in range(par_server_num + mod_server_num): + launcher = RpcAgentServerLauncher( + host="localhost", + port=base_port + i, + custom_agents=[Moderator, RandomParticipant, LLMParticipant], + ) + launcher.launch() + launchers.append(launcher) + benchmark( + run_main_process, + hosts=["localhost"], + base_port=23300, + server_per_host=4, + model_per_host=1, + participant_num=400, + moderator_per_host=1, + agent_type="random", + max_value=100, + sleep_time=1.0, + ) + for launcher in launchers: + launcher.shutdown() From acff838df3a78349f629d1221cee31ec9cf4e46a Mon Sep 17 00:00:00 2001 From: "panxuchen.pxc" Date: Wed, 10 Jul 2024 20:57:07 +0800 Subject: [PATCH 2/8] fix pre-commit --- .github/workflows/benchmark.yml | 1 + examples/distributed_simulation/bench.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0b981fa13..58c0c6393 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -32,6 +32,7 @@ jobs: with: name: Python Benchmark with pytest-benchmark tool: 'pytest' + gh-pages-branch: gh-bench-pages output-file-path: examples/distributed_simulation/output.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true diff --git a/examples/distributed_simulation/bench.py b/examples/distributed_simulation/bench.py index 80dae65a7..64a8537b0 100644 --- a/examples/distributed_simulation/bench.py +++ b/examples/distributed_simulation/bench.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """Simulation benchmark Please install pytest and pytest-benchmark to run this benchmark. @@ -5,12 +6,12 @@ import pytest # pylint: disable=W0611 from participant import Moderator, RandomParticipant, LLMParticipant -from main import run_main_process +from main import run_main_process # pylint: disable=E0611 import agentscope from agentscope.server import RpcAgentServerLauncher -def test_simulation(benchmark): +def test_simulation(benchmark): # type: ignore[no-untyped-def] """A single benchmark for the simulation""" base_port = 23300 par_server_num = 4 From 6cca9020b5f643d4622b3e2832b575dd686bfe03 Mon Sep 17 00:00:00 2001 From: "panxuchen.pxc" Date: Thu, 11 Jul 2024 11:34:26 +0800 Subject: [PATCH 3/8] update bench workflow --- .github/workflows/benchmark.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 58c0c6393..cce9c17f8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,6 +2,9 @@ name: AgentScope Benchmark on: [push, pull_request] +permissions: + contents: write + jobs: bench: runs-on: ${{ matrix.os }} From 75557ba02cc7909184c9fcb13c0106c8216bd88f Mon Sep 17 00:00:00 2001 From: "panxuchen.pxc" Date: Thu, 11 Jul 2024 11:55:35 +0800 Subject: [PATCH 4/8] update bench workflow --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cce9c17f8..5f28e0409 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,6 +3,7 @@ name: AgentScope Benchmark on: [push, pull_request] permissions: + deployments: write contents: write jobs: @@ -42,5 +43,4 @@ jobs: # Show alert with commit comment on detecting possible performance regression alert-threshold: '150%' comment-on-alert: true - fail-on-alert: true alert-comment-cc-users: '@pan-x-c' \ No newline at end of file From 34391138aaa598eb1cb8bf30747f902193d82c26 Mon Sep 17 00:00:00 2001 From: "panxuchen.pxc" Date: Thu, 11 Jul 2024 12:01:19 +0800 Subject: [PATCH 5/8] update bench workflow permissions --- .github/workflows/benchmark.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5f28e0409..60033424d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,12 +2,11 @@ name: AgentScope Benchmark on: [push, pull_request] -permissions: - deployments: write - contents: write jobs: bench: + permissions: + contents: write runs-on: ${{ matrix.os }} strategy: fail-fast: True From 49a6d4590b7be7d7288b449980f02fe2549be53a Mon Sep 17 00:00:00 2001 From: "panxuchen.pxc" Date: Thu, 11 Jul 2024 15:30:31 +0800 Subject: [PATCH 6/8] update bench workflow --- .github/workflows/benchmark.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 60033424d..433cab3f5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,7 +1,8 @@ name: AgentScope Benchmark on: [push, pull_request] - +permissions: + contents: write jobs: bench: @@ -35,11 +36,6 @@ jobs: with: name: Python Benchmark with pytest-benchmark tool: 'pytest' - gh-pages-branch: gh-bench-pages output-file-path: examples/distributed_simulation/output.json github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '150%' - comment-on-alert: true - alert-comment-cc-users: '@pan-x-c' \ No newline at end of file + auto-push: false \ No newline at end of file From 23b17ba38fc71cfe9cb9304583e6bb2ca8508e42 Mon Sep 17 00:00:00 2001 From: "panxuchen.pxc" Date: Fri, 12 Jul 2024 11:32:04 +0800 Subject: [PATCH 7/8] update benchmark --- .github/workflows/benchmark.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 433cab3f5..95b5a65a0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -30,12 +30,4 @@ jobs: run: | cd examples/distributed_simulation pytest bench.py --benchmark-json output.json - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/distributed_simulation/output.json - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: false \ No newline at end of file + cat output.json| jq -r ".benchmarks[] | .stats \ No newline at end of file From 8c7eb93d763d50d1042a1d00fc7c8b81c0d63ccc Mon Sep 17 00:00:00 2001 From: "panxuchen.pxc" Date: Fri, 12 Jul 2024 11:41:05 +0800 Subject: [PATCH 8/8] update benchmark --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 95b5a65a0..8c0643554 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -30,4 +30,4 @@ jobs: run: | cd examples/distributed_simulation pytest bench.py --benchmark-json output.json - cat output.json| jq -r ".benchmarks[] | .stats \ No newline at end of file + cat output.json| jq -r ".benchmarks[] | .stats" \ No newline at end of file