test: codspeed CI workflow

.github/workflows/codspeed.yml

@@ -0,0 +1,28 @@
+name: codspeed benchmarks
+
+on:
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  codspeed-benchmarks:
+    name: codspeed benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install uv
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install dependencies
+        run: uv pip install -e . -r requirements-dev.txt --system
+      - name: show-deps
+        run: uv pip freeze
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          run: pytest tests/tpch -m "benchmark" --codspeed

pyproject.toml

@@ -96,10 +96,16 @@ filterwarnings = [
   'ignore:.*but when imported by',
 ]
 xfail_strict = true
-markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"]
+markers = [
+  "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+  "benchmark: marks tests as benchmark (deselect with '-m \"not benchmark\"')"
+]
 env = [
     "MODIN_ENGINE=python",
 ]
+addopts = [
+  "-m not benchmark",
+]
 
 [tool.coverage.run]
 plugins = ["covdefaults"]

requirements-dev.txt

@@ -4,6 +4,8 @@ polars
 pre-commit
 pyarrow
 pytest
+pytest-benchmark
+pytest-codspeed
 pytest-cov
 pytest-env
 hypothesis

tests/tpch/q1_test.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from datetime import date
+from typing import Any
+
+import pandas as pd
+import polars as pl
+import pyarrow.parquet as pq
+import pytest
+
+import narwhals.stable.v1 as nw
+from narwhals.utils import parse_version
+
+pytestmark = pytest.mark.benchmark
+
+
+def q1(lineitem_ds: Any) -> Any:
+    var_1 = date(1998, 9, 2)
+    query_result = (
+        lineitem_ds.filter(nw.col("l_shipdate") <= var_1)
+        .with_columns(
+            disc_price=nw.col("l_extendedprice") * (1 - nw.col("l_discount")),
+            charge=(
+                nw.col("l_extendedprice")
+                * (1.0 - nw.col("l_discount"))
+                * (1.0 + nw.col("l_tax"))
+            ),
+        )
+        .group_by(["l_returnflag", "l_linestatus"])
+        .agg(
+            [
+                nw.col("l_quantity").sum().alias("sum_qty"),
+                nw.col("l_extendedprice").sum().alias("sum_base_price"),
+                nw.col("disc_price").sum().alias("sum_disc_price"),
+                nw.col("charge").sum().alias("sum_charge"),
+                nw.col("l_quantity").mean().alias("avg_qty"),
+                nw.col("l_extendedprice").mean().alias("avg_price"),
+                nw.col("l_discount").mean().alias("avg_disc"),
+                nw.len().alias("count_order"),
+            ],
+        )
+        .sort(["l_returnflag", "l_linestatus"])
+    )
+    return query_result.collect()
+
+
+@pytest.mark.parametrize("library", ["pandas", "polars", "pyarrow", "dask"])
+def test_q1(benchmark: Any, library: str, request: Any) -> None:
+    if library == "pandas" and parse_version(pd.__version__) < (1, 5):
+        request.applymarker(pytest.mark.xfail)
+
+    import dask.dataframe as dd
+
+    lib_to_reader = {
+        "pandas": pd.read_parquet,
+        "polars": pl.scan_parquet,
+        "dask": lambda path: dd.read_parquet(path, dtype_backend="pyarrow"),
+        "pyarrow": pq.read_table,
+    }
+
+    read_fn = lib_to_reader[library]
+    lineitem_ds = nw.from_native(read_fn("tests/data/lineitem.parquet")).lazy()
+
+    _ = benchmark(q1, lineitem_ds)

tests/tpch/q2_test.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+import polars as pl
+import pyarrow.parquet as pq
+import pytest
+
+import narwhals.stable.v1 as nw
+from narwhals.utils import parse_version
+
+pytestmark = pytest.mark.benchmark
+
+
+def q2(
+    region_ds: Any,
+    nation_ds: Any,
+    supplier_ds: Any,
+    part_ds: Any,
+    part_supp_ds: Any,
+) -> Any:
+    var_1 = 15
+    var_2 = "BRASS"
+    var_3 = "EUROPE"
+
+    tmp = (
+        part_ds.join(part_supp_ds, left_on="p_partkey", right_on="ps_partkey")
+        .join(supplier_ds, left_on="ps_suppkey", right_on="s_suppkey")
+        .join(nation_ds, left_on="s_nationkey", right_on="n_nationkey")
+        .join(region_ds, left_on="n_regionkey", right_on="r_regionkey")
+        .filter(
+            nw.col("p_size") == var_1,
+            nw.col("p_type").str.ends_with(var_2),
+            nw.col("r_name") == var_3,
+        )
+    )
+
+    final_cols = [
+        "s_acctbal",
+        "s_name",
+        "n_name",
+        "p_partkey",
+        "p_mfgr",
+        "s_address",
+        "s_phone",
+        "s_comment",
+    ]
+
+    query_result = (
+        tmp.group_by("p_partkey")
+        .agg(nw.col("ps_supplycost").min().alias("ps_supplycost"))
+        .join(
+            tmp,
+            left_on=["p_partkey", "ps_supplycost"],
+            right_on=["p_partkey", "ps_supplycost"],
+        )
+        .select(final_cols)
+        .sort(
+            ["s_acctbal", "n_name", "s_name", "p_partkey"],
+            descending=[True, False, False, False],
+        )
+        .head(100)
+    )
+    return query_result.collect()
+
+
+@pytest.mark.parametrize("library", ["pandas", "polars", "pyarrow", "dask"])
+def test_q2(benchmark: Any, library: str, request: Any) -> None:
+    if library == "pandas" and parse_version(pd.__version__) < (1, 5):
+        request.applymarker(pytest.mark.xfail)
+
+    import dask.dataframe as dd
+
+    lib_to_reader = {
+        "pandas": pd.read_parquet,
+        "polars": pl.scan_parquet,
+        "dask": lambda path: dd.read_parquet(path, dtype_backend="pyarrow"),
+        "pyarrow": pq.read_table,
+    }
+
+    read_fn = lib_to_reader[library]
+    region_ds = nw.from_native(read_fn("tests/data/region.parquet")).lazy()
+    nation_ds = nw.from_native(read_fn("tests/data/nation.parquet")).lazy()
+    supplier_ds = nw.from_native(read_fn("tests/data/supplier.parquet")).lazy()
+    part_ds = nw.from_native(read_fn("tests/data/part.parquet")).lazy()
+    part_supp_ds = nw.from_native(read_fn("tests/data/partsupp.parquet")).lazy()
+
+    args = (region_ds, nation_ds, supplier_ds, part_ds, part_supp_ds)
+
+    _ = benchmark(q2, *args)
+
+    # Need to create expected compare_dicts(result, q2_expected)