v0.1.1

CHANGELOG.md

@@ -1,3 +1,10 @@
-# Version 0.1.0
+
+# v0.1.1
+
+## Bug Fixes
+- Fix HEBO integration (#149)
+- Fix plotting and data gathering
+
+# v0.1.0
 
 - Initial version of CARP-S.

CITATION.cff

@@ -10,7 +10,7 @@ url: "https://automl.github.io/CARP-S/main/"
 
 repository-code: "https://github.com/automl/CARP-S"
 
-version: "0.1.0"
+version: "0.1.1"
 
 type: "template"
 keywords:

Makefile

@@ -5,7 +5,7 @@ SHELL := /bin/bash
 
 NAME := CARP-S
 PACKAGE_NAME := carps
-VERSION := 0.1.0
+VERSION := 0.1.1
 
 DIST := dist
 

carps/analysis/concat_rundata.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+import pandas as pd
+from carps.analysis.gather_data import load_set, convert_mixed_types_to_str
+
+def concat_rundata():
+    paths = {
+        "BBsubset": {
+            "dev": ["runs_subset_BB/dev"],
+            "test": ["runs_subset_BB/test"],
+        },
+        "MFsubset": {
+            "dev": ["runs_subset_MF/dev"],
+            "test": ["runs_subset_MF/test"],
+        },
+        "MOsubset": {
+            "dev": ["runs_subset_MO/dev"],
+            "test": ["runs_subset_MO/test"],
+        },
+        "MOMFsubset": {
+            "dev": ["runs_subset_MOMF/dev"],
+            "test": ["runs_subset_MOMF/test"],
+        },
+    }
+
+    args = []
+    for item in paths.values():
+        for k,v in item.items():
+            from pathlib import Path
+            args.append((v,k))
+    res = [load_set(paths=a[0], set_id=a[1]) for a in args]
+    df = pd.concat([r[0] for r in res]).reset_index(drop=True)
+    df = convert_mixed_types_to_str(df)
+    df.to_parquet("rundata.parquet")
+
+if __name__ == "__main__":
+    concat_rundata()

carps/analysis/performance_over_time.py

@@ -14,8 +14,8 @@
 if TYPE_CHECKING:
     import pandas as pd
 
-def get_order_by_mean(df: pd.DataFrame) -> list[str]:
-    final_df = filter_only_final_performance(df)
+def get_order_by_mean(df: pd.DataFrame, budget_var: str = "n_trials_norm") -> list[str]:
+    final_df = filter_only_final_performance(df, budget_var=budget_var)
     reduced = final_df.groupby(by="optimizer_id")["trial_value__cost_inc_norm"].apply(np.nanmean)
     reduced = reduced.sort_values()
     return reduced.index.tolist()
@@ -33,7 +33,7 @@ def plot_performance_over_time(
     **lineplot_kwargs,
 ) -> tuple[plt.Figure, matplotlib.axes.Axes]:
     setup_seaborn(font_scale=1.5)
-    sorter = get_order_by_mean(df=df)
+    sorter = get_order_by_mean(df=df, budget_var=x)
     df = df.sort_values(by="optimizer_id", key=lambda column: column.map(lambda e: sorter.index(e)))
     palette = get_color_palette(df)
     fig = plt.figure(figsize=figsize)

carps/benchmarks/hpo_b.py

@@ -234,7 +234,7 @@ def __init__(
         if not surrogates_file.is_file():
             raise RuntimeError(
                 "It seems that the surrogate files have not been downloaded. Please run "
-                "'bash container_recipes/benchmarks/hpob/download_data.sh' to download the "
+                "'bash container_recipes/benchmarks/HPOB/download_data.sh' to download the "
                 "surrogates."
             )
         with open(str(surrogates_file)) as f:

carps/optimizers/hebo.py

@@ -271,7 +271,6 @@ def tell(self, trial_info: TrialInfo, trial_value: TrialValue) -> None:
         cost = trial_value.cost
         suggestion = self.convert_from_trial(trial_info=trial_info)
 
-        self.trial_counter += 1
         cost = np.asarray([cost]) if isinstance(cost, abc.Sequence) else np.asarray(cost)
 
         self._solver.observe(suggestion, np.asarray([cost]))

carps/utils/check_missing.py

@@ -30,7 +30,7 @@ def get_experiment_status(path: Path) -> dict:
     if trial_logs_fn.is_file():
         trial_logs = read_jsonl_content(str(trial_logs_fn))
         n_trials_done = trial_logs["n_trials"].max()
-        status = RunStatus.COMPLETED if n_trials == n_trials_done else RunStatus.TRUNCATED
+        status = RunStatus.COMPLETED if n_trials >= n_trials_done else RunStatus.TRUNCATED
 
     overrides = OmegaConf.load(path.parent / "overrides.yaml")
     # TODO maybe filter cluster

notebooks/calc_runtime.py

@@ -1,39 +1,10 @@
 from __future__ import annotations
 
-from carps.analysis.gather_data import load_set, convert_mixed_types_to_str
 from carps.analysis.utils import filter_only_final_performance
 import pandas as pd
 import multiprocessing
 
-paths = {
-    "BBsubset": {
-        "dev": ["../runs_subset_BB/dev"],
-        "test": ["../runs_subset_BB/test"],
-    },
-    "MFsubset": {
-        "dev": ["../runs_subset_MF/dev"],
-        "test": ["../runs_subset_MF/test"],
-    },
-    "MOsubset": {
-        "dev": ["../runs_subset_MO/dev"],
-        "test": ["../runs_subset_MO/test"],
-    },
-    "MOMFsubset": {
-        "dev": ["../runs_subset_MOMF/dev"],
-        "test": ["../runs_subset_MOMF/test"],
-    },
-}
-
-
-
-# args = []
-# for item in paths.values():
-#     for k,v in item.items():
-#         args.append((v,k))
-# res = [load_set(paths=a[0], set_id=a[1]) for a in args]
-# df = pd.concat([r[0] for r in res]).reset_index(drop=True)
-# df = convert_mixed_types_to_str(df)
-# df.to_parquet("rundata.parquet")
+concat_rundata()
 
 df  = pd.read_parquet("rundata.parquet")
 df_final = filter_only_final_performance(df=df, budget_var="n_trials")