Added MO for HPOBench

carps/benchmarks/hpo_bench.py

@@ -4,6 +4,7 @@
 
 import time
 from typing import Any, Optional
+from omegaconf import ListConfig
 
 from ConfigSpace import ConfigurationSpace
 from hpobench.benchmarks.ml.lr_benchmark import LRBenchmark
@@ -28,6 +29,7 @@ def __init__(
         seed: int,
         model: str | None = None,
         task_id: int | None = None,
+        metric: str | list[str] = "function_value",
         problem: AbstractBenchmarkClient | None = None,
         budget_type: Optional[str] = None,
         loggers: list[AbstractLogger] | None = None,
@@ -57,6 +59,9 @@ def __init__(
         self._problem = problem if problem else get_hpobench_problem(
             task_id=task_id, model=model, seed=seed, budget_type=self.budget_type
         )
+        if not isinstance(metric, (list, ListConfig)):
+            metric = [metric]
+        self.metric = metric
         self._configspace = self._problem.get_configuration_space(seed=seed)
 
     @property
@@ -102,9 +107,12 @@ def _evaluate(self, trial_info: TrialInfo) -> TrialValue:
         endtime = time.time()
         T = endtime - starttime
         virtual_time = result_dict["cost"] if trial_info.budget is None else 0.0
+        costs = [result_dict[metric] for metric in self.metric]
+        if len(costs) == 1:
+            costs = costs[0]
         # function_value is 1 - accuracy on the validation set
         trial_value = TrialValue(
-            cost=result_dict["function_value"],
+            cost=costs,
             time=T,
             starttime=starttime,
             endtime=endtime,

carps/configs/problem/HPOBench/MO/tab/cfg_ml_lr_10101.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+benchmark_id: HPOBench
+problem_id: hpobench/MO/tab/ml/lr/10101
+problem:
+  _target_: carps.benchmarks.hpo_bench.HPOBenchProblem
+  model: lr
+  task_id: 10101
+  metric:
+  - function_value
+  - cost
+  budget_type: null
+  seed: ${seed}
+task:
+  n_trials: 77
+  time_budget: null
+  n_workers: 1
+  n_objectives: 2
+  objectives:
+  - function_value
+  - cost
+  is_multifidelity: false
+  fidelity_type: null
+  min_budget: null
+  max_budget: null
+  has_constraints: false
+  domain: ML
+  objective_function_approximation: tabular
+  has_virtual_time: false
+  deterministic: false
+  dimensions: 2
+  search_space_n_categoricals: 0
+  search_space_n_ordinals: 2
+  search_space_n_integers: 0
+  search_space_n_floats: 0
+  search_space_has_conditionals: false
+  search_space_has_forbiddens: false
+  search_space_has_priors: false

carps/configs/problem/HPOBench/MO/tab/cfg_ml_lr_12.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+benchmark_id: HPOBench
+problem_id: hpobench/MO/tab/ml/lr/12
+problem:
+  _target_: carps.benchmarks.hpo_bench.HPOBenchProblem
+  model: lr
+  task_id: 12
+  metric:
+  - function_value
+  - cost
+  budget_type: null
+  seed: ${seed}
+task:
+  n_trials: 77
+  time_budget: null
+  n_workers: 1
+  n_objectives: 2
+  objectives:
+  - function_value
+  - cost
+  is_multifidelity: false
+  fidelity_type: null
+  min_budget: null
+  max_budget: null
+  has_constraints: false
+  domain: ML
+  objective_function_approximation: tabular
+  has_virtual_time: false
+  deterministic: false
+  dimensions: 2
+  search_space_n_categoricals: 0
+  search_space_n_ordinals: 2
+  search_space_n_integers: 0
+  search_space_n_floats: 0
+  search_space_has_conditionals: false
+  search_space_has_forbiddens: false
+  search_space_has_priors: false

carps/configs/problem/HPOBench/MO/tab/cfg_ml_lr_146212.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+benchmark_id: HPOBench
+problem_id: hpobench/MO/tab/ml/lr/146212
+problem:
+  _target_: carps.benchmarks.hpo_bench.HPOBenchProblem
+  model: lr
+  task_id: 146212
+  metric:
+  - function_value
+  - cost
+  budget_type: null
+  seed: ${seed}
+task:
+  n_trials: 77
+  time_budget: null
+  n_workers: 1
+  n_objectives: 2
+  objectives:
+  - function_value
+  - cost
+  is_multifidelity: false
+  fidelity_type: null
+  min_budget: null
+  max_budget: null
+  has_constraints: false
+  domain: ML
+  objective_function_approximation: tabular
+  has_virtual_time: false
+  deterministic: false
+  dimensions: 2
+  search_space_n_categoricals: 0
+  search_space_n_ordinals: 2
+  search_space_n_integers: 0
+  search_space_n_floats: 0
+  search_space_has_conditionals: false
+  search_space_has_forbiddens: false
+  search_space_has_priors: false

carps/configs/problem/HPOBench/MO/tab/cfg_ml_lr_146606.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+benchmark_id: HPOBench
+problem_id: hpobench/MO/tab/ml/lr/146606
+problem:
+  _target_: carps.benchmarks.hpo_bench.HPOBenchProblem
+  model: lr
+  task_id: 146606
+  metric:
+  - function_value
+  - cost
+  budget_type: null
+  seed: ${seed}
+task:
+  n_trials: 77
+  time_budget: null
+  n_workers: 1
+  n_objectives: 2
+  objectives:
+  - function_value
+  - cost
+  is_multifidelity: false
+  fidelity_type: null
+  min_budget: null
+  max_budget: null
+  has_constraints: false
+  domain: ML
+  objective_function_approximation: tabular
+  has_virtual_time: false
+  deterministic: false
+  dimensions: 2
+  search_space_n_categoricals: 0
+  search_space_n_ordinals: 2
+  search_space_n_integers: 0
+  search_space_n_floats: 0
+  search_space_has_conditionals: false
+  search_space_has_forbiddens: false
+  search_space_has_priors: false

carps/configs/problem/HPOBench/MO/tab/cfg_ml_lr_146818.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+benchmark_id: HPOBench
+problem_id: hpobench/MO/tab/ml/lr/146818
+problem:
+  _target_: carps.benchmarks.hpo_bench.HPOBenchProblem
+  model: lr
+  task_id: 146818
+  metric:
+  - function_value
+  - cost
+  budget_type: null
+  seed: ${seed}
+task:
+  n_trials: 77
+  time_budget: null
+  n_workers: 1
+  n_objectives: 2
+  objectives:
+  - function_value
+  - cost
+  is_multifidelity: false
+  fidelity_type: null
+  min_budget: null
+  max_budget: null
+  has_constraints: false
+  domain: ML
+  objective_function_approximation: tabular
+  has_virtual_time: false
+  deterministic: false
+  dimensions: 2
+  search_space_n_categoricals: 0
+  search_space_n_ordinals: 2
+  search_space_n_integers: 0
+  search_space_n_floats: 0
+  search_space_has_conditionals: false
+  search_space_has_forbiddens: false
+  search_space_has_priors: false

carps/configs/problem/HPOBench/MO/tab/cfg_ml_lr_146821.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+benchmark_id: HPOBench
+problem_id: hpobench/MO/tab/ml/lr/146821
+problem:
+  _target_: carps.benchmarks.hpo_bench.HPOBenchProblem
+  model: lr
+  task_id: 146821
+  metric:
+  - function_value
+  - cost
+  budget_type: null
+  seed: ${seed}
+task:
+  n_trials: 77
+  time_budget: null
+  n_workers: 1
+  n_objectives: 2
+  objectives:
+  - function_value
+  - cost
+  is_multifidelity: false
+  fidelity_type: null
+  min_budget: null
+  max_budget: null
+  has_constraints: false
+  domain: ML
+  objective_function_approximation: tabular
+  has_virtual_time: false
+  deterministic: false
+  dimensions: 2
+  search_space_n_categoricals: 0
+  search_space_n_ordinals: 2
+  search_space_n_integers: 0
+  search_space_n_floats: 0
+  search_space_has_conditionals: false
+  search_space_has_forbiddens: false
+  search_space_has_priors: false

carps/configs/problem/HPOBench/MO/tab/cfg_ml_lr_146822.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+benchmark_id: HPOBench
+problem_id: hpobench/MO/tab/ml/lr/146822
+problem:
+  _target_: carps.benchmarks.hpo_bench.HPOBenchProblem
+  model: lr
+  task_id: 146822
+  metric:
+  - function_value
+  - cost
+  budget_type: null
+  seed: ${seed}
+task:
+  n_trials: 77
+  time_budget: null
+  n_workers: 1
+  n_objectives: 2
+  objectives:
+  - function_value
+  - cost
+  is_multifidelity: false
+  fidelity_type: null
+  min_budget: null
+  max_budget: null
+  has_constraints: false
+  domain: ML
+  objective_function_approximation: tabular
+  has_virtual_time: false
+  deterministic: false
+  dimensions: 2
+  search_space_n_categoricals: 0
+  search_space_n_ordinals: 2
+  search_space_n_integers: 0
+  search_space_n_floats: 0
+  search_space_has_conditionals: false
+  search_space_has_forbiddens: false
+  search_space_has_priors: false

carps/configs/problem/HPOBench/MO/tab/cfg_ml_lr_14965.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+benchmark_id: HPOBench
+problem_id: hpobench/MO/tab/ml/lr/14965
+problem:
+  _target_: carps.benchmarks.hpo_bench.HPOBenchProblem
+  model: lr
+  task_id: 14965
+  metric:
+  - function_value
+  - cost
+  budget_type: null
+  seed: ${seed}
+task:
+  n_trials: 77
+  time_budget: null
+  n_workers: 1
+  n_objectives: 2
+  objectives:
+  - function_value
+  - cost
+  is_multifidelity: false
+  fidelity_type: null
+  min_budget: null
+  max_budget: null
+  has_constraints: false
+  domain: ML
+  objective_function_approximation: tabular
+  has_virtual_time: false
+  deterministic: false
+  dimensions: 2
+  search_space_n_categoricals: 0
+  search_space_n_ordinals: 2
+  search_space_n_integers: 0
+  search_space_n_floats: 0
+  search_space_has_conditionals: false
+  search_space_has_forbiddens: false
+  search_space_has_priors: false

carps/configs/problem/HPOBench/MO/tab/cfg_ml_lr_167119.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+benchmark_id: HPOBench
+problem_id: hpobench/MO/tab/ml/lr/167119
+problem:
+  _target_: carps.benchmarks.hpo_bench.HPOBenchProblem
+  model: lr
+  task_id: 167119
+  metric:
+  - function_value
+  - cost
+  budget_type: null
+  seed: ${seed}
+task:
+  n_trials: 77
+  time_budget: null
+  n_workers: 1
+  n_objectives: 2
+  objectives:
+  - function_value
+  - cost
+  is_multifidelity: false
+  fidelity_type: null
+  min_budget: null
+  max_budget: null
+  has_constraints: false
+  domain: ML
+  objective_function_approximation: tabular
+  has_virtual_time: false
+  deterministic: false
+  dimensions: 2
+  search_space_n_categoricals: 0
+  search_space_n_ordinals: 2
+  search_space_n_integers: 0
+  search_space_n_floats: 0
+  search_space_has_conditionals: false
+  search_space_has_forbiddens: false
+  search_space_has_priors: false