From b7c4151e1cd71646ca22eb4e24dc1313f6f45f05 Mon Sep 17 00:00:00 2001 From: Jan Ernsting Date: Fri, 21 Jul 2023 12:10:02 -0400 Subject: [PATCH 1/3] Added simple percentile config selector --- examples/ohbm2023/config_selectors.py | 18 ++++++++++++++++++ examples/ohbm2023/run.py | 7 ++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/examples/ohbm2023/config_selectors.py b/examples/ohbm2023/config_selectors.py index da168670..0a0466f9 100644 --- a/examples/ohbm2023/config_selectors.py +++ b/examples/ohbm2023/config_selectors.py @@ -47,3 +47,21 @@ def __call__(self, list_of_non_failed_configs, metric, fold_operation, maximize_ return best_config_outer_fold + +class PercentileConfigSelector(BaseConfigSelector): + + def __init__(self, percentile: int = 50): + self.percentile = percentile + + def __call__(self, list_of_non_failed_configs, metric, fold_operation, maximize_metric): + + all_metrics_mean, all_metrics_std = self.prepare_metrics(list_of_non_failed_configs, metric) + best_config_metric_values = [c.get_test_metric(metric, fold_operation) for c in list_of_non_failed_configs] + + position = (self.percentile / 100.) if maximize_metric else 1 - (self.percentile / 100.) + index = position * (len(best_config_metric_values) - 1) + index = int(index + 0.5) # index to integer conversion + # following line is a hack for speedup + best_config_outer_fold = list_of_non_failed_configs[np.argpartition(best_config_metric_values, index)[index]] + + return best_config_outer_fold diff --git a/examples/ohbm2023/run.py b/examples/ohbm2023/run.py index b35894e8..d1932281 100644 --- a/examples/ohbm2023/run.py +++ b/examples/ohbm2023/run.py @@ -1,19 +1,20 @@ import pandas as pd from run_elements import * -from config_selectors import DefaultConfigSelector, RandomConfigSelector +from config_selectors import DefaultConfigSelector, RandomConfigSelector, PercentileConfigSelector from collect_results import ResultCollector from multiprocessing import Process import os list_of_config_selectors = {'default': DefaultConfigSelector, - 'random': RandomConfigSelector} + 'percentile': PercentileConfigSelector} + #'random': RandomConfigSelector} config_selector_name = 'default' multiprocessing = False -calculate = False +calculate = True list_of_dataset_runners = { From 4dc2fa0c2e8c09269e9f48ac54c172ddd3a36ace Mon Sep 17 00:00:00 2001 From: Jan Ernsting Date: Fri, 21 Jul 2023 15:17:12 -0400 Subject: [PATCH 2/3] Update examples/ohbm2023/run.py --- examples/ohbm2023/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ohbm2023/run.py b/examples/ohbm2023/run.py index d1932281..86013f40 100644 --- a/examples/ohbm2023/run.py +++ b/examples/ohbm2023/run.py @@ -9,7 +9,7 @@ list_of_config_selectors = {'default': DefaultConfigSelector, 'percentile': PercentileConfigSelector} - #'random': RandomConfigSelector} + 'random': RandomConfigSelector} config_selector_name = 'default' From 40da81fd35c531388870072b69e1b1592b4195ea Mon Sep 17 00:00:00 2001 From: Jan Ernsting Date: Fri, 21 Jul 2023 15:17:17 -0400 Subject: [PATCH 3/3] Update examples/ohbm2023/config_selectors.py --- examples/ohbm2023/config_selectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ohbm2023/config_selectors.py b/examples/ohbm2023/config_selectors.py index 0a0466f9..a4d0b7b6 100644 --- a/examples/ohbm2023/config_selectors.py +++ b/examples/ohbm2023/config_selectors.py @@ -50,7 +50,7 @@ def __call__(self, list_of_non_failed_configs, metric, fold_operation, maximize_ class PercentileConfigSelector(BaseConfigSelector): - def __init__(self, percentile: int = 50): + def __init__(self, percentile: int = 75): self.percentile = percentile def __call__(self, list_of_non_failed_configs, metric, fold_operation, maximize_metric):