From 11dd9a8758bbfcce389b4834d9322d12c11715c1 Mon Sep 17 00:00:00 2001
From: Daniel <63580393+danrgll@users.noreply.github.com>
Date: Tue, 2 Jul 2024 09:27:22 +0200
Subject: [PATCH 1/7] change path of yaml test directories to tests_tmpdir

---
 .../test_declarative_usage_docs/customizing_neps_optimizer.yaml | 2 +-
 .../test_declarative_usage_docs/defining_hooks.yaml             | 2 +-
 .../full_configuration_template.yaml                            | 2 +-
 .../test_declarative_usage_docs/loading_own_optimizer.yaml      | 2 +-
 .../loading_pipeline_space_dict.yaml                            | 2 +-
 .../test_declarative_usage_docs/outsourcing_optimizer.yaml      | 2 +-
 .../test_declarative_usage_docs/outsourcing_pipeline_space.yaml | 2 +-
 .../test_declarative_usage_docs/simple_example.yaml             | 2 +-
 .../simple_example_including_run_pipeline.yaml                  | 2 +-
 .../test_run_args_by_neps_run/loading_optimizer.yaml            | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/customizing_neps_optimizer.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/customizing_neps_optimizer.yaml
index d31d1f3f..5ddaf23e 100644
--- a/tests/test_yaml_run_args/test_declarative_usage_docs/customizing_neps_optimizer.yaml
+++ b/tests/test_yaml_run_args/test_declarative_usage_docs/customizing_neps_optimizer.yaml
@@ -15,7 +15,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   batch_size: 64
 
-root_directory: path/to/results/custominizing_neps_optimizer       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/custominizing_neps_optimizer"
 max_evaluations_total: 20               # Budget
 searcher:
   strategy: bayesian_optimization
diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/defining_hooks.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/defining_hooks.yaml
index 922bb542..83842f93 100644
--- a/tests/test_yaml_run_args/test_declarative_usage_docs/defining_hooks.yaml
+++ b/tests/test_yaml_run_args/test_declarative_usage_docs/defining_hooks.yaml
@@ -16,7 +16,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   batch_size: 64
 
-root_directory: path/to/results/hooks       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/hooks"
 max_evaluations_total: 20               # Budget
 
 pre_load_hooks:
diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/full_configuration_template.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/full_configuration_template.yaml
index b79616a4..21851c88 100644
--- a/tests/test_yaml_run_args/test_declarative_usage_docs/full_configuration_template.yaml
+++ b/tests/test_yaml_run_args/test_declarative_usage_docs/full_configuration_template.yaml
@@ -16,7 +16,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   batch_size: 64
 
-root_directory: path/to/results/full_config      # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/full_config"
 max_evaluations_total: 20             # Budget
 max_cost_total:
 
diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/loading_own_optimizer.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/loading_own_optimizer.yaml
index 2e0e664b..fce52034 100644
--- a/tests/test_yaml_run_args/test_declarative_usage_docs/loading_own_optimizer.yaml
+++ b/tests/test_yaml_run_args/test_declarative_usage_docs/loading_own_optimizer.yaml
@@ -11,7 +11,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   epochs: 50
 
-root_directory: path/to/results       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/loading_own_optimizer"
 max_evaluations_total: 20               # Budget
 searcher:
   path: "neps/optimizers/bayesian_optimization/optimizer.py"
diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/loading_pipeline_space_dict.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/loading_pipeline_space_dict.yaml
index 64366e3a..909e9559 100644
--- a/tests/test_yaml_run_args/test_declarative_usage_docs/loading_pipeline_space_dict.yaml
+++ b/tests/test_yaml_run_args/test_declarative_usage_docs/loading_pipeline_space_dict.yaml
@@ -7,5 +7,5 @@ pipeline_space:
   path: tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.py
   name: pipeline_space                  # Name of the dict instance
 
-root_directory: path/to/results_loading_pipeline_space       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/results_loading_pipeline_space"
 max_evaluations_total: 20               # Budget
diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_optimizer.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_optimizer.yaml
index c905f55b..e8e00aa3 100644
--- a/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_optimizer.yaml
+++ b/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_optimizer.yaml
@@ -12,7 +12,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   epochs: 50
 
-root_directory: path/to/results_outsourcing_optimizer      # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/outsourcing_optimizer"
 max_evaluations_total: 20               # Budget
 
 searcher: tests/test_yaml_run_args/test_declarative_usage_docs/set_up_optimizer.yaml
diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_pipeline_space.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_pipeline_space.yaml
index f28781ec..0810b62e 100644
--- a/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_pipeline_space.yaml
+++ b/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_pipeline_space.yaml
@@ -5,6 +5,6 @@ run_pipeline:
 
 pipeline_space: tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.yaml
 
-root_directory: path/to/results_outsourcing_pipeline_space       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/outsourcing_pipeline_space"
 max_evaluations_total: 20               # Budget
 
diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example.yaml
index a167964c..920d15cd 100644
--- a/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example.yaml
+++ b/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example.yaml
@@ -12,7 +12,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   batch_size: 64
 
-root_directory: path/to/results/simple_example       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/simple_example"
 max_evaluations_total: 20             # Budget
 
 
diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example_including_run_pipeline.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example_including_run_pipeline.yaml
index 53cde312..fbf36873 100644
--- a/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example_including_run_pipeline.yaml
+++ b/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example_including_run_pipeline.yaml
@@ -16,7 +16,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   batch_size: 64
 
-root_directory: path/to/results/simple_example_including_run_pipeline       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/simple_example_including_run_pipeline"
 max_evaluations_total: 20             # Budget
 
 overwrite_working_directory: True
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml
index b42d234e..0233239b 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml
@@ -2,7 +2,7 @@ run_pipeline:
   path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
   name: "run_pipeline"
 pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
-root_directory: "tests/test_yaml_run_args/test_run_args_by_neps_run/results"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/results"
 
 max_evaluations_total: 1
 max_cost_total:

From ae6a92c5eb103d0b0c0eb36a5f266ec12ac1ba4a Mon Sep 17 00:00:00 2001
From: Daniel <63580393+danrgll@users.noreply.github.com>
Date: Tue, 2 Jul 2024 09:34:07 +0200
Subject: [PATCH 2/7] change path of yaml test directories to tests_tmpdir

---
 tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml  | 2 +-
 .../test_run_args_by_neps_run/loading_optimizer.yaml            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml
index fda6d4cd..a5015aba 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml
@@ -2,7 +2,7 @@ run_pipeline:
   path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
   name: run_pipeline
 pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
-root_directory: "tests/test_yaml_run_args/test_run_args_by_neps_run/results"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/results2"
 
 max_evaluations_total: 1
 max_cost_total:
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml
index 0233239b..17f375dc 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml
@@ -2,7 +2,7 @@ run_pipeline:
   path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
   name: "run_pipeline"
 pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
-root_directory: "tests_tmpdir/test_run_args_by_neps_run/results"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/results1"
 
 max_evaluations_total: 1
 max_cost_total:

From c67ab2670a39491949ad90eef34b301dbfe90747 Mon Sep 17 00:00:00 2001
From: Daniel <63580393+danrgll@users.noreply.github.com>
Date: Tue, 2 Jul 2024 18:00:55 +0200
Subject: [PATCH 3/7] fix yaml dumping and default args selection for optimizer
 loaded via run_args

---
 neps/api.py                                   | 40 ++++++++++++-------
 neps/optimizers/default_searchers/pibo.yaml   |  2 +-
 .../solution_yamls/pibo_neps_decided.yaml     |  2 +-
 3 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/neps/api.py b/neps/api.py
index ab089d9f..3f8923ed 100644
--- a/neps/api.py
+++ b/neps/api.py
@@ -338,18 +338,27 @@ def _run_args(
         message = f"The pipeline_space has invalid type: {type(pipeline_space)}"
         raise TypeError(message) from e
 
+    # Load the information of the optimizer
     if isinstance(searcher, (str, Path)) and searcher not in \
         SearcherConfigs.get_searchers() and searcher != "default":
-        # The users has their own custom searcher.
+        # The users have their own custom searcher provided via yaml.
         logging.info("Preparing to run user created searcher")
 
-        config, searcher = get_searcher_data(searcher, loading_custom_searcher=True)
+        searcher_config, file_name = get_searcher_data(searcher,
+                                                      loading_custom_searcher=True)
+        # name defined via key or the filename of the yaml
+        searcher_name = searcher_config.pop("name", file_name)
         searcher_info["searcher_selection"] = "user-yaml"
         searcher_info["neps_decision_tree"] = False
     elif isinstance(searcher, dict):
-        config = searcher
-        searcher = config.pop("name", "unnamed-custom-searcher")
-        searcher_info["searcher_selection"] = "user-yaml"
+        custom_config = searcher
+        default_config, searcher_name = get_searcher_data(searcher["strategy"])
+        searcher_config = {**default_config, **custom_config}
+        if "name" not in searcher_config:
+            searcher_name = "custom_" + searcher_name
+        else:
+            searcher_name = searcher_config.pop("name")
+        searcher_info["searcher_selection"] = "user-run_args-yaml"
         searcher_info["neps_decision_tree"] = False
     else:
         if searcher in ["default", None]:
@@ -368,29 +377,30 @@ def _run_args(
             searcher_info["neps_decision_tree"] = False
             searcher_info["searcher_selection"] = "neps-default"
         # Fetching the searcher data, throws an error when the searcher is not found
-        config, searcher = get_searcher_data(searcher)
+        searcher_config, searcher_name = get_searcher_data(searcher)
 
     # Check for deprecated 'algorithm' argument
-    if "algorithm" in config:
+    if "algorithm" in searcher_config:
         warnings.warn(
-            "The 'algorithm' argument is deprecated and will be removed in future versions. Please use 'strategy' instead.",
+            "The 'algorithm' argument is deprecated and will be removed in "
+            "future versions. Please use 'strategy' instead.",
             DeprecationWarning
         )
         # Map the old 'algorithm' argument to 'strategy'
-        config['strategy'] = config.pop("algorithm")
+        searcher_config['strategy'] = searcher_config.pop("algorithm")
 
-    if "strategy" in config:
-        searcher_alg = config.pop("strategy")
+    if "strategy" in searcher_config:
+        searcher_alg = searcher_config.pop("strategy")
     else:
-        raise KeyError(f"Missing key strategy in searcher config:{config}")
-    searcher_config = config
+        raise KeyError(f"Missing key strategy in searcher config:{searcher_config}")
+
 
-    logger.info(f"Running {searcher} as the searcher")
+    logger.info(f"Running {searcher_name} as the searcher")
     logger.info(f"Strategy: {searcher_alg}")
 
     # Used to create the yaml holding information about the searcher.
     # Also important for testing and debugging the api.
-    searcher_info["searcher_name"] = searcher
+    searcher_info["searcher_name"] = searcher_name
     searcher_info["searcher_alg"] = searcher_alg
 
     # Updating searcher arguments from searcher_kwargs
diff --git a/neps/optimizers/default_searchers/pibo.yaml b/neps/optimizers/default_searchers/pibo.yaml
index 8274ea4b..9c386069 100644
--- a/neps/optimizers/default_searchers/pibo.yaml
+++ b/neps/optimizers/default_searchers/pibo.yaml
@@ -1,4 +1,4 @@
-strategy: bayesian_optimization
+strategy: pibo
 # Arguments that can be modified by the user
 initial_design_size: 10
 surrogate_model: gp  # or {"gp_hierarchy"}
diff --git a/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml b/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml
index dce4d40e..7d5f19da 100644
--- a/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml
+++ b/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml
@@ -1,5 +1,5 @@
 searcher_name: pibo
-searcher_alg: bayesian_optimization
+searcher_alg: pibo
 searcher_selection: neps-default
 neps_decision_tree: true
 searcher_args:

From 94ef5564f72cc3ebbd15b9b2bae31f3589f37856 Mon Sep 17 00:00:00 2001
From: Daniel <63580393+danrgll@users.noreply.github.com>
Date: Tue, 2 Jul 2024 22:00:54 +0200
Subject: [PATCH 4/7] add tests for checking optimizer loading via run_args

---
 .../test_run_args_by_neps_run/config.yaml     |  2 +-
 .../config_hyperband_mixed_args.yaml          | 30 +++++++++
 .../config_priorband_with_args.yaml           | 41 +++++++++++++
 .../config_select_bo.yaml                     | 23 +++++++
 .../loading_optimizer.yaml                    |  2 +-
 .../loading_pipeline_space.yaml               |  2 +-
 .../test_run_args_by_neps_run/neps_run.py     | 23 +++++--
 .../hyperband_searcher_kwargs_yaml_args.yaml  | 11 ++++
 .../priorband_args_run_args.yaml              | 23 +++++++
 .../optimizer_yamls/select_bo_run_args.yaml   | 13 ++++
 .../search_space_with_fidelity.yaml           | 11 ++++
 .../search_space_with_priors.yaml             | 15 +++++
 .../test_neps_run.py                          | 61 ++++++++++++++++---
 13 files changed, 239 insertions(+), 18 deletions(-)
 create mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml
 create mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml
 create mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/config_select_bo.yaml
 create mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml
 create mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml
 create mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/select_bo_run_args.yaml
 create mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml
 create mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml

diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml
index a5015aba..967e2c9c 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml
@@ -4,7 +4,7 @@ run_pipeline:
 pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
 root_directory: "tests_tmpdir/test_run_args_by_neps_run/results2"
 
-max_evaluations_total: 1
+max_evaluations_total: 5
 max_cost_total:
 
 monitoring:
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml
new file mode 100644
index 00000000..8299555f
--- /dev/null
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml
@@ -0,0 +1,30 @@
+# args of optimizer from searcher kwargs (neps.run) and from run_args (yaml)
+
+run_pipeline:
+  path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
+  name: run_pipeline
+pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_hyperband"
+
+max_evaluations_total: 5
+max_cost_total:
+
+monitoring:
+  overwrite_working_directory: true
+  post_run_summary: false
+  development_stage_id: None
+  task_id: None
+
+parallelization_setup:
+  max_evaluations_per_run: None
+  continue_until_max_evaluation_completed:
+
+searcher:
+  strategy: hyperband
+  name: my_hyperband
+  eta: 8
+  initial_design_type: max_budget
+  use_priors: false
+
+
+pre_load_hooks: None
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml
new file mode 100644
index 00000000..2cb448ec
--- /dev/null
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml
@@ -0,0 +1,41 @@
+run_pipeline:
+  path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
+  name: run_pipeline
+pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_priorband"
+
+max_evaluations_total: 5
+max_cost_total:
+
+monitoring:
+  overwrite_working_directory: true
+  post_run_summary: false
+  development_stage_id: None
+  task_id: None
+
+parallelization_setup:
+  max_evaluations_per_run: None
+  continue_until_max_evaluation_completed:
+
+searcher:
+  strategy: "priorband"
+  eta: 3
+  initial_design_type: max_budget
+  prior_confidence: medium
+  random_interleave_prob: 0.0
+  sample_default_first: true
+  sample_default_at_target: false
+  prior_weight_type: geometric
+  inc_sample_type: mutation
+  inc_mutation_rate: 0.5
+  inc_mutation_std: 0.25
+  inc_style: dynamic
+  model_based: true
+  modelling_type: joint
+  initial_design_size: 5
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation
+
+pre_load_hooks: None
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_select_bo.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_select_bo.yaml
new file mode 100644
index 00000000..6d1ca26e
--- /dev/null
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_select_bo.yaml
@@ -0,0 +1,23 @@
+run_pipeline:
+  path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
+  name: run_pipeline
+pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_bo"
+
+max_evaluations_total: 5
+max_cost_total:
+
+monitoring:
+  overwrite_working_directory: true
+  post_run_summary: false
+  development_stage_id: None
+  task_id: None
+
+parallelization_setup:
+  max_evaluations_per_run: None
+  continue_until_max_evaluation_completed:
+
+searcher:
+  strategy: "bayesian_optimization"
+
+pre_load_hooks: None
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml
index 17f375dc..b44fafe1 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml
@@ -4,7 +4,7 @@ run_pipeline:
 pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
 root_directory: "tests_tmpdir/test_run_args_by_neps_run/results1"
 
-max_evaluations_total: 1
+max_evaluations_total: 5
 max_cost_total:
 
 monitoring:
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_pipeline_space.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_pipeline_space.yaml
index 2a41902d..fab9349a 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_pipeline_space.yaml
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_pipeline_space.yaml
@@ -7,7 +7,7 @@ pipeline_space:
   name: "pipeline_space"
 root_directory: "tests/test_yaml_run_args/test_run_args_by_neps_run/results"
 
-max_evaluations_total: 1
+max_evaluations_total: 5
 max_cost_total:
 
 monitoring:
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py b/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py
index f44957eb..0bbebc6b 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py
@@ -22,9 +22,20 @@ def run_pipeline(learning_rate, epochs, optimizer, batch_size):
 )
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Run NEPS optimization with run_args.yml."
-    )
-    parser.add_argument("run_args", type=str, help="Path to the YAML configuration file.")
-    args = parser.parse_args()
-    neps.run(run_args=args.run_args)
+    if __name__ == "__main__":
+        parser = argparse.ArgumentParser(
+            description="Run NEPS optimization with run_args.yml.")
+        parser.add_argument("run_args", type=str,
+                            help="Path to the YAML configuration file.")
+        parser.add_argument("--kwargs_flag", action="store_true",
+                            help="Additional keyword arguments")
+        args = parser.parse_args()
+
+        hyperband_args_optimizer = {"random_interleave_prob": 0.9,
+                                    "sample_default_first": False,
+                                    "sample_default_at_target": False}
+
+        if args.kwargs_flag:
+            neps.run(run_args=args.run_args, **hyperband_args_optimizer)
+        else:
+            neps.run(run_args=args.run_args)
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml
new file mode 100644
index 00000000..c18d36c9
--- /dev/null
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml
@@ -0,0 +1,11 @@
+searcher_name: my_hyperband
+searcher_alg: hyperband
+searcher_selection: user-run_args-yaml
+neps_decision_tree: false
+searcher_args:
+  eta: 8
+  initial_design_type: max_budget
+  use_priors: false
+  random_interleave_prob: 0.9
+  sample_default_first: false
+  sample_default_at_target: false
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml
new file mode 100644
index 00000000..83ffa517
--- /dev/null
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml
@@ -0,0 +1,23 @@
+searcher_name: custom_priorband
+searcher_alg: priorband
+searcher_selection: user-run_args-yaml
+neps_decision_tree: false
+searcher_args:
+  eta: 3
+  initial_design_type: max_budget
+  prior_confidence: medium
+  random_interleave_prob: 0.0
+  sample_default_first: true
+  sample_default_at_target: false
+  prior_weight_type: geometric
+  inc_sample_type: mutation
+  inc_mutation_rate: 0.5
+  inc_mutation_std: 0.25
+  inc_style: dynamic
+  model_based: true
+  modelling_type: joint
+  initial_design_size: 5
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/select_bo_run_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/select_bo_run_args.yaml
new file mode 100644
index 00000000..af5259d0
--- /dev/null
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/select_bo_run_args.yaml
@@ -0,0 +1,13 @@
+searcher_name: custom_bayesian_optimization
+searcher_alg: bayesian_optimization
+searcher_selection: user-run_args-yaml
+neps_decision_tree: false
+searcher_args:
+  initial_design_size: 10
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation
+  random_interleave_prob: 0.0
+  disable_priors: true
+  sample_default_first: false
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml
new file mode 100644
index 00000000..4a44434d
--- /dev/null
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml
@@ -0,0 +1,11 @@
+epochs:
+  lower: 1
+  upper: 3
+  is_fidelity: true
+learning_rate:
+  lower: 1e-6
+  upper: 1e-1
+  log: False
+optimizer:
+  choices: ["a", "b", "c"]
+batch_size: 64
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml
new file mode 100644
index 00000000..ede66cc7
--- /dev/null
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml
@@ -0,0 +1,15 @@
+epochs:
+  lower: 1
+  upper: 3
+  is_fidelity: True
+learning_rate:
+  lower: 1e-6
+  upper: 1e-1
+  log: False
+  default: 1e-3
+  default_confidence: "low"
+optimizer:
+  choices: ["a", "b", "c"]
+  default: "b"
+  default_confidence: "high"
+batch_size: 64
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py b/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py
index 0179fa77..0d224fe7 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py
@@ -2,22 +2,65 @@
 import subprocess
 import os
 import sys
+import yaml
 BASE_PATH = "tests/test_yaml_run_args/test_run_args_by_neps_run/"
 
 
 @pytest.mark.neps_api
-@pytest.mark.parametrize("yaml_file", [
-    "config.yaml",
-    "loading_pipeline_space.yaml",
-    "loading_optimizer.yaml"
+@pytest.mark.parametrize("config", [
+    {"file_name": "config.yaml"},
+    {"file_name": "loading_pipeline_space.yaml"},
+    {"file_name": "loading_optimizer.yaml"},
+    {"file_name": "config_select_bo.yaml", "check_optimizer": True, "optimizer_path":
+        "select_bo_run_args.yaml",
+     "result_path": "tests_tmpdir/test_run_args_by_neps_run/optimizer_bo"
+                    "/.optimizer_info.yaml"},
+    {"file_name": "config_priorband_with_args.yaml", "check_optimizer": True,
+     "optimizer_path": "priorband_args_run_args.yaml",
+     "result_path": "tests_tmpdir/test_run_args_by_neps_run/optimizer_priorband"
+                    "/.optimizer_info.yaml"},
+    {"file_name": "config_hyperband_mixed_args.yaml", "check_optimizer": True,
+     "optimizer_path": "hyperband_searcher_kwargs_yaml_args.yaml",
+     "result_path": "tests_tmpdir/test_run_args_by_neps_run/optimizer_hyperband"
+                    "/.optimizer_info.yaml", "args": True}
 ])
-def test_run_with_yaml(yaml_file: str) -> None:
+def test_run_with_yaml(config: dict) -> None:
     """Test "neps.run" with various run_args.yaml settings to simulate loading options
     for variables."""
-    assert os.path.exists(BASE_PATH + yaml_file), f"{yaml_file} does not exist."
+    file_name = config["file_name"]
+    check_optimizer = config.pop("check_optimizer", False)
+    assert os.path.exists(os.path.join(BASE_PATH, file_name)), (f"{file_name} "
+                                                                f"does not exist.")
+
+    cmd = [sys.executable, os.path.join(BASE_PATH, 'neps_run.py'),
+           os.path.join(BASE_PATH, file_name)]
+    if "args" in config:
+        cmd.append('--kwargs_flag')
 
     try:
-        subprocess.check_call([sys.executable, BASE_PATH + 'neps_run.py', BASE_PATH +
-                               yaml_file])
+        subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
-        pytest.fail(f"NePS run failed for configuration: {yaml_file}")
+        pytest.fail(f"NePS run failed for configuration: {file_name}")
+
+    if check_optimizer:
+        optimizer_path = config.pop("optimizer_path")
+        result_path = config.pop("result_path")
+        compare_generated_yaml(result_path, optimizer_path)
+
+
+def compare_generated_yaml(result_path, optimizer_path):
+
+    assert os.path.exists(result_path), \
+        "Generated YAML file does not exist."
+
+    assert os.path.exists(BASE_PATH + "optimizer_yamls/" + optimizer_path), \
+        "Solution YAML file does not exist."
+
+    with open(result_path, 'r') as gen_file:
+        generated_content = yaml.safe_load(gen_file)
+
+    with open(BASE_PATH + "optimizer_yamls/" + optimizer_path, 'r') as ref_file:
+        reference_content = yaml.safe_load(ref_file)
+
+    assert generated_content == reference_content, \
+        "The generated YAML does not match the reference YAML"

From 6ea6186e7838d88a81f90a87229240c6a58e4023 Mon Sep 17 00:00:00 2001
From: Daniel <63580393+danrgll@users.noreply.github.com>
Date: Tue, 2 Jul 2024 22:13:24 +0200
Subject: [PATCH 5/7] minor change in test

---
 tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py | 3 ++-
 .../optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml   | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py b/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py
index 0bbebc6b..f823e75e 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py
@@ -33,7 +33,8 @@ def run_pipeline(learning_rate, epochs, optimizer, batch_size):
 
         hyperband_args_optimizer = {"random_interleave_prob": 0.9,
                                     "sample_default_first": False,
-                                    "sample_default_at_target": False}
+                                    "sample_default_at_target": False,
+                                    "eta": 7}
 
         if args.kwargs_flag:
             neps.run(run_args=args.run_args, **hyperband_args_optimizer)
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml
index c18d36c9..4300ca79 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml
@@ -3,7 +3,7 @@ searcher_alg: hyperband
 searcher_selection: user-run_args-yaml
 neps_decision_tree: false
 searcher_args:
-  eta: 8
+  eta: 7
   initial_design_type: max_budget
   use_priors: false
   random_interleave_prob: 0.9

From 11cde477ca55bbbbaee7f1ddeda862d244b0e58c Mon Sep 17 00:00:00 2001
From: Daniel <63580393+danrgll@users.noreply.github.com>
Date: Tue, 2 Jul 2024 22:26:00 +0200
Subject: [PATCH 6/7] minor change in test

---
 .../config_hyperband_mixed_args.yaml                          | 1 -
 .../test_run_args_by_neps_run/config_priorband_with_args.yaml | 4 +---
 .../optimizer_yamls/priorband_args_run_args.yaml              | 2 +-
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml
index 8299555f..597cf20a 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml
@@ -24,7 +24,6 @@ searcher:
   name: my_hyperband
   eta: 8
   initial_design_type: max_budget
-  use_priors: false
 
 
 pre_load_hooks: None
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml
index 2cb448ec..fca3a6d6 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml
@@ -19,15 +19,13 @@ parallelization_setup:
 
 searcher:
   strategy: "priorband"
-  eta: 3
   initial_design_type: max_budget
   prior_confidence: medium
-  random_interleave_prob: 0.0
   sample_default_first: true
   sample_default_at_target: false
   prior_weight_type: geometric
   inc_sample_type: mutation
-  inc_mutation_rate: 0.5
+  inc_mutation_rate: 0.2
   inc_mutation_std: 0.25
   inc_style: dynamic
   model_based: true
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml
index 83ffa517..0f326330 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml
@@ -11,7 +11,7 @@ searcher_args:
   sample_default_at_target: false
   prior_weight_type: geometric
   inc_sample_type: mutation
-  inc_mutation_rate: 0.5
+  inc_mutation_rate: 0.2
   inc_mutation_std: 0.25
   inc_style: dynamic
   model_based: true

From f1b7bd82568064fd0fc246e7ef14b00a8f439e4a Mon Sep 17 00:00:00 2001
From: Daniel <63580393+danrgll@users.noreply.github.com>
Date: Wed, 3 Jul 2024 10:55:03 +0200
Subject: [PATCH 7/7] minor code fix + comment

---
 .../test_run_args_by_neps_run/neps_run.py     | 31 +++++++++----------
 .../test_neps_run.py                          |  2 +-
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py b/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py
index f823e75e..f2ee0c9c 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py
@@ -22,21 +22,20 @@ def run_pipeline(learning_rate, epochs, optimizer, batch_size):
 )
 
 if __name__ == "__main__":
-    if __name__ == "__main__":
-        parser = argparse.ArgumentParser(
-            description="Run NEPS optimization with run_args.yml.")
-        parser.add_argument("run_args", type=str,
-                            help="Path to the YAML configuration file.")
-        parser.add_argument("--kwargs_flag", action="store_true",
-                            help="Additional keyword arguments")
-        args = parser.parse_args()
+    parser = argparse.ArgumentParser(
+        description="Run NEPS optimization with run_args.yaml.")
+    parser.add_argument("run_args", type=str,
+                        help="Path to the YAML configuration file.")
+    parser.add_argument("--kwargs_flag", action="store_true",
+                        help="flag for adding kwargs")
+    args = parser.parse_args()
 
-        hyperband_args_optimizer = {"random_interleave_prob": 0.9,
-                                    "sample_default_first": False,
-                                    "sample_default_at_target": False,
-                                    "eta": 7}
+    hyperband_args_optimizer = {"random_interleave_prob": 0.9,
+                                "sample_default_first": False,
+                                "sample_default_at_target": False,
+                                "eta": 7}
 
-        if args.kwargs_flag:
-            neps.run(run_args=args.run_args, **hyperband_args_optimizer)
-        else:
-            neps.run(run_args=args.run_args)
+    if args.kwargs_flag:
+        neps.run(run_args=args.run_args, **hyperband_args_optimizer)
+    else:
+        neps.run(run_args=args.run_args)
diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py b/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py
index 0d224fe7..30d6e178 100644
--- a/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py
+++ b/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py
@@ -49,7 +49,7 @@ def test_run_with_yaml(config: dict) -> None:
 
 
 def compare_generated_yaml(result_path, optimizer_path):
-
+    """compare generated optimizer settings and solution settings"""
     assert os.path.exists(result_path), \
         "Generated YAML file does not exist."