Refactor declarative usage (#112)

neps/api.py

@@ -338,18 +338,27 @@ def _run_args(
         message = f"The pipeline_space has invalid type: {type(pipeline_space)}"
         raise TypeError(message) from e
 
+    # Load the information of the optimizer
     if isinstance(searcher, (str, Path)) and searcher not in \
         SearcherConfigs.get_searchers() and searcher != "default":
-        # The users has their own custom searcher.
+        # The users have their own custom searcher provided via yaml.
         logging.info("Preparing to run user created searcher")
 
-        config, searcher = get_searcher_data(searcher, loading_custom_searcher=True)
+        searcher_config, file_name = get_searcher_data(searcher,
+                                                      loading_custom_searcher=True)
+        # name defined via key or the filename of the yaml
+        searcher_name = searcher_config.pop("name", file_name)
         searcher_info["searcher_selection"] = "user-yaml"
         searcher_info["neps_decision_tree"] = False
     elif isinstance(searcher, dict):
-        config = searcher
-        searcher = config.pop("name", "unnamed-custom-searcher")
-        searcher_info["searcher_selection"] = "user-yaml"
+        custom_config = searcher
+        default_config, searcher_name = get_searcher_data(searcher["strategy"])
+        searcher_config = {**default_config, **custom_config}
+        if "name" not in searcher_config:
+            searcher_name = "custom_" + searcher_name
+        else:
+            searcher_name = searcher_config.pop("name")
+        searcher_info["searcher_selection"] = "user-run_args-yaml"
         searcher_info["neps_decision_tree"] = False
     else:
         if searcher in ["default", None]:
@@ -368,29 +377,31 @@ def _run_args(
             searcher_info["neps_decision_tree"] = False
             searcher_info["searcher_selection"] = "neps-default"
         # Fetching the searcher data, throws an error when the searcher is not found
-        config, searcher = get_searcher_data(searcher)
+        searcher_config, searcher_name = get_searcher_data(searcher)
 
     # Check for deprecated 'algorithm' argument
-    if "algorithm" in config:
+    if "algorithm" in searcher_config:
         warnings.warn(
-            "The 'algorithm' argument is deprecated and will be removed in future versions. Please use 'strategy' instead.",
+            "The 'algorithm' argument is deprecated and will be removed in "
+            "future versions. Please use 'strategy' instead.",
             DeprecationWarning
         )
         # Map the old 'algorithm' argument to 'strategy'
-        config['strategy'] = config.pop("algorithm")
+        searcher_config['strategy'] = searcher_config.pop("algorithm")
 
-    if "strategy" in config:
-        searcher_alg = config.pop("strategy")
+    if "strategy" in searcher_config:
+        searcher_alg = searcher_config.pop("strategy")
     else:
-        raise KeyError(f"Missing key strategy in searcher config:{config}")
-    searcher_config = config
+        raise KeyError(f"Missing key strategy in searcher config:{searcher_config}")
+
+
+    logger.info(f"Running {searcher_name} as the searcher")
+    logger.info(f"Strategy: {searcher_alg}")
 
-    logger.info(f"Running {searcher} as the searcher")
-    logger.debug(f"Strategy: {searcher_alg}")
 
     # Used to create the yaml holding information about the searcher.
     # Also important for testing and debugging the api.
-    searcher_info["searcher_name"] = searcher
+    searcher_info["searcher_name"] = searcher_name
     searcher_info["searcher_alg"] = searcher_alg
 
     # Updating searcher arguments from searcher_kwargs

neps/optimizers/default_searchers/pibo.yaml

@@ -1,4 +1,4 @@
-strategy: bayesian_optimization
+strategy: pibo
 # Arguments that can be modified by the user
 initial_design_size: 10
 surrogate_model: gp  # or {"gp_hierarchy"}

tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml

@@ -1,5 +1,5 @@
 searcher_name: pibo
-searcher_alg: bayesian_optimization
+searcher_alg: pibo
 searcher_selection: neps-default
 neps_decision_tree: true
 searcher_args:

tests/test_yaml_run_args/test_declarative_usage_docs/customizing_neps_optimizer.yaml

@@ -15,7 +15,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   batch_size: 64
 
-root_directory: path/to/results/custominizing_neps_optimizer       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/custominizing_neps_optimizer"
 max_evaluations_total: 20               # Budget
 searcher:
   strategy: bayesian_optimization

tests/test_yaml_run_args/test_declarative_usage_docs/defining_hooks.yaml

@@ -16,7 +16,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   batch_size: 64
 
-root_directory: path/to/results/hooks       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/hooks"
 max_evaluations_total: 20               # Budget
 
 pre_load_hooks:

tests/test_yaml_run_args/test_declarative_usage_docs/full_configuration_template.yaml

@@ -16,7 +16,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   batch_size: 64
 
-root_directory: path/to/results/full_config      # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/full_config"
 max_evaluations_total: 20             # Budget
 max_cost_total:
 

tests/test_yaml_run_args/test_declarative_usage_docs/loading_own_optimizer.yaml

@@ -11,7 +11,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   epochs: 50
 
-root_directory: path/to/results       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/loading_own_optimizer"
 max_evaluations_total: 20               # Budget
 searcher:
   path: "neps/optimizers/bayesian_optimization/optimizer.py"

tests/test_yaml_run_args/test_declarative_usage_docs/loading_pipeline_space_dict.yaml

@@ -7,5 +7,5 @@ pipeline_space:
   path: tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.py
   name: pipeline_space                  # Name of the dict instance
 
-root_directory: path/to/results_loading_pipeline_space       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/results_loading_pipeline_space"
 max_evaluations_total: 20               # Budget

tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_optimizer.yaml

@@ -12,7 +12,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   epochs: 50
 
-root_directory: path/to/results_outsourcing_optimizer      # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/outsourcing_optimizer"
 max_evaluations_total: 20               # Budget
 
 searcher: tests/test_yaml_run_args/test_declarative_usage_docs/set_up_optimizer.yaml

tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_pipeline_space.yaml

@@ -5,6 +5,6 @@ run_pipeline:
 
 pipeline_space: tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.yaml
 
-root_directory: path/to/results_outsourcing_pipeline_space       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/outsourcing_pipeline_space"
 max_evaluations_total: 20               # Budget
 

tests/test_yaml_run_args/test_declarative_usage_docs/simple_example.yaml

@@ -12,7 +12,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   batch_size: 64
 
-root_directory: path/to/results/simple_example       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/simple_example"
 max_evaluations_total: 20             # Budget
 
 

tests/test_yaml_run_args/test_declarative_usage_docs/simple_example_including_run_pipeline.yaml

@@ -16,7 +16,7 @@ pipeline_space:
     choices: [adam, sgd, adamw]
   batch_size: 64
 
-root_directory: path/to/results/simple_example_including_run_pipeline       # Directory for result storage
+root_directory: "tests_tmpdir/test_declarative_usage_docs/simple_example_including_run_pipeline"
 max_evaluations_total: 20             # Budget
 
 overwrite_working_directory: True

tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml

@@ -2,9 +2,9 @@ run_pipeline:
   path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
   name: run_pipeline
 pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
-root_directory: "tests/test_yaml_run_args/test_run_args_by_neps_run/results"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/results2"
 
-max_evaluations_total: 1
+max_evaluations_total: 5
 max_cost_total:
 
 monitoring:

tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml

@@ -0,0 +1,29 @@
+# args of optimizer from searcher kwargs (neps.run) and from run_args (yaml)
+
+run_pipeline:
+  path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
+  name: run_pipeline
+pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_hyperband"
+
+max_evaluations_total: 5
+max_cost_total:
+
+monitoring:
+  overwrite_working_directory: true
+  post_run_summary: false
+  development_stage_id: None
+  task_id: None
+
+parallelization_setup:
+  max_evaluations_per_run: None
+  continue_until_max_evaluation_completed:
+
+searcher:
+  strategy: hyperband
+  name: my_hyperband
+  eta: 8
+  initial_design_type: max_budget
+
+
+pre_load_hooks: None

tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml

@@ -0,0 +1,39 @@
+run_pipeline:
+  path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
+  name: run_pipeline
+pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_priorband"
+
+max_evaluations_total: 5
+max_cost_total:
+
+monitoring:
+  overwrite_working_directory: true
+  post_run_summary: false
+  development_stage_id: None
+  task_id: None
+
+parallelization_setup:
+  max_evaluations_per_run: None
+  continue_until_max_evaluation_completed:
+
+searcher:
+  strategy: "priorband"
+  initial_design_type: max_budget
+  prior_confidence: medium
+  sample_default_first: true
+  sample_default_at_target: false
+  prior_weight_type: geometric
+  inc_sample_type: mutation
+  inc_mutation_rate: 0.2
+  inc_mutation_std: 0.25
+  inc_style: dynamic
+  model_based: true
+  modelling_type: joint
+  initial_design_size: 5
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation
+
+pre_load_hooks: None

tests/test_yaml_run_args/test_run_args_by_neps_run/config_select_bo.yaml

@@ -0,0 +1,23 @@
+run_pipeline:
+  path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
+  name: run_pipeline
+pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_bo"
+
+max_evaluations_total: 5
+max_cost_total:
+
+monitoring:
+  overwrite_working_directory: true
+  post_run_summary: false
+  development_stage_id: None
+  task_id: None
+
+parallelization_setup:
+  max_evaluations_per_run: None
+  continue_until_max_evaluation_completed:
+
+searcher:
+  strategy: "bayesian_optimization"
+
+pre_load_hooks: None

tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml

@@ -2,9 +2,9 @@ run_pipeline:
   path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
   name: "run_pipeline"
 pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
-root_directory: "tests/test_yaml_run_args/test_run_args_by_neps_run/results"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/results1"
 
-max_evaluations_total: 1
+max_evaluations_total: 5
 max_cost_total:
 
 monitoring:

tests/test_yaml_run_args/test_run_args_by_neps_run/loading_pipeline_space.yaml

@@ -7,7 +7,7 @@ pipeline_space:
   name: "pipeline_space"
 root_directory: "tests/test_yaml_run_args/test_run_args_by_neps_run/results"
 
-max_evaluations_total: 1
+max_evaluations_total: 5
 max_cost_total:
 
 monitoring:

tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py

@@ -23,8 +23,19 @@ def run_pipeline(learning_rate, epochs, optimizer, batch_size):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        description="Run NEPS optimization with run_args.yml."
-    )
-    parser.add_argument("run_args", type=str, help="Path to the YAML configuration file.")
+        description="Run NEPS optimization with run_args.yaml.")
+    parser.add_argument("run_args", type=str,
+                        help="Path to the YAML configuration file.")
+    parser.add_argument("--kwargs_flag", action="store_true",
+                        help="flag for adding kwargs")
     args = parser.parse_args()
-    neps.run(run_args=args.run_args)
+
+    hyperband_args_optimizer = {"random_interleave_prob": 0.9,
+                                "sample_default_first": False,
+                                "sample_default_at_target": False,
+                                "eta": 7}
+
+    if args.kwargs_flag:
+        neps.run(run_args=args.run_args, **hyperband_args_optimizer)
+    else:
+        neps.run(run_args=args.run_args)

tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml

@@ -0,0 +1,11 @@
+searcher_name: my_hyperband
+searcher_alg: hyperband
+searcher_selection: user-run_args-yaml
+neps_decision_tree: false
+searcher_args:
+  eta: 7
+  initial_design_type: max_budget
+  use_priors: false
+  random_interleave_prob: 0.9
+  sample_default_first: false
+  sample_default_at_target: false

tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml

@@ -0,0 +1,23 @@
+searcher_name: custom_priorband
+searcher_alg: priorband
+searcher_selection: user-run_args-yaml
+neps_decision_tree: false
+searcher_args:
+  eta: 3
+  initial_design_type: max_budget
+  prior_confidence: medium
+  random_interleave_prob: 0.0
+  sample_default_first: true
+  sample_default_at_target: false
+  prior_weight_type: geometric
+  inc_sample_type: mutation
+  inc_mutation_rate: 0.2
+  inc_mutation_std: 0.25
+  inc_style: dynamic
+  model_based: true
+  modelling_type: joint
+  initial_design_size: 5
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation

tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/select_bo_run_args.yaml

@@ -0,0 +1,13 @@
+searcher_name: custom_bayesian_optimization
+searcher_alg: bayesian_optimization
+searcher_selection: user-run_args-yaml
+neps_decision_tree: false
+searcher_args:
+  initial_design_size: 10
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation
+  random_interleave_prob: 0.0
+  disable_priors: true
+  sample_default_first: false

tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml

@@ -0,0 +1,11 @@
+epochs:
+  lower: 1
+  upper: 3
+  is_fidelity: true
+learning_rate:
+  lower: 1e-6
+  upper: 1e-1
+  log: False
+optimizer:
+  choices: ["a", "b", "c"]
+batch_size: 64

tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml

@@ -0,0 +1,15 @@
+epochs:
+  lower: 1
+  upper: 3
+  is_fidelity: True
+learning_rate:
+  lower: 1e-6
+  upper: 1e-1
+  log: False
+  default: 1e-3
+  default_confidence: "low"
+optimizer:
+  choices: ["a", "b", "c"]
+  default: "b"
+  default_confidence: "high"
+batch_size: 64