diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 95617323..54b648f5 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -21,9 +21,9 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -41,4 +41,5 @@ jobs: - name: Run pytest timeout-minutes: 15 - run: poetry run pytest -m "all_examples or metahyper or summary_csv" + run: poetry run pytest -m "all_examples or metahyper or neps_api or summary_csv" + diff --git a/CITATION.cff b/CITATION.cff index 8dea2157..fde949e5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -13,10 +13,12 @@ authors: given-names: Samir - family-names: Abou Chakra given-names: Tarek - - family-names: Hvarfner - given-names: Carl + - family-names: Rogalla + given-names: Daniel - family-names: Bergman given-names: Eddie + - family-names: Hvarfner + given-names: Carl - family-names: Binxin given-names: Ru - family-names: Kober @@ -26,6 +28,6 @@ authors: - family-names: Hutter given-names: Frank title: "Neural Pipeline Search (NePS)" -version: 0.10.0 +version: 0.11.1 date-released: 2023-10-25 url: "https://github.com/automl/neps" diff --git a/README.md b/README.md index 43687ca4..a83ac11d 100644 --- a/README.md +++ b/README.md @@ -5,23 +5,33 @@ [![License](https://img.shields.io/pypi/l/neural-pipeline-search?color=informational)](LICENSE) [![Tests](https://github.com/automl/neps/actions/workflows/tests.yaml/badge.svg)](https://github.com/automl/neps/actions) -NePS helps deep learning experts to optimize the hyperparameters and/or architecture of their deep learning pipeline with: +Welcome to NePS, a powerful and flexible Python library for hyperparameter optimization (HPO) and neural architecture search (NAS) with its primary goal: enable HPO adoption in practice for deep learners! -- Hyperparameter Optimization (HPO) ([example](neps_examples/basic_usage/hyperparameters.py)) -- Neural Architecture Search (NAS) ([example](neps_examples/basic_usage/architecture.py), [paper](https://openreview.net/forum?id=Ok58hMNXIQ)) -- Joint Architecture and Hyperparameter Search (JAHS) ([example](neps_examples/basic_usage/architecture_and_hyperparameters.py), [paper](https://openreview.net/forum?id=_HLcjaVlqJ)) +NePS houses recently published and some more well-established algorithms that are all capable of being run massively parallel on any distributed setup, with tools to analyze runs, restart runs, etc. -For efficiency and convenience NePS allows you to +Take a look at our [documentation](https://automl.github.io/neps/latest/) and continue following through current README for instructions on how to use NePS! -- Add your intuition as priors for the search ([example HPO](neps_examples/efficiency/expert_priors_for_hyperparameters.py), [example JAHS](neps_examples/experimental/expert_priors_for_architecture_and_hyperparameters.py), [paper](https://openreview.net/forum?id=MMAeCXIa89)) -- Utilize low fidelity (e.g., low epoch) evaluations to focus on promising configurations ([example](neps_examples/efficiency/multi_fidelity.py), [paper](https://openreview.net/forum?id=ds21dwfBBH)) -- Trivially parallelize across machines ([example](neps_examples/efficiency/parallelization.md), [documentation](https://automl.github.io/neps/latest/parallelization/)) -Or [all of the above](neps_examples/efficiency/multi_fidelity_and_expert_priors.py) for maximum efficiency! +## Key Features -### Note +In addition to the common features offered by traditional HPO and NAS libraries, NePS stands out with the following key features: -As indicated with the `v0.x.x` version number, NePS is early stage code and APIs might change in the future. +1. [**Hyperparameter Optimization (HPO) With Prior Knowledge:**](neps_examples/template/priorband_template.py) + - NePS excels in efficiently tuning hyperparameters using algorithms that enable users to make use of their prior knowledge within the search space. This is leveraged by the insights presented in: + - [PriorBand: Practical Hyperparameter Optimization in the Age of Deep Learning](https://arxiv.org/abs/2306.12370) + - [πBO: Augmenting Acquisition Functions with User Beliefs for Bayesian Optimization](https://arxiv.org/abs/2204.11051) + +2. [**Neural Architecture Search (NAS) With Context-free Grammar Search Spaces:**](neps_examples/basic_usage/architecture.py) + - NePS is equipped to handle context-free grammar search spaces, providing advanced capabilities for designing and optimizing architectures. this is leveraged by the insights presented in: + - [Construction of Hierarchical Neural Architecture Search Spaces based on Context-free Grammars](https://arxiv.org/abs/2211.01842) + +3. [**Easy Parallelization and Resumption of Runs:**](docs/parallelization.md) + - NePS simplifies the process of parallelizing optimization tasks both on individual computers and in distributed + computing environments. It also allows users to conveniently resume these optimization tasks after completion to + ensure a seamless and efficient workflow for long-running experiments. + +4. [**Seamless User Code Integration:**](neps_examples/template/) + - NePS's modular design ensures flexibility and extensibility. Integrate NePS effortlessly into existing machine learning workflows. ## Getting Started @@ -33,13 +43,16 @@ Using pip: pip install neural-pipeline-search ``` +> Note: As indicated with the `v0.x.x` version number, NePS is early stage code and APIs might change in the future. + ### 2. Basic Usage Using `neps` always follows the same pattern: - 1. Define a `run_pipeline` function that evaluates architectures/hyperparameters for your problem - 1. Define a search space `pipeline_space` of architectures/hyperparameters - 1. Call `neps.run` to optimize `run_pipeline` over `pipeline_space` +1. Define a `run_pipeline` function capable of evaluating different architectural and/or hyperparameter configurations + for your problem. +2. Define a search space named `pipeline_space` of those Parameters e.g. via a dictionary +3. Call `neps.run` to optimize `run_pipeline` over `pipeline_space` In code, the usage pattern can look like this: @@ -47,53 +60,88 @@ In code, the usage pattern can look like this: import neps import logging -# 1. Define a function that accepts hyperparameters and returns the validation error -def run_pipeline(hyperparameter_a: float, hyperparameter_b: int, architecture_parameter: str): - # create your model + +# 1. Define a function that accepts hyperparameters and computes the validation error +def run_pipeline( + hyperparameter_a: float, hyperparameter_b: int, architecture_parameter: str +) -> dict: + # Create your model model = MyModel(architecture_parameter) - # train and evaluate the model with your training pipeline - validation_error = train_and_eval_model(model, hyperparameter_a, hyperparameter_b) - return validation_error + # Train and evaluate the model with your training pipeline + validation_error, training_error = train_and_eval( + model, hyperparameter_a, hyperparameter_b + ) + + return { # dict or float(validation error) + "loss": validation_error, + "info_dict": { + "training_error": training_error + # + Other metrics + }, + } -# 2. Define a search space of hyperparameters; use the same names as in run_pipeline + +# 2. Define a search space of parameters; use the same names for the parameters as in run_pipeline pipeline_space = dict( hyperparameter_b=neps.IntegerParameter( - lower=1, - upper=100, - is_fidelity=True), # Mark 'is_fidelity' as true for a multi-fidelity approach. + lower=1, upper=42, is_fidelity=True + ), # Mark 'is_fidelity' as true for a multi-fidelity approach. hyperparameter_a=neps.FloatParameter( - lower=0.0, - upper=1.0, - log=True), # If True, the search space is sampled in log space. - architecture_parameter=neps.CategoricalParameter(["option_a", "option_b", "option_c"]), + lower=0.001, upper=0.1, log=True + ), # If True, the search space is sampled in log space. + architecture_parameter=neps.CategoricalParameter( + ["option_a", "option_b", "option_c"] + ), ) -if __name__=="__main__": +if __name__ == "__main__": # 3. Run the NePS optimization logging.basicConfig(level=logging.INFO) neps.run( run_pipeline=run_pipeline, pipeline_space=pipeline_space, - root_directory="path/to/save/results", # Replace with the actual path. + root_directory="path/to/save/results", # Replace with the actual path. max_evaluations_total=100, - searcher="hyperband" # Optional specifies the search strategy, + searcher="hyperband" # Optional specifies the search strategy, # otherwise NePs decides based on your data. ) ``` +## Examples + +Discover how NePS works through these practical examples: +* **[Pipeline Space via YAML](neps_examples/basic_usage/defining_search_space)**: Explore how to define the `pipeline_space` using a + YAML file instead of a dictionary. + +* **[Hyperparameter Optimization (HPO)](neps_examples/basic_usage/hyperparameters.py)**: Learn the essentials of hyperparameter optimization with NePS. + +* **[Architecture Search with Primitives](neps_examples/basic_usage/architecture.py)**: Dive into architecture search using primitives in NePS. + +* **[Multi-Fidelity Optimization](neps_examples/efficiency/multi_fidelity.py)**: Understand how to leverage multi-fidelity optimization for efficient model tuning. + +* **[Utilizing Expert Priors for Hyperparameters](neps_examples/efficiency/expert_priors_for_hyperparameters.py)**: Learn how to incorporate expert priors for more efficient hyperparameter selection. + +* **[Additional NePS Examples](neps_examples/)**: Explore more examples, including various use cases and advanced configurations in NePS. + ## Documentation -For more details and features please have a look at our [documentation](https://automl.github.io/neps/latest/) and [examples](neps_examples) +For more details and features please have a look at our [documentation](https://automl.github.io/neps/latest/) ## Analysing runs See our [documentation on analysing runs](https://automl.github.io/neps/latest/analyse). -## Alternatives - -NePS does not cover your use-case? Have a look at [some alternatives](https://automl.github.io/neps/latest/alternatives). - ## Contributing Please see the [documentation for contributors](https://automl.github.io/neps/latest/contributing/). + +## Citations + +Please consider citing us if you use our tool! + +Refer to our [documentation on citations](https://automl.github.io/neps/latest/citations/). + +## Alternatives + +NePS does not cover your use-case? Have a look at [some alternatives](https://automl.github.io/neps/latest/alternatives). diff --git a/docs/README.md b/docs/README.md index b58e43d6..28f2f0dd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,9 +1,32 @@ -# Introduction and Installation +# Neural Pipeline Search (NePS) -## Installation +[![PyPI version](https://img.shields.io/pypi/v/neural-pipeline-search?color=informational)](https://pypi.org/project/neural-pipeline-search/) +[![Python versions](https://img.shields.io/pypi/pyversions/neural-pipeline-search)](https://pypi.org/project/neural-pipeline-search/) +[![License](https://img.shields.io/pypi/l/neural-pipeline-search?color=informational)](LICENSE) +[![Tests](https://github.com/automl/neps/actions/workflows/tests.yaml/badge.svg)](https://github.com/automl/neps/actions) -Using pip +Welcome to NePS, a powerful and flexible Python library for hyperparameter optimization (HPO) and neural architecture search (NAS) with its primary goal: enable HPO adoption in practice for deep learners! -```bash -pip install neural-pipeline-search -``` +NePS houses recently published and some more well-established algorithms that are all capable of being run massively parallel on any distributed setup, with tools to analyze runs, restart runs, etc. + + +## Key Features + +In addition to the common features offered by traditional HPO and NAS libraries, NePS stands out with the following key features: + +1. [**Hyperparameter Optimization (HPO) With Prior Knowledge:**](https://github.com/automl/neps/tree/master/neps_examples/template/priorband_template.py) + - NePS excels in efficiently tuning hyperparameters using algorithms that enable users to make use of their prior knowledge within the search space. This is leveraged by the insights presented in: + - [PriorBand: Practical Hyperparameter Optimization in the Age of Deep Learning](https://arxiv.org/abs/2306.12370) + - [πBO: Augmenting Acquisition Functions with User Beliefs for Bayesian Optimization](https://arxiv.org/abs/2204.11051) + +2. [**Neural Architecture Search (NAS) With Context-free Grammar Search Spaces:**](https://github.com/automl/neps/tree/master/neps_examples/basic_usage/architecture.py) + - NePS is equipped to handle context-free grammar search spaces, providing advanced capabilities for designing and optimizing architectures. this is leveraged by the insights presented in: + - [Construction of Hierarchical Neural Architecture Search Spaces based on Context-free Grammars](https://arxiv.org/abs/2211.01842) + +3. [**Easy Parallelization and Resumption of Runs:**](https://automl.github.io/neps/latest/parallelization) + - NePS simplifies the process of parallelizing optimization tasks both on individual computers and in distributed + computing environments. It also allows users to conveniently resume these optimization tasks after completion to + ensure a seamless and efficient workflow for long-running experiments. + +4. [**Seamless User Code Integration:**](https://github.com/automl/neps/tree/master/neps_examples/template/) + - NePS's modular design ensures flexibility and extensibility. Integrate NePS effortlessly into existing machine learning workflows. diff --git a/docs/analyse.md b/docs/analyse.md index 40d4af7c..cd78a14d 100644 --- a/docs/analyse.md +++ b/docs/analyse.md @@ -17,6 +17,154 @@ ROOT_DIRECTORY ├── best_loss_trajectory.txt └── best_loss_with_config_trajectory.txt ``` +## Summary CSV + +The argument `post_run_summary` in `neps.run` allows for the automatic generation of CSV files after a run is complete. The new root directory after utilizing this argument will look like the following: + +``` +ROOT_DIRECTORY +├── results +│ └── config_1 +│ ├── config.yaml +│ ├── metadata.yaml +│ └── result.yaml +├── summary_csv +│ ├── config_data.csv +│ └── run_status.csv +├── all_losses_and_configs.txt +├── best_loss_trajectory.txt +└── best_loss_with_config_trajectory.txt +``` + +- *`config_data.csv`*: Contains all configuration details in CSV format, ordered by ascending `loss`. Details include configuration hyperparameters, any returned result from the `run_pipeline` function, and metadata information. + +- *`run_status.csv`*: Provides general run details, such as the number of sampled configs, best configs, number of failed configs, best loss, etc. + +## TensorBoard Integration + +### Introduction + +[TensorBoard](https://www.tensorflow.org/tensorboard) serves as a valuable tool for visualizing machine learning experiments, offering the ability to observe losses and metrics throughout the model training process. In NePS, we use this powerful tool to show metrics of configurations during training in addition to comparisons to different hyperparameters used in the search for better diagnosis of the model. + +### The Logging Function + +The `tblogger.log` function is invoked within the model's training loop to facilitate logging of key metrics. + +!!! tip + + The logger function is primarily designed for implementation within the `run_pipeline` function during the training of the neural network. + +- **Signature:** +```python +tblogger.log( + loss: float, + current_epoch: int, + write_config_scalar: bool = False, + write_config_hparam: bool = True, + write_summary_incumbent: bool = False, + extra_data: dict | None = None +) +``` + +- **Parameters:** + - `loss` (float): The loss value to be logged. + - `current_epoch` (int): The current epoch or iteration number. + - `write_config_scalar` (bool, optional): Set to `True` for a live loss trajectory for each configuration. + - `write_config_hparam` (bool, optional): Set to `True` for live parallel coordinate, scatter plot matrix, and table view. + - `write_summary_incumbent` (bool, optional): Set to `True` for a live incumbent trajectory. + - `extra_data` (dict, optional): Additional data to be logged, provided as a dictionary. + +### Extra Custom Logging + +NePS provides dedicated functions for customized logging using the `extra_data` argument. + +!!! note "Custom Logging Instructions" + + Name the dictionary keys as the names of the values you want to log and pass one of the following functions as the values for a successful logging process. + +#### 1- Extra Scalar Logging + +Logs new scalar data during training. Uses `current_epoch` from the log function as its `global_step`. + +- **Signature:** +```python +tblogger.scalar_logging(value: float) +``` +- **Parameters:** + - `value` (float): Any scalar value to be logged at the current epoch of `tblogger.log` function. + +#### 2- Extra Image Logging + +Logs images during training. Images can be resized, randomly selected, and a specified number can be logged at specified intervals. Uses `current_epoch` from the log function as its `global_step`. + +- **Signature:** +```python +tblogger.image_logging( + image: torch.Tensor, + counter: int = 1, + resize_images: list[None | int] | None = None, + random_images: bool = True, + num_images: int = 20, + seed: int | np.random.RandomState | None = None, +) +``` + +- **Parameters:** + - `image` (torch.Tensor): Image tensor to be logged. + - `counter` (int): Log images every counter epochs (i.e., when current_epoch % counter equals 0). + - `resize_images` (list of int, optional): List of integers for image sizes after resizing (default: [32, 32]). + - `random_images` (bool, optional): Images are randomly selected if True (default: True). + - `num_images` (int, optional): Number of images to log (default: 20). + - `seed` (int or np.random.RandomState or None, optional): Seed value or RandomState instance to control randomness and reproducibility (default: None). + +### Logging Example + +For illustration purposes, we have employed a straightforward example involving the tuning of hyperparameters for a model utilized in the classification of the MNIST dataset provided by [torchvision](https://pytorch.org/vision/main/generated/torchvision.datasets.MNIST.html). + +You can find this example [here](https://github.com/automl/neps/blob/master/neps_examples/convenience/neps_tblogger_tutorial.py) + +!!! info "Important" + We have optimized the example for computational efficiency. If you wish to replicate the exact results showcased in the following section, we recommend the following modifications: + + 1- Increase maximum epochs from 2 to 10 + + 2- Set the `write_summary_incumbent` argument to `True` + + 3- Change the searcher from `random_search` to `bayesian_optimization` + + 4- Increase the maximum evaluations before disabling `tblogger` from 2 to 14 + + 5- Increase the maximum evaluations after disabling `tblogger` from 3 to 15 + +### Visualization Results + +The following command will open a local host for TensorBoard visualizations, allowing you to view them either in real-time or after the run is complete. + +```bash +tensorboard --logdir path/to/root_directory +``` + +This image shows visualizations related to scalar values logged during training. Scalars typically include metrics such as loss, incumbent trajectory, a summary of losses for all configurations, and any additional data provided via the `extra_data` argument in the `tblogger.log` function. + +![scalar_loggings](doc_images/tensorboard/tblogger_scalar.jpg) + +This image represents visualizations related to logged images during training. It could include snapshots of input data, model predictions, or any other image-related information. In our case, we use images to depict instances of incorrect predictions made by the model. + +![image_loggings](doc_images/tensorboard/tblogger_image.jpg) + +The following images showcase visualizations related to hyperparameter logging in TensorBoard. These plots include three different views, providing insights into the relationship between different hyperparameters and their impact on the model. + +In the table view, you can explore hyperparameter configurations across five different trials. The table displays various hyperparameter values alongside corresponding evaluation metrics. + +![hparam_loggings1](doc_images/tensorboard/tblogger_hparam1.jpg) + +The parallel coordinate plot offers a holistic perspective on hyperparameter configurations. By presenting multiple hyperparameters simultaneously, this view allows you to observe the interactions between variables, providing insights into their combined influence on the model. + +![hparam_loggings2](doc_images/tensorboard/tblogger_hparam2.jpg) + +The scatter plot matrix view provides an in-depth analysis of pairwise relationships between different hyperparameters. By visualizing correlations and patterns, this view aids in identifying key interactions that may influence the model's performance. + +![hparam_loggings3](doc_images/tensorboard/tblogger_hparam3.jpg) ## Status @@ -38,7 +186,7 @@ To show the status repeatedly, on unix systems you can use watch --interval 30 python -m neps.status ROOT_DIRECTORY ``` -## Visualizations +## CLI commands To generate plots to the root directory, run diff --git a/docs/citations.md b/docs/citations.md new file mode 100644 index 00000000..b697c49f --- /dev/null +++ b/docs/citations.md @@ -0,0 +1,54 @@ +# Citations + +## Citation of The Software + +For citing NePS, please refer to the following: + +### APA Style + +```apa +Stoll, D., Mallik, N., Schrodi, S., Janowski, M., Garibov, S., Abou Chakra, T., Rogalla, D., Bergman, E., Hvarfner, C., Binxin, R., Kober, N., Vallaeys, T., & Hutter, F. (2023). Neural Pipeline Search (NePS) (Version 0.11.0) [Computer software]. https://github.com/automl/neps +``` + +### BibTex Style + +```bibtex +@software{Stoll_Neural_Pipeline_Search_2023, +author = {Stoll, Danny and Mallik, Neeratyoy and Schrodi, Simon and Janowski, Maciej and Garibov, Samir and Abou Chakra, Tarek and Rogalla, Daniel and Bergman, Eddie and Hvarfner, Carl and Binxin, Ru and Kober, Nils and Vallaeys, Théophane and Hutter, Frank}, +month = oct, +title = {{Neural Pipeline Search (NePS)}}, +url = {https://github.com/automl/neps}, +version = {0.11.0}, +year = {2023} +} +``` + +## Citation of Papers + +### PriorBand + +If you have used [PriorBand](https://openreview.net/forum?id=uoiwugtpCH) as the optimizer, please use the bibtex below: + +```bibtex +@inproceedings{mallik2023priorband, +title = {PriorBand: Practical Hyperparameter Optimization in the Age of Deep Learning}, +author = {Neeratyoy Mallik and Eddie Bergman and Carl Hvarfner and Danny Stoll and Maciej Janowski and Marius Lindauer and Luigi Nardi and Frank Hutter}, +year = {2023}, +booktitle = {Thirty-seventh Conference on Neural Information Processing Systems (NeurIPS 2023)}, +keywords = {} +} +``` + +### Hierarchichal NAS with Context-free Grammars + +If you have used the context-free grammar search space and the graph kernels implemented in NePS for the paper [Hierarchical NAS](https://openreview.net/forum?id=Hpt1i5j6wh), please use the bibtex below: + +```bibtex +@inproceedings{schrodi2023hierarchical, +title = {Construction of Hierarchical Neural Architecture Search Spaces based on Context-free Grammars}, +author = {Simon Schrodi and Danny Stoll and Binxin Ru and Rhea Sanjay Sukthanker and Thomas Brox and Frank Hutter}, +year = {2023}, +booktitle = {Thirty-seventh Conference on Neural Information Processing Systems (NeurIPS 2023)}, +keywords = {} +} +``` diff --git a/docs/contributing/roadmap.md b/docs/contributing/roadmap.md index 85007640..a27c3d7d 100644 --- a/docs/contributing/roadmap.md +++ b/docs/contributing/roadmap.md @@ -1,43 +1,33 @@ # Roadmap -## Before 0.10.0 +## Before 0.12.0 ### Features -- Utility to get best HPs and architecture to pass to run_pipeline -- Plot in hours / days, but also show grid corresponding to 1x and 5x etc. - -### Fixes - -- Do not plot log y axis per default - -### Documentation - -- Fill up the core documentation pages - -## Before 0.11.0 - -### Features - -- Evolution as acq sampler +- Allow yaml based input of search space and the target function source to `neps.run` - Generate plot after each evaluation +- Support conditionals in ConfigSpace search space +- Support logging of optimizer state details ### Fixes - Open never closes (talk to Nils) - Deadlock in ASHA-like optimizers (talk to Neeratyoy) +- Extra metahyper sample in [issue 42](https://github.com/automl/neps/issues/42) +- Tighter type check in search space +- Unify MFObservedData class for both Hyperband-like fidelities and one-step fidelities ### Documentation -- Document summary function +- Improved documentation on all basic usage +- Improved README on github that links to the documentations +- Adequate examples targeting different user groups ### Refactoring - Merge GP and hierarchical GP - Merge gpytorch branch - Rethink summary/status API -- Utility to get incumbent losses over time -- Restructure folder structure - Improve placement of \_post_evaluation_hook_function - maintained vs unmaintained optimizers - Read and sample at the same time metahyper @@ -66,9 +56,9 @@ - Print search space upon run - Add comprehensive regression tests to run manually on the cluster on each version release - Utility to generate code for best architecture -- 3.11 support -- Deprecate 3.7 - Core Feature set in terms of research +- Modular plug-and-play of BoTorch acquisition functions +- Exploring gradient-based HPO in the NePS framework ### Fixes diff --git a/docs/doc_images/tensorboard/tblogger_hparam1.jpg b/docs/doc_images/tensorboard/tblogger_hparam1.jpg new file mode 100644 index 00000000..19ea6f11 Binary files /dev/null and b/docs/doc_images/tensorboard/tblogger_hparam1.jpg differ diff --git a/docs/doc_images/tensorboard/tblogger_hparam2.jpg b/docs/doc_images/tensorboard/tblogger_hparam2.jpg new file mode 100644 index 00000000..4341ce49 Binary files /dev/null and b/docs/doc_images/tensorboard/tblogger_hparam2.jpg differ diff --git a/docs/doc_images/tensorboard/tblogger_hparam3.jpg b/docs/doc_images/tensorboard/tblogger_hparam3.jpg new file mode 100644 index 00000000..af2121f9 Binary files /dev/null and b/docs/doc_images/tensorboard/tblogger_hparam3.jpg differ diff --git a/docs/doc_images/tensorboard/tblogger_image.jpg b/docs/doc_images/tensorboard/tblogger_image.jpg new file mode 100644 index 00000000..25807197 Binary files /dev/null and b/docs/doc_images/tensorboard/tblogger_image.jpg differ diff --git a/docs/doc_images/tensorboard/tblogger_scalar.jpg b/docs/doc_images/tensorboard/tblogger_scalar.jpg new file mode 100644 index 00000000..69d421bf Binary files /dev/null and b/docs/doc_images/tensorboard/tblogger_scalar.jpg differ diff --git a/docs/getting_started.md b/docs/getting_started.md index e69de29b..7f87e21b 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -0,0 +1,101 @@ +# Getting Started + +Getting started with NePS involves a straightforward yet powerful process, centering around its three main components. +This approach ensures flexibility and efficiency in evaluating different architecture and hyperparameter configurations +for your problem. + +## The 3 Main Components +1. **Define a [`run_pipeline`](https://automl.github.io/neps/latest/run_pipeline) Function**: This function is essential +for evaluating different configurations. You'll implement the specific logic for your problem within this function. +For detailed instructions on initializing and effectively using `run_pipeline`, refer to the guide. + +2. **Establish a [`pipeline_space`](https://automl.github.io/neps/latest/pipeline_space)**: Your search space for +defining parameters. You can structure this in various formats, including dictionaries, YAML, or ConfigSpace. +The guide offers insights into defining and configuring your search space. + +3. **Execute with [`neps.run`](https://automl.github.io/neps/latest/neps_run)**: Optimize your `run_pipeline` over +the `pipeline_space` using this function. For a thorough overview of the arguments and their explanations, +check out the detailed documentation. + +By following these steps and utilizing the extensive resources provided in the guides, you can tailor NePS to meet +your specific requirements, ensuring a streamlined and effective optimization process. + +## Basic Usage +In code, the usage pattern can look like this: + +```python +import neps +import logging + + +# 1. Define a function that accepts hyperparameters and computes the validation error +def run_pipeline( + hyperparameter_a: float, hyperparameter_b: int, architecture_parameter: str +) -> dict: + # insert here your own model + model = MyModel(architecture_parameter) + + # insert here your training/evaluation pipeline + validation_error, training_error = train_and_eval( + model, hyperparameter_a, hyperparameter_b + ) + + return { # dict or float(validation error) + "loss": validation_error, + "info_dict": { + "training_error": training_error + # + Other metrics + }, + } + + +# 2. Define a search space of the parameters of interest; ensure that the names are consistent with those defined +# in the run_pipeline function +pipeline_space = dict( + hyperparameter_b=neps.IntegerParameter( + lower=1, upper=42, is_fidelity=True + ), # Mark 'is_fidelity' as true for a multi-fidelity approach. + hyperparameter_a=neps.FloatParameter( + lower=0.001, upper=0.1, log=True + ), # If True, the search space is sampled in log space. + architecture_parameter=neps.CategoricalParameter( + ["option_a", "option_b", "option_c"] + ), +) + +if __name__ == "__main__": + # 3. Run the NePS optimization + logging.basicConfig(level=logging.INFO) + neps.run( + run_pipeline=run_pipeline, + pipeline_space=pipeline_space, + root_directory="path/to/save/results", # Replace with the actual path. + max_evaluations_total=100, + searcher="hyperband" # Optional specifies the search strategy, + # otherwise NePs decides based on your data. + ) +``` + +## Examples + +Discover the features of NePS through these practical examples: + +* **[Hyperparameter Optimization (HPO)]( +https://github.com/automl/neps/blob/master/neps_examples/template/basic_template.py)**: Learn the essentials of +hyperparameter optimization with NePS. + +* **[Architecture Search with Primitives]( +https://github.com/automl/neps/tree/master/neps_examples/basic_usage/architecture.py)**: Dive into architecture search +using primitives in NePS. + +* **[Multi-Fidelity Optimization]( +https://github.com/automl/neps/tree/master/neps_examples/efficiency/multi_fidelity.py)**: Understand how to leverage +multi-fidelity optimization for efficient model tuning. + +* **[Utilizing Expert Priors for Hyperparameters]( +https://github.com/automl/neps/blob/master/neps_examples/template/priorband_template.py)**: +Learn how to incorporate expert priors for more efficient hyperparameter selection. + +* **[Additional NePS Examples]( +https://github.com/automl/neps/tree/master/neps_examples/)**: Explore more examples, including various use cases and +advanced configurations in NePS. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 00000000..90066f37 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,24 @@ +# Installation + +## Prerequisites + +Ensure you have Python version 3.8, 3.9, 3.10, or 3.11 installed. NePS installation will automatically handle +any additional dependencies via pip. + +## Install from pip + +```bash +pip install neural-pipeline-search +``` +> Note: As indicated with the `v0.x.x` version number, NePS is early stage code and APIs might change in the future. + +## Install from source + +!!! note + We use [poetry](https://python-poetry.org/docs/) to manage dependecies. + +```bash +git clone https://github.com/automl/neps.git +cd neps +poetry install --no-dev +``` diff --git a/docs/neps_run.md b/docs/neps_run.md index e69de29b..886ff91c 100644 --- a/docs/neps_run.md +++ b/docs/neps_run.md @@ -0,0 +1,100 @@ +# Configuring and Running Optimizations + +The `neps.run` function is the core of the NePS optimization process, where the search for the best hyperparameters +and architectures takes place. This document outlines the arguments and options available within this function, +providing a detailed guide to customize the optimization process to your specific needs. + +## Search Strategy +At default NePS intelligently selects the most appropriate search strategy based on your defined configurations in +`pipeline_space`. +The characteristics of your search space, as represented in the `pipeline_space`, play a crucial role in determining +which optimizer NePS will choose. This automatic selection process ensures that the strategy aligns perfectly +with the specific requirements and nuances of your search space, thereby optimizing the effectiveness of the +hyperparameter and/or architecture optimization. You can also manually select a specific or custom optimizer that better +matches your specific needs. For more information, refer [here](https://automl.github.io/neps/latest/optimizers). + +## Arguments + +### Mandatory Arguments +- **`run_pipeline`** (function): The objective function, targeted by NePS for minimization, by evaluation various + configurations. It requires these configurations as input and should return either a dictionary or a sole loss + value as the +output. For correct setup instructions, refer to [here](https://automl.github.io/neps/latest/run_pipeline) +- **`pipeline_space`** (dict | yaml | configspace): This defines the search space for the configurations from which the + optimizer samples. It accepts either a dictionary with the configuration names as keys, a path to a YAML + configuration file, or a configSpace.ConfigurationSpace object. For comprehensive information and examples, + please refer to the detailed guide available [here](https://automl.github.io/neps/latest/pipeline_space) + +- **`root_directory`** (str): The directory path where the information about the optimization and its progress gets + stored. This is also used to synchronize multiple calls to run(.) for parallelization. + +- **Budget**: +To define a budget, provide either or both of the following parameters: + + - **`max_evaluations_total`** (int, default: None): Specifies the total number of evaluations to conduct before + halting the optimization process. + - **`max_cost_total`** (int, default: None): Prevents the initiation of new evaluations once this cost + threshold is surpassed. This requires adding a cost value to the output of the `run_pipeline` function, + for example, return {'loss': loss, 'cost': cost}. For more details, please refer + [here](https://automl.github/io/neps/latest/run_pipeline) + +### Optional Arguments +##### Further Monitoring Options + - **`overwrite_working_directory`** (bool, default: False): When set to True, the working directory + specified by + `root_directory` will be + cleared at the beginning of the run. This is e.g. useful when debugging a `run_pipeline` function. + - **`post_run_summary`** (bool, default: False): When enabled, this option generates a summary CSV file + upon the + completion of the + optimization process. The summary includes details of the optimization procedure, such as the best configuration, + the number of errors occurred, and the final performance metrics. + - **`development_stage_id`** (int | float | str, default: None): An optional identifier used when working with + multiple development stages. Instead of creating new root directories, use this identifier to save the results + of an optimization run in a separate dev_id folder within the root_directory. + - **`task_id`** (int | float | str, default: None): An optional identifier used when the optimization process + involves multiple tasks. This functions similarly to `development_stage_id`, but it creates a folder named + after the task_id instead of dev_id, providing an organized way to separate results for different tasks within + the `root_directory`. +##### Parallelization Setup + - **`max_evaluations_per_run`** (int, default: None): Limits the number of evaluations for this specific call of + `neps.run`. + - **`continue_until_max_evaluation_completed`** (bool, default: False): In parallel setups, pending evaluations + normally count towards max_evaluations_total, halting new ones when this limit is reached. Setting this to + True enables continuous sampling of new evaluations until the total of completed ones meets max_evaluations_total, + optimizing resource use in time-sensitive scenarios. + +For an overview and further resources on how NePS supports parallelization in distributed systems, refer to +the [Parallelization Overview](#parallelization). +##### Handling Errors + - **`loss_value_on_error`** (float, default: None): When set, any error encountered in an evaluated configuration + will not halt the process; instead, the specified loss value will be used for that configuration. + - **`cost_value_on_error`** (float, default: None): Similar to `loss_value_on_error`, but for the cost value. + - **`ignore_errors`** (bool, default: False): If True, errors encountered during the evaluation of configurations + will be ignored, and the optimization will continue. Note: This error configs still count towards + max_evaluations_total. +##### Search Strategy Customization + - **`searcher`** (Literal["bayesian_optimization", "hyperband",..] | BaseOptimizer, default: "default"): Specifies + manually which of the optimization strategy to use. Provide a string identifying one of the built-in + search strategies or an instance of a custom `BaseOptimizer`. + - **`searcher_path`** (Path | str, default: None): A path to a custom searcher implementation. + - **`**searcher_kwargs`**: Additional keyword arguments to be passed to the searcher. + + For more information about the available searchers and how to customize your own, refer +[here](https://automl.github.io/neps/latest/optimizers). +##### Others + - **`pre_load_hooks`** (Iterable, default: None): A list of hook functions to be called before loading results. + +## Parallelization + +`neps.run` can be called multiple times with multiple processes or machines, to parallelize the optimization process. +Ensure that `root_directory` points to a shared location across all instances to synchronize the optimization efforts. +For more information [look here](https://automl.github.io/neps/latest/parallelization) + +## Customization + +The `neps.run` function allows for extensive customization through its arguments, enabling to adapt the +optimization process to the complexities of your specific problems. + +For a deeper understanding of how to use `neps.run` in a practical scenario, take a look at our +[examples and templates](https://github.com/automl/neps/tree/master/neps_examples). diff --git a/src/neps/optimizers/README.md b/docs/optimizers.md similarity index 81% rename from src/neps/optimizers/README.md rename to docs/optimizers.md index 4d2a0a47..c6155ed6 100644 --- a/src/neps/optimizers/README.md +++ b/docs/optimizers.md @@ -1,50 +1,60 @@ ## Optimizer Configuration Options -Before running the optimizer for your AutoML tasks, you have several configuration options to tailor the optimization process to your specific needs. These options allow you to customize the optimizer's behavior according to your preferences and requirements. +Before running the optimizer for your AutoML tasks, you have several configuration options to tailor the optimization +process to your specific needs. These options allow you to customize the optimizer's behavior according to your +preferences and requirements. ### 1. Automatic Optimizer Selection -If you prefer not to specify a particular optimizer for your AutoML task, you can simply pass `default` or `None` for the neps searcher. NePS will automatically choose the best optimizer based on the characteristics of your search space. This provides a hassle-free way to get started quickly. +If you prefer not to specify a particular optimizer for your AutoML task, you can simply pass `"default"` or `None` +for the neps searcher. NePS will automatically choose the best optimizer based on the characteristics of your search +space. This provides a hassle-free way to get started quickly. -The optimizer selection is based on the following characteristics of your search space: +The optimizer selection is based on the following characteristics of your `pipeline_space`: - If it has fidelity: `hyperband` - If it has both fidelity and a prior: `priorband` - If it has a prior: `pibo` - If it has neither: `bayesian_optimization` -For example, running the following format, without specifying a searcher will choose an optimizer depending on the `pipeline_space` passed. +For example, running the following format, without specifying a searcher will choose an optimizer depending on +the `pipeline_space` passed. ```python neps.run( run_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + max_evaluations_total=25, # no searcher specified ) ``` ### 2. Choosing one of NePS Optimizers -We have also prepared some optimizers with specific hyperparameters that we believe can generalize well to most AutoML tasks and use cases. For more details on the available default optimizers and the algorithms that can be called, please refer to the next section on [SearcherConfigs](#Searcher-Configurations). +We have also prepared some optimizers with specific hyperparameters that we believe can generalize well to most AutoML +tasks and use cases. For more details on the available default optimizers and the algorithms that can be called, +please refer to the next section on [SearcherConfigs](#searcher-configurations). ```python neps.run( run_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + max_evaluations_total=25, # searcher specified, along with an argument searcher="bayesian_optimization", initial_design_size=5, ) ``` -For more optimizers, please refer [here](#List-Available-Searchers) . +For more optimizers, please refer [here](#list-available-searchers) . ### 3. Custom Optimizer Configuration via YAML -For users who want more control over the optimizer's hyperparameters, you can create your own YAML configuration file. In this file, you can specify the hyperparameters for your preferred optimizer. To use this custom configuration, provide the path to your YAML file using the `searcher_path` parameter when running the optimizer. The library will then load your custom settings and use them for optimization. +For users who want more control over the optimizer's hyperparameters, you can create your own YAML configuration file. +In this file, you can specify the hyperparameters for your preferred optimizer. To use this custom configuration, +provide the path to your YAML file using the `searcher_path` parameter when running the optimizer. +The library will then load your custom settings and use them for optimization. Here's the format of a custom YAML (`custom_bo.yaml`) configuration using `Bayesian Optimization` as an example: @@ -68,7 +78,7 @@ neps.run( run_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + max_evaluations_total=25, # searcher specified, along with an argument searcher_path = "custom/path/to/directory" # `custom_bo.yaml` should be in `searcher_path` @@ -85,11 +95,11 @@ neps.run( run_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", - max_evaluations_total=25, + max_evaluations_total=25, # searcher specified, along with an argument searcher_path = "custom/path/to/directory" # `custom_bo.yaml` should be in `searcher_path` - searcher="custom_bo", + searcher="custom_bo", initial_design_size=5, # overrides value in custom_bo.yaml random_interleave_prob: 0.25 # overrides value in custom_bo.yaml ) diff --git a/docs/parallelization.md b/docs/parallelization.md index ff001186..9a265a40 100644 --- a/docs/parallelization.md +++ b/docs/parallelization.md @@ -1,4 +1,46 @@ -# Parallelization +# Parallelization and Resuming Runs -In order to run a neural pipeline search with multiple processes or multiple machines, simply call `neps.run` multiple times. -All calls to `neps.run` need to use the same `root_directory` on the same filesystem, otherwise there is no synchronization between the `neps.run`'s. +NePS utilizes files as a means of communication for implementing parallelization and resuming runs. As a result, +when `neps.run` is called multiple times with the same `root_directory` in the file system, NePS will automatically +load the optimizer state, allowing seamless parallelization of the run across different processes or machines. +This concept also applies to resuming runs even after termination. + +Example: + +!!! note + The following example assumes all necessary imports are included, in addition to already having defined the [pipeline_space](https://automl.github.io/neps/latest/pipeline_space/) and the [run_pipeline](https://automl.github.io/neps/latest/run_pipeline/) functions. One can apply the same idea on [this](https://github.com/automl/neps/blob/master/neps_examples/basic_usage/hyperparameters.py) example. + +```python +logging.basicConfig(level=logging.INFO) + +# Initial run +neps.run( + run_pipeline=run_pipeline, + pipeline_space=pipeline_space, + root_directory="results/my_example", + max_evaluations_total=5, +) +``` + +After the initial run, NePS will log the following message: + +```bash +INFO:neps:Maximum total evaluations is reached, shutting down +``` + +If you wish to extend the search with more evaluations, simply update the `max_evaluations_total` parameter: + +```python +logging.basicConfig(level=logging.INFO) + + +# Resuming run with increased evaluations +neps.run( + run_pipeline=run_pipeline, + pipeline_space=pipeline_space, + root_directory="results/my_example", + max_evaluations_total=10, +) +``` + +Now, NePS will continue the search, loading the latest information for the searcher. For parallelization, as mentioned above, you can also run this code multiple times on different processes or machines. The file system communication will link them, as long as the `root_directory` has the same location. diff --git a/docs/pipeline_space.md b/docs/pipeline_space.md index e69de29b..ab49a221 100644 --- a/docs/pipeline_space.md +++ b/docs/pipeline_space.md @@ -0,0 +1,180 @@ +# Initializing the Pipeline Space + +In NePS, a pivotal step is the definition of the search space, termed `pipeline_space`. This space can be structured +through various approaches, including a Python dictionary, a YAML file, or ConfigSpace. Each of these methods allows +you to specify a set of parameter types, ranging from Float and Categorical to specialized architecture parameters. +Whether you choose a dictionary, YAML file, or ConfigSpace, your selected method serves as a container or framework +within which these parameters are defined and organized. This section not only guides you through the process of +setting up your `pipeline_space` using these methods but also provides detailed instructions and examples on how to +effectively incorporate various parameter types, ensuring that NePS can utilize them in the optimization process. + + +## Methods for Defining the NePS Pipeline Space +### Option 1: Using a Python Dictionary + +To define the `pipeline_space` using a Python dictionary, follow these steps: + +Create a Python dictionary that specifies the parameters and their respective ranges. For example: + +```python +pipeline_space = { + "learning_rate": neps.FloatParameter(lower=0.00001, upper=0.1, log=True), + "num_epochs": neps.IntegerParameter(lower=3, upper=30, is_fidelity=True), + "optimizer": neps.CategoricalParameter(choices=["adam", "sgd", "rmsprop"]), + "dropout_rate": neps.FloatParameter(value=0.5), +} +``` + +### Option 2: Using a YAML File + +Create a YAML file (e.g., pipeline_space.yaml) with the parameter definitions following this structure. + +```yaml +pipeline_space: # important to start with + learning_rate: + lower: 2e-3 + upper: 0.1 + log: true + + num_epochs: + type: int # or "integer", optional if u want to manually set this + lower: 3 + upper: 30 + is_fidelity: True + + optimizer: + choices: ["adam", "sgd", "rmsprop"] + + dropout_rate: + value: 0.5 +... +``` + +Ensure your YAML file starts with `pipeline_space:`. +This is the root key under which all parameter configurations are defined. + +!!! note "Note" + The various types of parameters displayed in the Dictionary of Option 1 are here automatically determined by the + data. If desired, you have the option to define them manually by providing the argument `type`. For more details, + refer to the section on [Supported Hyperparameter Types](#supported-hyperparameter-types). + + +### Option 3: Using ConfigSpace + +For users familiar with the ConfigSpace library, can also define the `pipeline_space` through +ConfigurationSpace(). + +```python +from configspace import ConfigurationSpace, UniformFloatHyperparameter + +configspace = ConfigurationSpace() +configspace.add_hyperparameter( + UniformFloatHyperparameter("learning_rate", 0.00001, 0.1, log=True) +) +``` + +For additional information on ConfigSpace and its features, please visit the following +[link](https://github.com/automl/ConfigSpace). +## Supported Hyperparameter Types + +### Float/Integer Parameter + +- **Expected Arguments:** + - `lower`: The minimum value of the parameter. + - `upper`: The maximum value of the parameter. + - Accepted values: int or float depending on the specific parameter type one wishes to use. +- **Optional Arguments:** + - `log`: Boolean that indicates if the parameter uses a logarithmic scale (default: False) + - [Details on how YAML interpret Boolean Values](#important-note-on-yaml-data-type-interpretation) + - `is_fidelity`: Boolean that marks the parameter as a fidelity parameter (default: False). + - `default`: Sets a prior central value for the parameter (default: None). + > Note: Currently, if you define a prior for one parameter, you must do so for all your variables. + - `default_confidence`: Specifies the confidence level of the default value, + indicating how strongly the prior + should be considered (default: 'low'). + - Accepted values: 'low', 'medium', or 'high'. + - `type`: Specifies the data type of the parameter. + - Accepted values: 'int', 'integer', or 'float'. + > Note: If type is not specified e notation gets converted to float + + !!! note "YAML Method Specific:" + The type argument, used to specify the data type of parameters as 'int', 'integer', or 'float', + is unique to defining the pipeline_space with a YAML file. This explicit specification of the parameter + type is not required when using a Python dictionary or ConfigSpace, as these methods inherently determine + the data types based on the syntax and structure of the code. + +### Categorical Parameter + +- **Expected Arguments:** + - `choices`: A list of discrete options (int | float | str) that the parameter can take. +- **Optional Arguments:** + - `is_fidelity`: Marks the parameter as a fidelity parameter (default: False). + - [Details on how YAML interpret Boolean Values](#important-note-on-yaml-data-type-interpretation) + - `default`: Sets a prior central value for the parameter (default: None). + > Note: Currently, if you define a prior for one parameter, you must do so for all your variables. + - `default_confidence`: Specifies the confidence level of the default value, + indicating how strongly the prior + should be considered (default: "low"). + - `type`: Specifies the data type of the parameter. + - Accepted values: 'cat' or 'categorical'. + > Note: Yaml Method Specific + +### Constant Parameter + +- **Expected Arguments:** + - `value`: The fixed value (int | float | str) for the parameter. +- **Optional Arguments:** + - `type`: Specifies the data type of the parameter. + - Accepted values: 'const' or 'constant'. + > Note: Yaml Method Specific + - `is_fidelity`: Marks the parameter as a fidelity parameter (default: False). + +### Important Note on YAML Data Type Interpretation + +When working with YAML files, it's essential to understand how the format interprets different data types: + +1. **Strings in Quotes:** + + - Any value enclosed in single (`'`) or double (`"`) quotes is treated as a string. + - Example: `"true"`, `'123'` are read as strings. + +2. **Boolean Interpretation:** + + - Specific unquoted values are interpreted as booleans. This includes: + - `true`, `True`, `TRUE` + - `false`, `False`, `FALSE` + - `on`, `On`, `ON` + - `off`, `Off`, `OFF` + - `yes`, `Yes`, `YES` + - `no`, `No`, `NO` + +3. **Numbers:** + + - Unquoted numeric values are interpreted as integers or floating-point numbers, depending on their format. + - By default, when the 'type' is not specified, any number in scientific notation (e.g., 1e3) is interpreted as a + floating-point number. This interpretation is unique to our system. + +4. **Empty Strings:** + + - An empty string `""` or a key with no value is always treated as `null` in YAML. + +5. **Unquoted Non-Boolean, Non-Numeric Strings:** + + - Unquoted values that don't match boolean patterns or numeric formats are treated as strings. + - Example: `example` is a string. + +Remember to use appropriate quotes and formats to ensure values are interpreted as intended. + +## Supported Architecture parameter Types + +!!! note "Note" + The configuration of `pipeline_space` from a YAML file does not currently support architecture parameter types. +!!! note "Note" + A comprehensive documentation for the Architecture parameter will be available soon. + If you are interested in exploring architecture parameters, you can find detailed + examples and usage in the following resources: + + - [Basic Usage Examples](https://github.com/automl/neps/tree/master/neps_examples/basic_usage) - Basic usage + examples that can help you understand the fundamentals of Architecture parameters. + - [Experimental Examples](https://github.com/automl/neps/tree/master/neps_examples/experimental) - For more advanced + and experimental use cases, including Hierarchical parameters, check out this collection of examples. diff --git a/docs/run_pipeline.md b/docs/run_pipeline.md index e69de29b..fb0e9bed 100644 --- a/docs/run_pipeline.md +++ b/docs/run_pipeline.md @@ -0,0 +1,150 @@ +# The run_pipeline Function + +## Introduction + +The `run_pipeline` function is crucial for NePS. It encapsulates the objective function to be minimized, which could range from a regular equation to a full training and evaluation pipeline for a neural network. + +This function receives the configuration to be utilized from the parameters defined in the search space. Consequently, it executes the same set of instructions or equations based on the provided configuration to minimize the objective function. + +We will show some basic usages and some functionalites this function would require for successful implementation. + +## Types of Returns + +### 1. Single Value + +Assuming the `pipeline_space` was already created (have a look at [pipeline space](https://automl.github.io/neps/latest/pipeline_space/) for more details). A `run_pipeline` function with an objective of minimizing the loss will resemble the following: + +```python +def run_pipeline( + **config, # The hyperparameters to be used in the pipeline +): + element_1 = config["element_1"] + element_2 = config["element_2"] + element_3 = config["element_3"] + + loss = element_1 - element_2 + element_3 + + return loss +``` + +### 2. Dictionary + +In this section, we will outline the special variables that are expected to be returned when the `run_pipeline` function returns a dictionary. + +#### Loss + +One crucial return variable is the `loss`. This metric serves as a fundamental indicator for the optimizer. One option is to return a dictionary with the `loss` as a key, along with other user-chosen metrics. + +!!! note + + Loss can be any value that is to be minimized by the objective function. + +```python +def run_pipeline( + **config, # The hyperparameters to be used in the pipeline +): + + element_1 = config["element_1"] + element_2 = config["element_2"] + element_3 = config["element_3"] + + loss = element_1 - element_2 + element_3 + reverse_loss = -loss + + return { + "loss": loss, + "info_dict": { + "reverse_loss": reverse_loss + ... + } + } +``` + +#### Cost + +Along with the return of the `loss`, the `run_pipeline` function would optionally need to return a `cost` in certain cases. Specifically when the `max_cost_total` parameter is being utilized in the `neps.run` function. + + +!!! note + + `max_cost_total` sums the cost from all returned configuration results and checks whether the maximum allowed cost has been reached (if so, the search will come to an end). + +```python +import neps +import logging + + +def run_pipeline( + **config, # The hyperparameters to be used in the pipeline +): + + element_1 = config["element_1"] + element_2 = config["element_2"] + element_3 = config["element_3"] + + loss = element_1 - element_2 + element_3 + cost = 2 + + return { + "loss": loss, + "cost": cost, + } + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + neps.run( + run_pipeline=run_pipeline, + pipeline_space=pipeline_space, # Assuming the pipeline space is defined + root_directory="results/bo", + max_cost_total=10, + searcher="bayesian_optimization", + ) +``` + +Each evaluation carries a cost of 2. Hence in this example, the Bayesian optimization search is set to perform 5 evaluations. + +## Arguments for Convenience + +NePS also provides the `pipeline_directory` and the `previous_pipeline_directory` as arguments in the `run_pipeline` function for user convenience. + +Regard an example to be run with a multi-fidelity searcher, some checkpointing would be advantageos such that one does not have to train the configuration from scratch when the configuration qualifies to higher fidelity brackets. + +```python +def run_pipeline( + pipeline_directory, # The directory where the config is saved + previous_pipeline_directory, # The directory of the immediate lower fidelity config + **config, # The hyperparameters to be used in the pipeline +): + # Assume element3 is our fidelity element + element_1 = config["element_1"] + element_2 = config["element_2"] + element_3 = config["element_3"] + + # Load any saved checkpoints + checkpoint_name = "checkpoint.pth" + start_element_3 = 0 + + if previous_pipeline_directory is not None: + # Read in state of the model after the previous fidelity rung + checkpoint = torch.load(previous_pipeline_directory / checkpoint_name) + prev_element_3 = checkpoint["element_3"] + else: + prev_element_3 = 0 + + start_element_3 += prev_element_3 + + loss = 0 + for i in range(start_element_3, element_3): + loss += element_1 - element_2 + + torch.save( + { + "element_3": element_3, + }, + pipeline_directory / checkpoint_name, + ) + + return loss +``` + +This could allow the proper navigation to the trained models and further train them on higher fidelities without repeating the entire training process. diff --git a/mkdocs.yml b/mkdocs.yml index 51d4a407..5afc9246 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: Neural Pipeline Search +site_name: NePS docs_dir: docs repo_url: https://github.com/automl/neps repo_name: automl/neps @@ -9,6 +9,18 @@ theme: icon: repo: fontawesome/brands/github + palette: + # Palette toggle for light mode + - scheme: default + toggle: + icon: material/weather-night + name: Switch to dark mode + # Palette toggle for dark mode + - scheme: slate + toggle: + icon: material/weather-sunny + name: Switch to light mode + markdown_extensions: - pymdownx.highlight: anchor_linenums: true @@ -16,19 +28,24 @@ markdown_extensions: - pymdownx.snippets - pymdownx.superfences - attr_list + - admonition + - attr_list + - md_in_html extra: version: provider: mike nav: - - Introduction and Installation: 'README.md' + - Installation: 'installation.md' - Getting Started: 'getting_started.md' - The run_pipeline Function: 'run_pipeline.md' - The pipeline_space: 'pipeline_space.md' - The neps.run Function: 'neps_run.md' + - Optimizers: 'optimizers.md' - Analysing Runs: 'analyse.md' - Parallelization: 'parallelization.md' + - Citations: 'citations.md' - Alternatives: 'alternatives.md' - Contributing: - Introduction: "contributing/README.md" diff --git a/src/neps/NOTICE b/neps/NOTICE similarity index 100% rename from src/neps/NOTICE rename to neps/NOTICE diff --git a/src/neps/__init__.py b/neps/__init__.py similarity index 100% rename from src/neps/__init__.py rename to neps/__init__.py diff --git a/src/neps/api.py b/neps/api.py similarity index 76% rename from src/neps/api.py rename to neps/api.py index 24efe7e8..2584fcbe 100644 --- a/src/neps/api.py +++ b/neps/api.py @@ -6,19 +6,21 @@ import logging import warnings from pathlib import Path -from typing import Callable, Literal, List +from typing import Callable, Iterable, Literal import ConfigSpace as CS -import metahyper -from metahyper import instance_from_map - +from .metahyper import instance_from_map, metahyper_run from .optimizers import BaseOptimizer, SearcherMapping from .plot.tensorboard_eval import tblogger from .search_spaces.parameter import Parameter -from .search_spaces.search_space import SearchSpace, pipeline_space_from_configspace +from .search_spaces.search_space import ( + SearchSpace, + pipeline_space_from_configspace, + pipeline_space_from_yaml, +) from .status.status import post_run_csv -from .utils.common import get_searcher_data +from .utils.common import get_searcher_data, get_value from .utils.result_utils import get_loss @@ -95,8 +97,14 @@ def write_loss_and_config(file_handle, loss_, config_id_, config_): def run( run_pipeline: Callable, - pipeline_space: dict[str, Parameter | CS.ConfigurationSpace] | CS.ConfigurationSpace, root_directory: str | Path, + pipeline_space: ( + dict[str, Parameter | CS.ConfigurationSpace] + | str + | Path + | CS.ConfigurationSpace + | None + ) = None, overwrite_working_directory: bool = False, post_run_summary: bool = False, development_stage_id=None, @@ -108,18 +116,20 @@ def run( ignore_errors: bool = False, loss_value_on_error: None | float = None, cost_value_on_error: None | float = None, - pre_load_hooks: List=[], - searcher: Literal[ - "default", - "bayesian_optimization", - "random_search", - "hyperband", - "priorband", - "mobster", - "asha", - "regularized_evolution", - ] - | BaseOptimizer = "default", + pre_load_hooks: Iterable | None = None, + searcher: ( + Literal[ + "default", + "bayesian_optimization", + "random_search", + "hyperband", + "priorband", + "mobster", + "asha", + "regularized_evolution", + ] + | BaseOptimizer + ) = "default", searcher_path: Path | str | None = None, **searcher_kwargs, ) -> None: @@ -169,7 +179,6 @@ def run( Raises: ValueError: If deprecated argument working_directory is used. ValueError: If root_directory is None. - TypeError: If pipeline_space has invalid type. Example: @@ -204,24 +213,35 @@ def run( ) max_cost_total = searcher_kwargs["budget"] del searcher_kwargs["budget"] - + + if pre_load_hooks is None: + pre_load_hooks = [] + logger = logging.getLogger("neps") logger.info(f"Starting neps.run using root directory {root_directory}") - + + # Used to create the yaml holding information about the searcher. + # Also important for testing and debugging the api. + searcher_info = { + "searcher_name": "", + "searcher_alg": "", + "searcher_selection": "", + "neps_decision_tree": True, + "searcher_args": {}, + } + if isinstance(searcher, BaseOptimizer): searcher_instance = searcher - searcher_name = "custom" - searcher_alg = searcher.whoami() - user_defined_searcher = True + searcher_info["searcher_name"] = "baseoptimizer" + searcher_info["searcher_alg"] = searcher.whoami() + searcher_info["searcher_selection"] = "user-instantiation" + searcher_info["neps_decision_tree"] = False else: - ( - searcher_name, - searcher_instance, - searcher_alg, - searcher_config, - searcher_info, - user_defined_searcher + ( + searcher_instance, + searcher_info, ) = _run_args( + searcher_info=searcher_info, pipeline_space=pipeline_space, max_cost_total=max_cost_total, ignore_errors=ignore_errors, @@ -233,52 +253,25 @@ def run( **searcher_kwargs, ) - # Used to create the yaml holding information about the searcher. - # Also important for testing and debugging the api. - searcher_info = { - "searcher_name": searcher_name, - "searcher_alg": searcher_alg, - "user_defined_searcher": user_defined_searcher, - "searcher_args_user_modified": False, - } - - # Check to verify if the target directory contains the history of another optimizer state + # Check to verify if the target directory contains history of another optimizer state # This check is performed only when the `searcher` is built during the run - if isinstance(searcher, BaseOptimizer): + if not isinstance(searcher, (BaseOptimizer, str)): + raise ValueError( + f"Unrecognized `searcher` of type {type(searcher)}. Not str or BaseOptimizer." + ) + elif isinstance(searcher, BaseOptimizer): # This check is not strict when a user-defined neps.optimizer is provided - logger.warn( + logger.warning( "An instantiated optimizer is provided. The safety checks of NePS will be " "skipped. Accurate continuation of runs can no longer be guaranteed!" ) - elif isinstance(searcher, str): - # Updating searcher arguments from searcher_kwargs - for key, value in searcher_kwargs.items(): - if user_defined_searcher: - if key not in searcher_config or searcher_config[key] != value: - searcher_config[key] = value - logger.info( - f"Updating the current searcher argument '{key}'" - f" with the value '{value}'" - ) - else: - logger.info( - f"The searcher argument '{key}' has the same" - f" value '{value}' as default." - ) - searcher_info["searcher_args_user_modified"] = True - else: - # No searcher argument updates when NePS decides the searcher. - logger.info(35 * "=" + "WARNING" + 35 * "=") - logger.info("CHANGING ARGUMENTS ONLY WORK WHEN SEARCHER IS DEFINED") - logger.info( - f"The searcher argument '{key}' will not change to '{value}'" - f" because NePS chose the searcher" - ) - searcher_info["searcher_args_user_modified"] = False - else: - raise ValueError(f"Unrecognized `searcher`. Not str or BaseOptimizer.") - - metahyper.run( + + if task_id is not None: + root_directory = Path(root_directory) / f"task_{task_id}" + if development_stage_id is not None: + root_directory = Path(root_directory) / f"dev_{development_stage_id}" + + metahyper_run( run_pipeline, searcher_instance, searcher_info, @@ -286,8 +279,6 @@ def run( max_evaluations_total=max_evaluations_total, max_evaluations_per_run=max_evaluations_per_run, continue_until_max_evaluation_completed=continue_until_max_evaluation_completed, - development_stage_id=development_stage_id, - task_id=task_id, logger=logger, post_evaluation_hook=_post_evaluation_hook_function( loss_value_on_error, ignore_errors @@ -296,35 +287,52 @@ def run( pre_load_hooks=pre_load_hooks, ) - if post_run_csv: + if post_run_summary: post_run_csv(root_directory, logger) def _run_args( - pipeline_space: dict[str, Parameter | CS.ConfigurationSpace] | CS.ConfigurationSpace, + searcher_info: dict, + pipeline_space: ( + dict[str, Parameter | CS.ConfigurationSpace] + | str + | Path + | CS.ConfigurationSpace + | None + ) = None, max_cost_total: int | float | None = None, ignore_errors: bool = False, loss_value_on_error: None | float = None, cost_value_on_error: None | float = None, logger=None, - searcher: Literal[ - "default", - "bayesian_optimization", - "random_search", - "hyperband", - "priorband", - "mobster", - "asha", - "regularized_evolution", - ] - | BaseOptimizer = "default", + searcher: ( + Literal[ + "default", + "bayesian_optimization", + "random_search", + "hyperband", + "priorband", + "mobster", + "asha", + "regularized_evolution", + ] + | BaseOptimizer + ) = "default", searcher_path: Path | str | None = None, **searcher_kwargs, -) -> None: +) -> tuple[BaseOptimizer, dict]: try: + # Raising an issue if pipeline_space is None + if pipeline_space is None: + raise ValueError( + "The choice of searcher requires a pipeline space to be provided" + ) # Support pipeline space as ConfigurationSpace definition if isinstance(pipeline_space, CS.ConfigurationSpace): pipeline_space = pipeline_space_from_configspace(pipeline_space) + # Support pipeline space as YAML file + elif isinstance(pipeline_space, (str, Path)): + pipeline_space = pipeline_space_from_yaml(pipeline_space) # Support pipeline space as mix of ConfigurationSpace and neps parameters new_pipeline_space: dict[str, Parameter] = dict() @@ -335,21 +343,20 @@ def _run_args( else: new_pipeline_space[key] = value pipeline_space = new_pipeline_space - + # Transform to neps internal representation of the pipeline space pipeline_space = SearchSpace(**pipeline_space) except TypeError as e: message = f"The pipeline_space has invalid type: {type(pipeline_space)}" raise TypeError(message) from e - user_defined_searcher = False - if isinstance(searcher, str) and searcher_path is not None: # The users has their own custom searcher. logging.info("Preparing to run user created searcher") config = get_searcher_data(searcher, searcher_path) - user_defined_searcher = True + searcher_info["searcher_selection"] = "user-yaml" + searcher_info["neps_decision_tree"] = False else: if searcher in ["default", None]: # NePS decides the searcher according to the pipeline space. @@ -361,42 +368,41 @@ def _run_args( if pipeline_space.has_fidelity else "bayesian_optimization" ) + searcher_info["searcher_selection"] = "neps-default" else: # Users choose one of NePS searchers. - user_defined_searcher = True + searcher_info["neps_decision_tree"] = False + searcher_info["searcher_selection"] = "neps-default" # Fetching the searcher data, throws an error when the searcher is not found config = get_searcher_data(searcher) searcher_alg = config["searcher_init"]["algorithm"] - searcher_config = {} if config["searcher_kwargs"] is None else config["searcher_kwargs"] + searcher_config = ( + {} if config["searcher_kwargs"] is None else config["searcher_kwargs"] + ) logger.info(f"Running {searcher} as the searcher") logger.info(f"Algorithm: {searcher_alg}") # Used to create the yaml holding information about the searcher. # Also important for testing and debugging the api. - searcher_info = { - "searcher_name": searcher, - "searcher_alg": searcher_alg, - "user_defined_searcher": user_defined_searcher, - "searcher_args_user_modified": False, - } + searcher_info["searcher_name"] = searcher + searcher_info["searcher_alg"] = searcher_alg # Updating searcher arguments from searcher_kwargs for key, value in searcher_kwargs.items(): - if user_defined_searcher: + if not searcher_info["neps_decision_tree"]: if key not in searcher_config or searcher_config[key] != value: searcher_config[key] = value logger.info( f"Updating the current searcher argument '{key}'" - f" with the value '{value}'" + f" with the value '{get_value(value)}'" ) else: logger.info( f"The searcher argument '{key}' has the same" - f" value '{value}' as default." + f" value '{get_value(value)}' as default." ) - searcher_info["searcher_args_user_modified"] = True else: # No searcher argument updates when NePS decides the searcher. logger.info(35 * "=" + "WARNING" + 35 * "=") @@ -405,7 +411,8 @@ def _run_args( f"The searcher argument '{key}' will not change to '{value}'" f" because NePS chose the searcher" ) - searcher_info["searcher_args_user_modified"] = False + + searcher_info["searcher_args"] = get_value(searcher_config) searcher_config.update( { @@ -414,7 +421,7 @@ def _run_args( "ignore_errors": ignore_errors, } ) - + searcher_instance = instance_from_map( SearcherMapping, searcher_alg, "searcher", as_class=True )( @@ -422,5 +429,8 @@ def _run_args( budget=max_cost_total, # TODO: use max_cost_total everywhere **searcher_config, ) - - return searcher, searcher_instance, searcher_alg, searcher_config, searcher_info, user_defined_searcher + + return ( + searcher_instance, + searcher_info, + ) diff --git a/neps/metahyper/__init__.py b/neps/metahyper/__init__.py new file mode 100644 index 00000000..6e2aa0f7 --- /dev/null +++ b/neps/metahyper/__init__.py @@ -0,0 +1,2 @@ +from .api import ConfigResult, Sampler, read, metahyper_run +from .utils import instance_from_map diff --git a/src/metahyper/_locker.py b/neps/metahyper/_locker.py similarity index 100% rename from src/metahyper/_locker.py rename to neps/metahyper/_locker.py diff --git a/src/metahyper/api.py b/neps/metahyper/api.py similarity index 96% rename from src/metahyper/api.py rename to neps/metahyper/api.py index abcd974e..a868e6c2 100644 --- a/src/metahyper/api.py +++ b/neps/metahyper/api.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import inspect import logging import shutil @@ -10,7 +11,7 @@ from copy import deepcopy from dataclasses import dataclass from pathlib import Path -from typing import Any, List +from typing import Any, Iterable from ._locker import Locker from .utils import YamlSerializer, find_files, non_empty_file @@ -124,17 +125,14 @@ def _process_sampler_info( thread safety, preventing potential conflicts when multiple threads access the file simultaneously. - Parameters: + Args: - serializer: The YAML serializer object used for loading and dumping data. - sampler_info: The dictionary containing information for the optimizer. - sampler_info_file: The path to the YAML file storing optimizer data if available. - decision_locker: The Locker file to use during multi-thread communication. - logger: An optional logger object for logging messages (default is 'neps'). - Note: - The file-locking mechanism is employed to avoid potential errors in multiple threads. - - The `Locker` class and `YamlSerializer` should be appropriately defined or imported. - - Ensure that potential side effects or dependencies are considered when using this function. """ if logger is None: logger = logging.getLogger("neps") @@ -159,6 +157,9 @@ def _process_sampler_info( else: # If the file is empty or doesn't exist, write the sampler_info serializer.dump(sampler_info, sampler_info_file, sort_keys=False) + except ValueError as ve: + # Handle specific value error + raise ve except Exception as e: raise RuntimeError(f"Error during data saving: {e}") from e finally: @@ -268,9 +269,10 @@ def _check_max_evaluations( max_evaluations, serializer, logger, - continue_until_max_evaluation_completed, + continue_until_max_evaluation_completed ): logger.debug("Checking if max evaluations is reached") + #TODO: maybe not read everything again? previous_results, pending_configs, pending_configs_free = read( optimization_dir, serializer, logger ) @@ -290,16 +292,19 @@ def _sample_config(optimization_dir, sampler, serializer, logger, pre_load_hooks ) base_result_directory = optimization_dir / "results" + logger.debug(f"Previous results: {previous_results}") + logger.debug(f"Pending configs: {pending_configs}") + logger.debug(f"Pending configs: {pending_configs_free}") logger.debug("Sampling a new configuration") - + for hook in pre_load_hooks: # executes operations on the sampler before setting its state # can be used for setting custom constraints on the optimizer state - # for example, can be used to input custom grid of configs, meta learning + # for example, can be used to input custom grid of configs, meta learning # information for surrogate building, any non-stationary auxiliary information sampler = hook(sampler) - + sampler.load_results(previous_results, pending_configs) config, config_id, previous_config_id = sampler.get_config_and_ids() @@ -313,6 +318,9 @@ def _sample_config(optimization_dir, sampler, serializer, logger, pre_load_hooks "communication, but most likely some configs crashed during their execution " "or a jobtime-limit was reached." ) + # write some extra data per configuration if the optimizer has any + # if hasattr(sampler, "evaluation_data"): + # sampler.evaluation_data.write_all(pipeline_directory) if previous_config_id is not None: previous_config_id_file = pipeline_directory / "previous_config.id" @@ -413,7 +421,7 @@ def _evaluate_config( return result, {"time_end": time.time()} -def run( +def metahyper_run( evaluation_fn, sampler: Sampler, sampler_info: dict, @@ -421,22 +429,15 @@ def run( max_evaluations_total=None, max_evaluations_per_run=None, continue_until_max_evaluation_completed=False, - development_stage_id=None, - task_id=None, logger=None, post_evaluation_hook=None, overwrite_optimization_dir=False, - pre_load_hooks: List=[], + pre_load_hooks: Iterable | None = None, ): serializer = YamlSerializer(sampler.load_config) if logger is None: logger = logging.getLogger("metahyper") - if task_id is not None: - optimization_dir = Path(optimization_dir) / f"task_{task_id}" - if development_stage_id is not None: - optimization_dir = Path(optimization_dir) / f"dev_{development_stage_id}" - optimization_dir = Path(optimization_dir) if overwrite_optimization_dir and optimization_dir.exists(): logger.warning("Overwriting working_directory") @@ -451,11 +452,13 @@ def run( decision_lock_file.touch(exist_ok=True) decision_locker = Locker(decision_lock_file, logger.getChild("_locker")) + # Configuring the .optimizer_info.yaml file _process_sampler_info( serializer, sampler_info, sampler_info_file, decision_locker, logger ) evaluations_in_this_run = 0 + while True: if max_evaluations_total is not None and _check_max_evaluations( optimization_dir, diff --git a/src/metahyper/example.py b/neps/metahyper/example.py similarity index 100% rename from src/metahyper/example.py rename to neps/metahyper/example.py diff --git a/src/metahyper/utils.py b/neps/metahyper/utils.py similarity index 88% rename from src/metahyper/utils.py rename to neps/metahyper/utils.py index 72fac45f..fede6adc 100644 --- a/src/metahyper/utils.py +++ b/neps/metahyper/utils.py @@ -48,6 +48,21 @@ def get_data_representation(data: Any): return data +class MissingDependencyError(Exception): + def __init__(self, dep: str, cause: Exception, *args: Any): + super().__init__(dep, cause, *args) + self.dep = dep + self.__cause__ = cause # This is what `raise a from b` does + + def __str__(self) -> str: + return ( + f"Some required dependency-({self.dep}) to use this optional feature is " + f"missing. Please, include neps[experimental] dependency group in your " + f"installation of neps to be able to use all the optional features." + f" Otherwise, just install ({self.dep})" + ) + + class YamlSerializer: SUFFIX = ".yaml" PRE_SERIALIZE = True @@ -97,7 +112,7 @@ def instance_from_map( name: str = "mapping", allow_any: bool = True, as_class: bool = False, - kwargs: dict = None, + kwargs: dict | None = None, ): """Get an instance of an class from a mapping. @@ -140,6 +155,9 @@ def instance_from_map( else: raise ValueError(f"Object {request} invalid key for {name}") + if isinstance(instance, MissingDependencyError): + raise instance + # Check if the request is a class if it is mandatory if (args_dict or as_class) and not is_partial_class(instance): raise ValueError( diff --git a/src/neps/optimizers/__init__.py b/neps/optimizers/__init__.py similarity index 100% rename from src/neps/optimizers/__init__.py rename to neps/optimizers/__init__.py diff --git a/src/neps/optimizers/base_optimizer.py b/neps/optimizers/base_optimizer.py similarity index 95% rename from src/neps/optimizers/base_optimizer.py rename to neps/optimizers/base_optimizer.py index 83381c59..33bd1435 100644 --- a/src/neps/optimizers/base_optimizer.py +++ b/neps/optimizers/base_optimizer.py @@ -5,15 +5,13 @@ from copy import deepcopy from typing import Any -import metahyper -from metahyper.api import ConfigResult - +from ..metahyper.api import ConfigResult, Sampler from ..search_spaces.search_space import SearchSpace from ..utils.common import get_rnd_state, set_rnd_state from ..utils.result_utils import get_cost, get_learning_curve, get_loss -class BaseOptimizer(metahyper.Sampler): +class BaseOptimizer(Sampler): """Base sampler class. Implements all the low-level work.""" def __init__( @@ -62,7 +60,7 @@ def load_state(self, state: Any): # pylint: disable=no-self-use super().load_state(state) def load_config(self, config_dict): - config = deepcopy(self.pipeline_space) + config = self.pipeline_space.copy() config.load_from(config_dict) return config diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py b/neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py new file mode 100644 index 00000000..45380755 --- /dev/null +++ b/neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from functools import partial +from typing import Callable + +from .ei import ComprehensiveExpectedImprovement +from .mf_ei import MFEI, MFEI_AtMax, MFEI_Dyna, MFEI_Random +from .mf_pi import MFPI, MFPI_AtMax, MFPI_Dyna, MFPI_Random, MFPI_Random_HiT +from .ucb import UpperConfidenceBound +from .mf_ucb import MF_UCB, MF_UCB_AtMax, MF_UCB_Dyna +from .mf_two_step import MF_TwoStep + + +AcquisitionMapping: dict[str, Callable] = { + "EI": partial( + ComprehensiveExpectedImprovement, + in_fill="best", + augmented_ei=False, + ), + "LogEI": partial( + ComprehensiveExpectedImprovement, + in_fill="best", + augmented_ei=False, + log_ei=True, + ), + # # Uses the augmented EI heuristic and changed the in-fill criterion to the best test location with + # # the highest *posterior mean*, which are preferred when the optimisation is noisy. + "AEI": partial( + ComprehensiveExpectedImprovement, + in_fill="posterior", + augmented_ei=True, + ), + "MFEI": partial( + MFEI, + in_fill="best", + augmented_ei=False, + ), + "MFEI-max": partial( + MFEI_AtMax, + in_fill="best", + augmented_ei=False, + ), + "MFEI-dyna": partial( + MFEI_Dyna, + in_fill="best", + augmented_ei=False, + ), + "MFEI-random": partial( + MFPI_Random, # code has been modified, rerun and use "MFEI-random2"! + in_fill="best", + augmented_ei=False, + ), + "MFEI-random2": partial( + MFEI_Random, + in_fill="best", + augmented_ei=False, + ), + "UCB": partial( + UpperConfidenceBound, + maximize=False, + ), + "MF-UCB": partial( + MF_UCB, + maximize=False, + ), + "MF-UCB-max": partial( + MF_UCB_AtMax, + maximize=False, + ), + "MF-UCB-dyna": partial( + MF_UCB_Dyna, + maximize=False, + ), + "MF_TwoStep": partial( + MF_TwoStep, + maximize=False, + ), + "MFPI": partial( + MFPI, + in_fill="best", + augmented_ei=False, + ), + "MFPI-max": partial( + MFPI_AtMax, + in_fill="best", + augmented_ei=False, + ), + "MFPI-thresh-max": partial( + MFPI_Random, + in_fill="best", + augmented_ei=False, + horizon="max", + threshold="random", + ), + "MFPI-random-horizon": partial( + MFPI_Random, + in_fill="best", + augmented_ei=False, + horizon="random", + threshold="0.0", + ), + "MFPI-dyna": partial( + MFPI_Dyna, + in_fill="best", + augmented_ei=False, + ), + "MFPI-random": partial( + MFPI_Random, + in_fill="best", + augmented_ei=False, + ), + "MFPI-random-hit": partial( + MFPI_Random_HiT, + in_fill="best", + augmented_ei=False, + ) +} diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_functions/_ehvi.py b/neps/optimizers/bayesian_optimization/acquisition_functions/_ehvi.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/acquisition_functions/_ehvi.py rename to neps/optimizers/bayesian_optimization/acquisition_functions/_ehvi.py diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_functions/base_acquisition.py b/neps/optimizers/bayesian_optimization/acquisition_functions/base_acquisition.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/acquisition_functions/base_acquisition.py rename to neps/optimizers/bayesian_optimization/acquisition_functions/base_acquisition.py diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_functions/cost_cooling.py b/neps/optimizers/bayesian_optimization/acquisition_functions/cost_cooling.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/acquisition_functions/cost_cooling.py rename to neps/optimizers/bayesian_optimization/acquisition_functions/cost_cooling.py diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_functions/ei.py b/neps/optimizers/bayesian_optimization/acquisition_functions/ei.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/acquisition_functions/ei.py rename to neps/optimizers/bayesian_optimization/acquisition_functions/ei.py diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py b/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py new file mode 100644 index 00000000..5b059db0 --- /dev/null +++ b/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py @@ -0,0 +1,430 @@ +# type: ignore +from typing import Any, Iterable, Tuple, Union + +import numpy as np +import pandas as pd +import torch +from torch.distributions import Normal + +from ....optimizers.utils import map_real_hyperparameters_from_tabular_ids +from ....search_spaces.search_space import IntegerParameter, SearchSpace +from ...multi_fidelity.utils import MFObservedData +from .base_acquisition import BaseAcquisition +from .ei import ComprehensiveExpectedImprovement + + +class MFStepBase(BaseAcquisition): + """A class holding common operations that can be inherited. + + WARNING: Unsafe use of self attributes, can break if not used correctly. + """ + def set_state( + self, + pipeline_space: SearchSpace, + surrogate_model: Any, + observations: MFObservedData, + b_step: Union[int, float], + **kwargs, + ): + # overload to select incumbent differently through observations + self.pipeline_space = pipeline_space + self.surrogate_model = surrogate_model + self.observations = observations + self.b_step = b_step + return + + def get_budget_level(self, config) -> int: + return int((config.fidelity.value - config.fidelity.lower) / self.b_step) + + + def preprocess_gp(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + x, inc_list = self.preprocess(x) + return x, inc_list + + def preprocess_deep_gp(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + x, inc_list = self.preprocess(x) + x_lcs = [] + for idx in x.index: + if idx in self.observations.df.index.levels[0]: + # TODO: Samir, check if `budget_id=None` is okay? + # budget_level = self.get_budget_level(x[idx]) + # extracting the available/observed learning curve + lc = self.observations.extract_learning_curve(idx, budget_id=None) + else: + # initialize a learning curve with a placeholder + # This is later padded accordingly for the Conv1D layer + lc = [] + x_lcs.append(lc) + self.surrogate_model.set_prediction_learning_curves(x_lcs) + return x, inc_list + + def preprocess_pfn(self, x: pd.Series) -> Tuple[torch.Tensor, pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point, as + required by the multi-fidelity Expected Improvement acquisition function. + """ + _x, inc_list = self.preprocess(x.copy()) + _x_tok = self.observations.tokenize(_x, as_tensor=True) + len_partial = len(self.observations.seen_config_ids) + z_min = x[0].fidelity.lower + z_max = x[0].fidelity.upper + # converting fidelity to the discrete budget level + # STRICT ASSUMPTION: fidelity is the second dimension + _x_tok[:len_partial, 1] = ( + _x_tok[:len_partial, 1] + self.b_step - z_min + ) / self.b_step + _x_tok[:, 1] = _x_tok[:, 1] / z_max + return _x, _x_tok, inc_list + + +# NOTE: the order of inheritance is important +class MFEI(MFStepBase, ComprehensiveExpectedImprovement): + def __init__( + self, + pipeline_space: SearchSpace, + surrogate_model_name: str = None, + augmented_ei: bool = False, + xi: float = 0.0, + in_fill: str = "best", + inc_normalization: bool = False, + log_ei: bool = False, + ): + super().__init__(augmented_ei, xi, in_fill, log_ei) + self.pipeline_space = pipeline_space + self.surrogate_model_name = surrogate_model_name + self.inc_normalization = inc_normalization + self.surrogate_model = None + self.observations = None + self.b_step = None + + def preprocess_inc_list(self, **kwargs) -> list: + assert "budget_list" in kwargs, "Requires a list of query step for candidate set." + budget_list = kwargs["budget_list"] + performances = self.observations.get_best_performance_for_each_budget() + inc_list = [] + for budget_level in budget_list: + if budget_level in performances.index: + inc = performances[budget_level] + else: + inc = self.observations.get_best_seen_performance() + inc_list.append(inc) + return inc_list + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point, as + required by the multi-fidelity Expected Improvement acquisition function. + """ + budget_list = [] + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + indices_to_drop = [] + for i, config in x.items(): + target_fidelity = config.fidelity.lower + if i <= max(self.observations.seen_config_ids): + # IMPORTANT to set the fidelity at which EI will be calculated only for + # the partial configs that have been observed already + target_fidelity = config.fidelity.value + self.b_step + + if np.less_equal(target_fidelity, config.fidelity.upper): + # only consider the configs with fidelity lower than the max fidelity + config.fidelity.value = target_fidelity + budget_list.append(self.get_budget_level(config)) + else: + # if the target_fidelity higher than the max drop the configuration + indices_to_drop.append(i) + else: + config.fidelity.value = target_fidelity + budget_list.append(self.get_budget_level(config)) + + # Drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + # Collecting incumbent list per configuration + inc_list = self.preprocess_inc_list(budget_list=budget_list) + + return x, torch.Tensor(inc_list) + + def eval(self, x: pd.Series, asscalar: bool = False) -> Tuple[np.ndarray, pd.Series]: + # deepcopy + _x = pd.Series([x.loc[idx].copy() for idx in x.index.values], index=x.index) + if self.surrogate_model_name == "pfn": + _x, _x_tok, inc_list = self.preprocess_pfn( + x.copy() + ) # IMPORTANT change from vanilla-EI + ei = self.eval_pfn_ei(_x_tok, inc_list) + elif self.surrogate_model_name in ["deep_gp", "dpl"]: + _x, inc_list = self.preprocess_deep_gp( + _x + ) # IMPORTANT change from vanilla-EI + ei = self.eval_gp_ei(_x.values.tolist(), inc_list) + elif self.surrogate_model_name == "gp": + _x, inc_list = self.preprocess_gp( + _x + ) # IMPORTANT change from vanilla-EI + ei = self.eval_gp_ei(_x.values.tolist(), inc_list) + else: + raise ValueError( + f"Unrecognized surrogate model name: {self.surrogate_model_name}" + ) + + if self.inc_normalization: + ei = ei / inc_list + + if ei.is_cuda: + ei = ei.cpu() + if len(_x) > 1 and asscalar: + return ei.detach().numpy(), _x + else: + return ei.detach().numpy().item(), _x + + def eval_pfn_ei( + self, x: Iterable, inc_list: Iterable + ) -> Union[np.ndarray, torch.Tensor, float]: + """PFN-EI modified to preprocess samples and accept list of incumbents.""" + ei = self.surrogate_model.get_ei(x.to(self.surrogate_model.device), inc_list) + if len(ei.shape) == 2: + ei = ei.flatten() + return ei + + def eval_gp_ei( + self, x: Iterable, inc_list: Iterable + ) -> Union[np.ndarray, torch.Tensor, float]: + """Vanilla-EI modified to preprocess samples and accept list of incumbents.""" + _x = x.copy() + try: + mu, cov = self.surrogate_model.predict(_x) + except ValueError as e: + raise e + # return -1.0 # in case of error. return ei of -1 + std = torch.sqrt(torch.diag(cov)) + + mu_star = inc_list.to(mu.device) # IMPORTANT change from vanilla-EI + + gauss = Normal(torch.zeros(1, device=mu.device), torch.ones(1, device=mu.device)) + # u = (mu - mu_star - self.xi) / std + # ei = std * updf + (mu - mu_star - self.xi) * ucdf + if self.log_ei: + # we expect that f_min is in log-space + f_min = mu_star - self.xi + v = (f_min - mu) / std + ei = torch.exp(f_min) * gauss.cdf(v) - torch.exp( + 0.5 * torch.diag(cov) + mu + ) * gauss.cdf(v - std) + else: + u = (mu_star - mu - self.xi) / std + ucdf = gauss.cdf(u) + updf = torch.exp(gauss.log_prob(u)) + ei = std * updf + (mu_star - mu - self.xi) * ucdf + # Clip ei if std == 0.0 + # ei = torch.where(torch.isclose(std, torch.tensor(0.0)), 0, ei) + if self.augmented_ei: + sigma_n = self.surrogate_model.likelihood + ei *= 1.0 - torch.sqrt(torch.tensor(sigma_n, device=mu.device)) / torch.sqrt( + sigma_n + torch.diag(cov) + ) + + # Save data for writing + self.mu_star = mu_star.detach().numpy().tolist() + self.mu = mu.detach().numpy().tolist() + self.std = std.detach().numpy().tolist() + return ei + + +class MFEI_AtMax(MFEI): + + def preprocess_inc_list(self, **kwargs) -> list: + assert "len_x" in kwargs, "Requires the length of the candidate set." + len_x = kwargs["len_x"] + # finds global incumbent + inc_value = min(self.observations.get_best_performance_for_each_budget()) + # uses the best seen value as the incumbent in EI computation for all candidates + inc_list = [inc_value] * len_x + return inc_list + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point. + Unlike the base class MFEI, sets the target fidelity to be max budget and the + incumbent choice to be the max seen across history for all candidates. + """ + budget_list = [] + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + + indices_to_drop = [] + for i, config in x.items(): + target_fidelity = config.fidelity.upper # change from MFEI + + if config.fidelity.value == target_fidelity: + # if the target_fidelity already reached, drop the configuration + indices_to_drop.append(i) + else: + config.fidelity.value = target_fidelity + budget_list.append(self.get_budget_level(config)) + + # drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + # create the same incumbent for all candidates + inc_list = self.preprocess_inc_list(len_x=len(x.index.values)) + + return x, torch.Tensor(inc_list) + + +class MFEI_Dyna(MFEI_AtMax): + """ + Computes extrapolation length of curves to maximum fidelity seen. + Uses the global incumbent as the best score in EI computation. + """ + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point. + Unlike the base class MFEI, sets the target fidelity to be max budget and the + incumbent choice to be the max seen across history for all candidates. + """ + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + + # find the maximum observed steps per config to obtain the current pseudo_z_max + max_z_level_per_x = self.observations.get_max_observed_fidelity_level_per_config() + pseudo_z_level_max = max_z_level_per_x.max() # highest seen fidelity step so far + # find the fidelity step at which the best seen performance was recorded + z_inc_level = self.observations.get_budget_level_for_best_performance() + # retrieving actual fidelity values from budget level + ## marker 1: the fidelity value at which the best seen performance was recorded + z_inc = self.b_step * z_inc_level + self.pipeline_space.fidelity.lower + ## marker 2: the maximum fidelity value recorded in observation history + pseudo_z_max = self.b_step * pseudo_z_level_max + self.pipeline_space.fidelity.lower + + # TODO: compare with this first draft logic + # def update_fidelity(config): + # ### DO NOT DELETE THIS FUNCTION YET + # # for all configs, set the min(max(current fidelity + step, z_inc), pseudo_z_max) + # ## that is, choose the next highest marker from 1 and 2 + # z_extrapolate = min( + # max(config.fidelity.value + self.b_step, z_inc), + # pseudo_z_max + # ) + # config.fidelity.value = z_extrapolate + # return config + + def update_fidelity(config): + # for all configs, set to pseudo_z_max + ## that is, choose the highest seen fidelity in observation history + z_extrapolate = pseudo_z_max + config.fidelity.value = z_extrapolate + return config + + # collect IDs for partial configurations + _partial_config_ids = (x.index <= max(self.observations.seen_config_ids)) + # filter for configurations that reached max budget + indices_to_drop = [ + _idx + for _idx, _x in x.loc[_partial_config_ids].items() + if _x.fidelity.value == self.pipeline_space.fidelity.upper + ] + # drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + # set fidelity for all partial configs + x = x.apply(update_fidelity) + + # create the same incumbent for all candidates + inc_list = self.preprocess_inc_list(len_x=len(x.index.values)) + + return x, torch.Tensor(inc_list) + + +class MFEI_Random(MFEI): + + BUDGET = 1000 + + def set_state( + self, + pipeline_space: SearchSpace, + surrogate_model: Any, + observations: MFObservedData, + b_step: Union[int, float], + **kwargs, + ): + # set RNG + self.rng = np.random.RandomState(seed=42) + for i in range(len(observations.completed_runs)): + self.rng.uniform(-4,-1) + self.rng.randint(1,51) + + return super().set_state(pipeline_space, surrogate_model, observations, b_step) + + def sample_horizon(self, steps_passed): + shortest = self.pipeline_space.fidelity.lower + longest = min(self.pipeline_space.fidelity.upper, self.BUDGET - steps_passed) + return self.rng.randint(shortest, longest+1) + + def sample_threshold(self, f_inc): + lu = 10**self.rng.uniform(-4,-1) # % of gap closed + return f_inc * (1 - lu) + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point, as + required by the multi-fidelity Expected Improvement acquisition function. + """ + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + + + indices_to_drop = [] + inc_list = [] + + steps_passed = len(self.observations.completed_runs) + print(f"Steps acquired: {steps_passed}") + + # Like EI-AtMax, use the global incumbent as a basis for the EI threshold + inc_value = min(self.observations.get_best_performance_for_each_budget()) + # Extension: Add a random min improvement threshold to encourage high risk high gain + inc_value = self.sample_threshold(inc_value) + print(f"Threshold for EI: {inc_value}") + + # Like MFEI: set fidelities to query using horizon as self.b_step + # Extension: Unlike DyHPO, we sample the horizon randomly over the full range + horizon = self.sample_horizon(steps_passed) + print(f"Horizon for EI: {horizon}") + for i, config in x.items(): + if i <= max(self.observations.seen_config_ids): + current_fidelity = config.fidelity.value + if np.equal(config.fidelity.value, config.fidelity.upper): + # this training run has ended, drop it from future selection + indices_to_drop.append(i) + else: + # a candidate partial training run to continue + target_fidelity = config.fidelity.value + horizon + config.fidelity.value = min(config.fidelity.value + horizon, config.fidelity.upper) # if horizon exceeds max, query at max + inc_list.append(inc_value) + else: + # a candidate new training run that we would need to start + current_fidelity = 0 + config.fidelity.value = horizon + inc_list.append(inc_value) + #print(f"- {x.index.values[i]}: {current_fidelity} --> {config.fidelity.value}") + + # Drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + assert len(inc_list) == len(x) + + return x, torch.Tensor(inc_list) diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/mf_pi.py b/neps/optimizers/bayesian_optimization/acquisition_functions/mf_pi.py new file mode 100644 index 00000000..3f02bbc5 --- /dev/null +++ b/neps/optimizers/bayesian_optimization/acquisition_functions/mf_pi.py @@ -0,0 +1,447 @@ +# type: ignore +from pathlib import Path +from typing import Any, Iterable, Tuple, Union + +import numpy as np +import pandas as pd +import torch +from torch.distributions import Normal + +from copy import deepcopy + +from ....optimizers.utils import map_real_hyperparameters_from_tabular_ids +from ....search_spaces.search_space import IntegerParameter, SearchSpace +from ...multi_fidelity.utils import MFObservedData +from .base_acquisition import BaseAcquisition +from .ei import ComprehensiveExpectedImprovement +from ....utils.common import SimpleCSVWriter +from .mf_ei import MFStepBase + +# NOTE: the order of inheritance is important +class MFPI(MFStepBase, ComprehensiveExpectedImprovement): + def __init__( + self, + pipeline_space: SearchSpace, + surrogate_model_name: str = None, + augmented_ei: bool = False, + xi: float = 0.0, + in_fill: str = "best", + log_ei: bool = False, + ): + super().__init__(augmented_ei, xi, in_fill, log_ei) + self.pipeline_space = pipeline_space + self.surrogate_model_name = surrogate_model_name + self.surrogate_model = None + self.observations = None + self.b_step = None + + def preprocess_inc_list(self, **kwargs) -> list: + assert "budget_list" in kwargs, "Requires a list of query step for candidate set." + budget_list = kwargs["budget_list"] + performances = self.observations.get_best_performance_for_each_budget() + inc_list = [] + for budget_level in budget_list: + if budget_level in performances.index: + inc = performances[budget_level] + else: + inc = self.observations.get_best_seen_performance() + inc_list.append(inc) + return inc_list + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point, as + required by the multi-fidelity Expected Improvement acquisition function. + """ + budget_list = [] + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + indices_to_drop = [] + for i, config in x.items(): + target_fidelity = config.fidelity.lower + if i <= max(self.observations.seen_config_ids): + # IMPORTANT to set the fidelity at which EI will be calculated only for + # the partial configs that have been observed already + target_fidelity = config.fidelity.value + self.b_step + + if np.less_equal(target_fidelity, config.fidelity.upper): + # only consider the configs with fidelity lower than the max fidelity + config.fidelity.value = target_fidelity + budget_list.append(self.get_budget_level(config)) + else: + # if the target_fidelity higher than the max drop the configuration + indices_to_drop.append(i) + else: + config.fidelity.value = target_fidelity + budget_list.append(self.get_budget_level(config)) + + # Drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + # Collecting incumbent list per configuration + inc_list = self.preprocess_inc_list(budget_list=budget_list) + + return x, torch.Tensor(inc_list) + + def eval(self, x: pd.Series, asscalar: bool = False) -> Tuple[np.ndarray, pd.Series]: + # deepcopy + _x = pd.Series([x.loc[idx].copy() for idx in x.index.values], index=x.index) + if self.surrogate_model_name == "pfn": + _x, _x_tok, inc_list = self.preprocess_pfn( + x.copy() + ) # IMPORTANT change from vanilla-EI + pi = self.eval_pfn_pi(_x_tok, inc_list) + elif self.surrogate_model_name in ["deep_gp", "dpl"]: + _x, inc_list = self.preprocess_deep_gp( + _x + ) # IMPORTANT change from vanilla-EI + pi = self.eval_gp_pi(_x.values.tolist(), inc_list) + elif self.surrogate_model_name == "gp": + _x, inc_list = self.preprocess_gp( + _x + ) # IMPORTANT change from vanilla-EI + pi = self.eval_gp_pi(_x.values.tolist(), inc_list) + else: + raise ValueError( + f"Unrecognized surrogate model name: {self.surrogate_model_name}" + ) + + if pi.is_cuda: + pi = ei.cpu() + if len(_x) > 1 and asscalar: + return pi.detach().numpy(), _x + else: + return pi.detach().numpy().item(), _x + + def eval_pfn_pi( + self, x: Iterable, inc_list: Iterable + ) -> Union[np.ndarray, torch.Tensor, float]: + """PFN-PI modified to preprocess samples and accept list of incumbents.""" + pi = self.surrogate_model.get_pi(x.to(self.surrogate_model.device), inc_list) + if len(pi.shape) == 2: + pi = pi.flatten() + print(f"Maximum PI: {pi.max()}") + return pi + + def eval_gp_pi( + self, x: Iterable, inc_list: Iterable + ) -> Union[np.ndarray, torch.Tensor, float]: + _x = x.copy() + try: + mu, cov = self.surrogate_model.predict(_x) + except ValueError as e: + raise e + std = torch.sqrt(torch.diag(cov)) + mu_star = inc_list.to(mu.device) + + gauss = Normal(torch.zeros(1, device=mu.device), torch.ones(1, device=mu.device)) + pi = gauss.cdf((mu - mu_star) / (std + 1E-9)) + return pi + + +class MFPI_AtMax(MFPI): + + def preprocess_inc_list(self, **kwargs) -> list: + assert "len_x" in kwargs, "Requires the length of the candidate set." + len_x = kwargs["len_x"] + # finds global incumbent + inc_value = min(self.observations.get_best_performance_for_each_budget()) + # uses the best seen value as the incumbent in EI computation for all candidates + inc_list = [inc_value] * len_x + return inc_list + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point. + Unlike the base class MFPI, sets the target fidelity to be max budget and the + incumbent choice to be the max seen across history for all candidates. + """ + budget_list = [] + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + + indices_to_drop = [] + for i, config in x.items(): + target_fidelity = config.fidelity.upper # change from MFEI + + if config.fidelity.value == target_fidelity: + # if the target_fidelity already reached, drop the configuration + indices_to_drop.append(i) + else: + config.fidelity.value = target_fidelity + budget_list.append(self.get_budget_level(config)) + + # drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + # create the same incumbent for all candidates + inc_list = self.preprocess_inc_list(len_x=len(x.index.values)) + + return x, torch.Tensor(inc_list) + + +class MFPI_Dyna(MFPI_AtMax): + """ + Computes extrapolation length of curves to maximum fidelity seen. + Uses the global incumbent as the best score in EI computation. + """ + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point. + Unlike the base class MFEI, sets the target fidelity to be max budget and the + incumbent choice to be the max seen across history for all candidates. + """ + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + + # find the maximum observed steps per config to obtain the current pseudo_z_max + max_z_level_per_x = self.observations.get_max_observed_fidelity_level_per_config() + pseudo_z_level_max = max_z_level_per_x.max() # highest seen fidelity step so far + # find the fidelity step at which the best seen performance was recorded + z_inc_level = self.observations.get_budget_level_for_best_performance() + # retrieving actual fidelity values from budget level + ## marker 1: the fidelity value at which the best seen performance was recorded + z_inc = self.b_step * z_inc_level + self.pipeline_space.fidelity.lower + ## marker 2: the maximum fidelity value recorded in observation history + pseudo_z_max = self.b_step * pseudo_z_level_max + self.pipeline_space.fidelity.lower + + # TODO: compare with this first draft logic + # def update_fidelity(config): + # ### DO NOT DELETE THIS FUNCTION YET + # # for all configs, set the min(max(current fidelity + step, z_inc), pseudo_z_max) + # ## that is, choose the next highest marker from 1 and 2 + # z_extrapolate = min( + # max(config.fidelity.value + self.b_step, z_inc), + # pseudo_z_max + # ) + # config.fidelity.value = z_extrapolate + # return config + + def update_fidelity(config): + # for all configs, set to pseudo_z_max + ## that is, choose the highest seen fidelity in observation history + z_extrapolate = pseudo_z_max + config.fidelity.value = z_extrapolate + return config + + # collect IDs for partial configurations + _partial_config_ids = (x.index <= max(self.observations.seen_config_ids)) + # filter for configurations that reached max budget + indices_to_drop = [ + _idx + for _idx, _x in x.loc[_partial_config_ids].items() + if _x.fidelity.value == self.pipeline_space.fidelity.upper + ] + # drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + # set fidelity for all partial configs + x = x.apply(update_fidelity) + + # create the same incumbent for all candidates + inc_list = self.preprocess_inc_list(len_x=len(x.index.values)) + + return x, torch.Tensor(inc_list) + + +class MFPI_Random(MFPI): + + BUDGET = 1000 + + def __init__( + self, + pipeline_space: SearchSpace, + horizon: str = "random", + threshold: str = "random", + surrogate_model_name: str = None, + augmented_ei: bool = False, + xi: float = 0.0, + in_fill: str = "best", + log_ei: bool = False, + ): + super().__init__(pipeline_space, surrogate_model_name, augmented_ei, xi, in_fill, log_ei) + self.horizon = horizon + self.threshold = threshold + + + + def set_state( + self, + pipeline_space: SearchSpace, + surrogate_model: Any, + observations: MFObservedData, + b_step: Union[int, float], + **kwargs, + ): + # set RNG + self.rng = np.random.RandomState(seed=42) + for i in range(len(observations.completed_runs)): + self.rng.uniform(-4,-1) + self.rng.randint(1,51) + + return super().set_state(pipeline_space, surrogate_model, observations, b_step) + + def sample_horizon(self, steps_passed): + if self.horizon == 'random': + shortest = self.pipeline_space.fidelity.lower + longest = min(self.pipeline_space.fidelity.upper, self.BUDGET - steps_passed) + return self.rng.randint(shortest, longest+1) + elif self.horizon == 'max': + return min(self.pipeline_space.fidelity.upper, self.BUDGET - steps_passed) + else: + return int(self.horizon) + + def sample_threshold(self, f_inc): + if self.threshold == 'random': + lu = 10**self.rng.uniform(-4,-1) # % of gap closed + else: + lu = float(self.threshold) + return f_inc * (1 - lu) + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point, as + required by the multi-fidelity Expected Improvement acquisition function. + """ + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + + + indices_to_drop = [] + inc_list = [] + + steps_passed = len(self.observations.completed_runs) + print(f"Steps acquired: {steps_passed}") + + # Like EI-AtMax, use the global incumbent as a basis for the EI threshold + inc_value = min(self.observations.get_best_performance_for_each_budget()) + # Extension: Add a random min improvement threshold to encourage high risk high gain + t_value = self.sample_threshold(inc_value) + print(f"Threshold for PI: {inc_value - t_value}") + inc_value = t_value + + # Like MFEI: set fidelities to query using horizon as self.b_step + # Extension: Unlike DyHPO, we sample the horizon randomly over the full range + horizon = self.sample_horizon(steps_passed) + print(f"Horizon for PI: {horizon}") + for i, config in x.items(): + if i <= max(self.observations.seen_config_ids): + current_fidelity = config.fidelity.value + if np.equal(config.fidelity.value, config.fidelity.upper): + # this training run has ended, drop it from future selection + indices_to_drop.append(i) + else: + # a candidate partial training run to continue + target_fidelity = config.fidelity.value + horizon + config.fidelity.value = min(config.fidelity.value + horizon, config.fidelity.upper) # if horizon exceeds max, query at max + inc_list.append(inc_value) + else: + # a candidate new training run that we would need to start + current_fidelity = 0 + config.fidelity.value = horizon + inc_list.append(inc_value) + #print(f"- {x.index.values[i]}: {current_fidelity} --> {config.fidelity.value}") + + # Drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + assert len(inc_list) == len(x) + + return x, torch.Tensor(inc_list) + + +class MFPI_Random_HiT(MFPI): + + BUDGET = 1000 + + def set_state( + self, + pipeline_space: SearchSpace, + surrogate_model: Any, + observations: MFObservedData, + b_step: Union[int, float], + **kwargs, + ): + # set RNG + self.rng = np.random.RandomState(seed=42) + for i in range(len(observations.completed_runs)): + self.rng.uniform(-4,0) + self.rng.randint(1,51) + + return super().set_state(pipeline_space, surrogate_model, observations, b_step) + + def sample_horizon(self, steps_passed): + shortest = self.pipeline_space.fidelity.lower + longest = min(self.pipeline_space.fidelity.upper, self.BUDGET - steps_passed) + return self.rng.randint(shortest, longest+1) + + def sample_threshold(self, f_inc): + lu = 10**self.rng.uniform(-4,0) # % of gap closed + return f_inc * (1 - lu) + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point, as + required by the multi-fidelity Expected Improvement acquisition function. + """ + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + + + indices_to_drop = [] + inc_list = [] + + steps_passed = len(self.observations.completed_runs) + print(f"Steps acquired: {steps_passed}") + + # Like EI-AtMax, use the global incumbent as a basis for the EI threshold + inc_value = min(self.observations.get_best_performance_for_each_budget()) + # Extension: Add a random min improvement threshold to encourage high risk high gain + t_value = self.sample_threshold(inc_value) + print(f"Threshold for EI: {inc_value - t_value}") + inc_value = t_value + + # Like MFEI: set fidelities to query using horizon as self.b_step + # Extension: Unlike DyHPO, we sample the horizon randomly over the full range + horizon = self.sample_horizon(steps_passed) + print(f"Horizon for EI: {horizon}") + for i, config in x.items(): + if i <= max(self.observations.seen_config_ids): + current_fidelity = config.fidelity.value + if np.equal(config.fidelity.value, config.fidelity.upper): + # this training run has ended, drop it from future selection + indices_to_drop.append(i) + else: + # a candidate partial training run to continue + target_fidelity = config.fidelity.value + horizon + config.fidelity.value = min(config.fidelity.value + horizon, config.fidelity.upper) # if horizon exceeds max, query at max + inc_list.append(inc_value) + else: + # a candidate new training run that we would need to start + current_fidelity = 0 + config.fidelity.value = horizon + inc_list.append(inc_value) + #print(f"- {x.index.values[i]}: {current_fidelity} --> {config.fidelity.value}") + + # Drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + assert len(inc_list) == len(x) + + return x, torch.Tensor(inc_list) diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/mf_two_step.py b/neps/optimizers/bayesian_optimization/acquisition_functions/mf_two_step.py new file mode 100644 index 00000000..3c9b3524 --- /dev/null +++ b/neps/optimizers/bayesian_optimization/acquisition_functions/mf_two_step.py @@ -0,0 +1,238 @@ +import numpy as np +import pandas as pd +from typing import Any, Tuple, Union + +from ....search_spaces.search_space import SearchSpace +from ...multi_fidelity.utils import MFObservedData +from .base_acquisition import BaseAcquisition +from .mf_ei import MFEI, MFEI_Dyna +from .mf_ucb import MF_UCB_Dyna + + +class MF_TwoStep(BaseAcquisition): + """ 2-step acquisition: employs 3 different acquisition calls. + """ + + # HYPER-PARAMETERS: Going with the Freeze-Thaw BO (Swersky et al. 2014) values + N_PARTIAL = 10 + N_NEW = 3 + + def __init__(self, + pipeline_space: SearchSpace, + surrogate_model_name: str = None, + beta: float=1.0, + maximize: bool=False, + augmented_ei: bool = False, + xi: float = 0.0, + in_fill: str = "best", + log_ei: bool = False, + ): + """Upper Confidence Bound (UCB) acquisition function. + + Args: + beta: Controls the balance between exploration and exploitation. + maximize: If True, maximize the given model, else minimize. + DEFAULT=False, assumes minimzation. + """ + super().__init__() + # Acquisition 1: For trimming down partial candidate set + self.acq_partial_filter = MFEI_Dyna_PartialFilter( # defined below + pipeline_space=pipeline_space, + surrogate_model_name=surrogate_model_name, + augmented_ei=augmented_ei, + xi=xi, + in_fill=in_fill, + log_ei=log_ei + ) + # Acquisition 2: For trimming down new candidate set + self.acq_new_filter = MFEI( + pipeline_space=pipeline_space, + surrogate_model_name=surrogate_model_name, + augmented_ei=augmented_ei, + xi=xi, + in_fill=in_fill, + log_ei=log_ei + ) + # Acquisition 3: For final selection of winners from Acquisitions 1 & 2 + self.acq_combined = MF_UCB_Dyna( + pipeline_space=pipeline_space, + surrogate_model_name=surrogate_model_name, + beta=beta, + maximize=maximize + ) + self.pipeline_space = pipeline_space + self.surrogate_model_name = surrogate_model_name + self.surrogate_model = None + self.observations = None + self.b_step = None + + def set_state( + self, + pipeline_space: SearchSpace, + surrogate_model: Any, + observations: MFObservedData, + b_step: Union[int, float], + **kwargs, + ): + # overload to select incumbent differently through observations + self.pipeline_space = pipeline_space + self.surrogate_model = surrogate_model + self.observations = observations + self.b_step = b_step + self.acq_partial_filter.set_state( + self.pipeline_space, + self.surrogate_model, + self.observations, + self.b_step + ) + self.acq_new_filter.set_state( + self.pipeline_space, + self.surrogate_model, + self.observations, + self.b_step + ) + self.acq_combined.set_state( + self.pipeline_space, + self.surrogate_model, + self.observations, + self.b_step + ) + + def eval(self, x: pd.Series, asscalar: bool = False) -> Tuple[np.ndarray, pd.Series]: + # Filter self.N_NEW among the new configuration IDs + # Filter self.N_PARTIAL among the partial configuration IDs + max_seen_id = max(self.observations.seen_config_ids) + total_seen_id = len(self.observations.seen_config_ids) + new_ids = x.index[x.index > max_seen_id].values + partial_ids = x.index[x.index <= max_seen_id].values + + # for new candidate set + acq, _samples = self.acq_new_filter.eval(x, asscalar=True) + acq = pd.Series(acq, index=_samples.index) + # drop partial configurations + acq.loc[_samples.index.values <= max_seen_id] = 0 + # NOTE: setting to 0 works as EI-based AF returns > 0 + # find configs not in top-N_NEW set as per acquisition value, to be dropped + not_top_new_idx = acq.sort_values().index[:-self.N_NEW] # len(acq) - N_NEW + # drop these configurations + acq.loc[not_top_new_idx] = 0 # to ignore in the argmax of the acquisition function + # NOTE: setting to 0 works as EI-based AF returns > 0 + # result of first round of filtering of new candidates + + acq_new_mask = pd.Series({ + idx: val for idx, val in _samples.items() if acq.loc[idx] > 0 + }) + # for partial candidate set + acq, _samples = self.acq_partial_filter.eval(x, asscalar=True) + acq = pd.Series(acq, index=_samples.index) + # weigh the acq value based on max seen for each config + acq = self._weigh_partial_acq_scores(acq=acq) + # drop new configurations + acq.loc[_samples.index.values > max_seen_id] = 0 # to ignore in the argmax of the acquisition function + # find configs not in top-N_NEW set as per acquisition value + _top_n_partial = min(self.N_PARTIAL, total_seen_id) + not_top_new_idx = acq.sort_values().index[:-_top_n_partial] # acq.argsort()[::-1][_top_n_partial:] # sorts in ascending-flips-leaves out top-N_PARTIAL + # drop these configurations + acq.loc[not_top_new_idx] = 0 # to ignore in the argmax of the acquisition function + # NOTE: setting to 0 works as EI-based AF returns > 0 + # result of first round of filtering of partial candidates + acq_partial_mask = pd.Series({ + idx: val for idx, val in _samples.items() if acq.loc[idx] > 0 + }) + + eligible_set = set( + np.concatenate([ + acq_partial_mask.index.values.tolist(), + acq_new_mask.index.values.tolist() + ]) + ) + + # for combined selection + acq, _samples = self.acq_combined.eval(x, asscalar=True) + acq = pd.Series(acq, index=_samples.index) + # applying mask from step-1 to make final selection among (N_NEW + N_PARTIAL) + mask = acq.index.isin(eligible_set) + # NOTE: setting to -np.inf works as MF-UCB here is max.(-LCB) instead of min.(LCB) + acq[~mask] = -np.inf # will be ignored in the argmax of the acquisition function + acq_combined = pd.Series({ + idx: acq.loc[idx] for idx, val in _samples.items() if acq.loc[idx] != -np.inf + }) + # NOTE: setting to -np.inf works as MF-UCB here is max.(-LCB) instead of min.(LCB) + acq_combined = acq_combined.reindex(acq.index, fill_value=-np.inf) + acq = acq_combined.values + + return acq, _samples + + def _weigh_partial_acq_scores(self, acq: pd.Series) -> pd.Series: + # find the best performance per configuration seen + inc_list_partial = self.observations.get_best_performance_per_config() + + # removing any config indicey that have not made it till here + _idx_drop = [_i for _i in inc_list_partial.index if _i not in acq.index] + inc_list_partial.drop(labels=_idx_drop, inplace=True) + + # normalize the scores based on relative best seen performance per config + _inc, _max = inc_list_partial.min(), inc_list_partial.max() + inc_list_partial = ( + (inc_list_partial - _inc) / (_max - _inc) if _inc < _max else inc_list_partial + ) + + # calculate weights per candidate + weights = pd.Series(1 - inc_list_partial, index=inc_list_partial.index) + + # scaling the acquisition score with weights + acq = acq * weights + + return acq + + +class MFEI_PartialFilter(MFEI): + """Custom redefinition of MF-EI with Dynamic extrapolation length to adjust incumbents. + """ + + def preprocess_inc_list(self, **kwargs) -> list: + # the assertion exists to forcibly check the call to the super().preprocess() + # this function overload should only affect the operation inside it + assert "budget_list" in kwargs, "Requires the length of the candidate set." + # we still need this as placeholder for the new candidate set + # in this class we only work on the partial candidate set + inc_list = super().preprocess_inc_list(budget_list=kwargs["budget_list"]) + + n_partial = len(self.observations.seen_config_ids) + + # NOTE: Here we set the incumbent for EI calculation for each config to the + # maximum it has seen, in a bid to get an expected improvement over its previous + # observed score. This could act as a filter to diverging configurations even if + # their overall score relative to the incumbent can be high. + inc_list_partial = self.observations.get_best_performance_per_config() + # updating incumbent for EI computation for the partial configs + inc_list[:n_partial] = inc_list_partial + + return inc_list + + +class MFEI_Dyna_PartialFilter(MFEI_Dyna): + """Custom redefinition of MF-EI with Dynamic extrapolation length to adjust incumbents. + """ + + def preprocess_inc_list(self, **kwargs) -> list: + # the assertion exists to forcibly check the call to the super().preprocess() + # this function overload should only affect the operation inside it + assert "len_x" in kwargs, "Requires the length of the candidate set." + # we still need this as placeholder for the new candidate set + # in this class we only work on the partial candidate set + inc_list = super().preprocess_inc_list(len_x=kwargs["len_x"]) + + n_partial = len(self.observations.seen_config_ids) + + # NOTE: Here we set the incumbent for EI calculation for each config to the + # maximum it has seen, in a bid to get an expected improvement over its previous + # observed score. This could act as a filter to diverging configurations even if + # their overall score relative to the incumbent can be high. + inc_list_partial = self.observations.get_best_performance_per_config() + + # updating incumbent for EI computation for the partial configs + inc_list[:n_partial] = inc_list_partial + + return inc_list + \ No newline at end of file diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ucb.py b/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ucb.py new file mode 100644 index 00000000..6bf1dc94 --- /dev/null +++ b/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ucb.py @@ -0,0 +1,230 @@ +from typing import Any, Iterable, Tuple, Union + +import numpy as np +import pandas as pd +import torch + +from ....optimizers.utils import map_real_hyperparameters_from_tabular_ids +from ....search_spaces.search_space import IntegerParameter, SearchSpace +from ...multi_fidelity.utils import MFObservedData +from .mf_ei import MFStepBase +from .ucb import UpperConfidenceBound + + +# NOTE: the order of inheritance is important +class MF_UCB(MFStepBase, UpperConfidenceBound): + def __init__(self, + pipeline_space: SearchSpace, + surrogate_model_name: str = None, + beta: float=1.0, + maximize: bool=False + ): + """Upper Confidence Bound (UCB) acquisition function. + + Args: + beta: Controls the balance between exploration and exploitation. + maximize: If True, maximize the given model, else minimize. + DEFAULT=False, assumes minimzation. + """ + super().__init__(beta, maximize) + self.pipeline_space = pipeline_space + self.surrogate_model_name = surrogate_model_name + self.surrogate_model = None + self.observations = None + self.b_step = None + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point, as + required by the multi-fidelity Expected Improvement acquisition function. + """ + budget_list = [] + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + + indices_to_drop = [] + betas = [] + for i, config in x.items(): + target_fidelity = config.fidelity.lower + if i <= max(self.observations.seen_config_ids): + # IMPORTANT to set the fidelity at which EI will be calculated only for + # the partial configs that have been observed already + target_fidelity = config.fidelity.value + self.b_step + if np.less_equal(target_fidelity, config.fidelity.upper): + # only consider the configs with fidelity lower than the max fidelity + config.fidelity.value = target_fidelity + budget_list.append(self.get_budget_level(config)) + # CAN ADAPT BETA PER-SAMPLE HERE + betas.append(self.beta) + else: + # if the target_fidelity higher than the max drop the configuration + indices_to_drop.append(i) + else: + config.fidelity.value = target_fidelity + budget_list.append(self.get_budget_level(config)) + # CAN ADAPT BETA PER-SAMPLE HERE + betas.append(self.beta) + + # Drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + return x, torch.Tensor(betas) + + def preprocess_gp( + self, x: pd.Series, surrogate_name: str = "gp" + ) -> Tuple[pd.Series, torch.Tensor]: + if surrogate_name == "gp": + x, inc_list = self.preprocess(x) + return x, inc_list + elif surrogate_name in ["deep_gp", "dpl"]: + x, inc_list = self.preprocess(x) + x_lcs = [] + for idx in x.index: + if idx in self.observations.df.index.levels[0]: + # extracting the available/observed learning curve + lc = self.observations.extract_learning_curve(idx, budget_id=None) + else: + # initialize a learning curve with a placeholder + # This is later padded accordingly for the Conv1D layer + lc = [] + x_lcs.append(lc) + self.surrogate_model.set_prediction_learning_curves(x_lcs) + return x, inc_list + else: + raise ValueError( + f"Unrecognized surrogate model name: {surrogate_name}" + ) + + def eval_pfn_ucb( + self, x: Iterable, beta: float=(1-.682)/2 + ) -> Union[np.ndarray, torch.Tensor, float]: + """PFN-UCB modified to preprocess samples and accept list of incumbents.""" + ucb = self.surrogate_model.get_ucb( + x_test=x.to(self.surrogate_model.device), + beta=beta # TODO: extend to have different betas for each candidates in x + ) + if len(ucb.shape) == 2: + ucb = ucb.flatten() + return ucb + + def eval(self, x: pd.Series, asscalar: bool = False) -> Tuple[np.ndarray, pd.Series]: + if self.surrogate_model_name == "pfn": + _x, _x_tok, _ = self.preprocess_pfn( + x.copy() + ) + ucb = self.eval_pfn_ucb(_x_tok) + elif self.surrogate_model_name in ["deep_gp", "gp", "dpl"]: + _x, betas = self.preprocess_gp( + x.copy(), + self.surrogate_model_name + ) + ucb = super().eval(_x.values.tolist(), betas, asscalar) + else: + raise ValueError( + f"Unrecognized surrogate model name: {self.surrogate_model_name}" + ) + + return ucb, _x + + +class MF_UCB_AtMax(MF_UCB): + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point. + Unlike the base class MFEI, sets the target fidelity to be max budget and the + incumbent choice to be the max seen across history for all candidates. + """ + budget_list = [] + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + + indices_to_drop = [] + betas = [] + for i, config in x.items(): + target_fidelity = config.fidelity.upper # change from MFEI + + if config.fidelity.value == target_fidelity: + # if the target_fidelity already reached, drop the configuration + indices_to_drop.append(i) + else: + config.fidelity.value = target_fidelity + budget_list.append(self.get_budget_level(config)) + + # CAN ADAPT BETA PER-SAMPLE HERE + betas.append(self.beta) + + # drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + return x, torch.Tensor(betas) + + +class MF_UCB_Dyna(MF_UCB): + + def preprocess(self, x: pd.Series) -> Tuple[pd.Series, torch.Tensor]: + """Prepares the configurations for appropriate EI calculation. + + Takes a set of points and computes the budget and incumbent for each point. + Unlike the base class MFEI, sets the target fidelity to be max budget and the + incumbent choice to be the max seen across history for all candidates. + """ + if self.pipeline_space.has_tabular: + # preprocess tabular space differently + # expected input: IDs pertaining to the tabular data + x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) + + # find the maximum observed steps per config to obtain the current pseudo_z_max + max_z_level_per_x = self.observations.get_max_observed_fidelity_level_per_config() + pseudo_z_level_max = max_z_level_per_x.max() # highest seen fidelity step so far + # find the fidelity step at which the best seen performance was recorded + z_inc_level = self.observations.get_budget_level_for_best_performance() + # retrieving actual fidelity values from budget level + ## marker 1: the fidelity value at which the best seen performance was recorded + z_inc = self.b_step * z_inc_level + self.pipeline_space.fidelity.lower + ## marker 2: the maximum fidelity value recorded in observation history + pseudo_z_max = self.b_step * pseudo_z_level_max + self.pipeline_space.fidelity.lower + + # TODO: compare with this first draft logic + # def update_fidelity(config): + # ### DO NOT DELETE THIS FUNCTION YET + # # for all configs, set the min(max(current fidelity + step, z_inc), pseudo_z_max) + # ## that is, choose the next highest marker from 1 and 2 + # z_extrapolate = min( + # max(config.fidelity.value + self.b_step, z_inc), + # pseudo_z_max + # ) + # config.fidelity.value = z_extrapolate + # return config + + def update_fidelity(config): + # for all configs, set to pseudo_z_max + ## that is, choose the highest seen fidelity in observation history + z_extrapolate = pseudo_z_max + config.fidelity.value = z_extrapolate + return config + + # collect IDs for partial configurations + _partial_config_ids = (x.index <= max(self.observations.seen_config_ids)) + # filter for configurations that reached max budget + indices_to_drop = [ + _idx + for _idx, _x in x.loc[_partial_config_ids].items() + if _x.fidelity.value == self.pipeline_space.fidelity.upper + ] + # drop unused configs + x.drop(labels=indices_to_drop, inplace=True) + + # set fidelity for all partial configs + x = x.apply(update_fidelity) + + # CAN ADAPT BETA PER-SAMPLE HERE + betas = [self.beta] * len(x) # TODO: have tighter order check to Pd.Series index + + return x, torch.Tensor(betas) diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_functions/prior_weighted.py b/neps/optimizers/bayesian_optimization/acquisition_functions/prior_weighted.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/acquisition_functions/prior_weighted.py rename to neps/optimizers/bayesian_optimization/acquisition_functions/prior_weighted.py diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_functions/ucb.py b/neps/optimizers/bayesian_optimization/acquisition_functions/ucb.py similarity index 72% rename from src/neps/optimizers/bayesian_optimization/acquisition_functions/ucb.py rename to neps/optimizers/bayesian_optimization/acquisition_functions/ucb.py index adf57266..beba8fa3 100644 --- a/src/neps/optimizers/bayesian_optimization/acquisition_functions/ucb.py +++ b/neps/optimizers/bayesian_optimization/acquisition_functions/ucb.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Iterable, Union import numpy as np @@ -18,7 +20,7 @@ def __init__(self, beta: float=1.0, maximize: bool=False): super().__init__() self.beta = beta # can be updated as part of the state for dynamism or a schedule self.maximize = maximize - + # to be initialized as part of the state self.surrogate_model = None @@ -30,9 +32,9 @@ def set_state(self, surrogate_model, **kwargs): self.beta = kwargs["beta"] else: self.logger.warning("Beta is a list, not updating beta value!") - + def eval( - self, x: Iterable, asscalar: bool = False + self, x: Iterable, betas: torch.Tensor | None = None, asscalar: bool = False ) -> Union[np.ndarray, torch.Tensor, float]: try: mu, cov = self.surrogate_model.predict(x) @@ -40,21 +42,13 @@ def eval( except ValueError as e: raise e sign = 1 if self.maximize else -1 # LCB is performed if minimize=True - ucb_scores = mu + sign * np.sqrt(self.beta) * std - # if LCB, minimize acquisition, or maximize -acquisition - ucb_scores = ucb_scores.detach().numpy() * sign - - return ucb_scores - - -class MF_UCB(UpperConfidenceBound): - - def preprocess(self, x: Iterable) -> Iterable: - performances = self.observations.get_best_performance_for_each_budget() - pass - - def eval( - self, x: Iterable, asscalar: bool = False - ) -> Union[np.ndarray, torch.Tensor, float]: - x = self.preprocess(x) - return self.eval(x, asscalar=asscalar) + ucb_scores = mu + sign * torch.sqrt(self.beta if betas is None else betas) * std + # if LCB, minimize acquisition, or maximize -acquisition + ucb_scores = ucb_scores * sign + + if ucb_scores.is_cuda: + ucb_scores = ucb_scores.cpu() + if len(x) > 1 and asscalar: + return ucb_scores.detach().numpy() + else: + return ucb_scores.detach().numpy().item() diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_samplers/__init__.py b/neps/optimizers/bayesian_optimization/acquisition_samplers/__init__.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/acquisition_samplers/__init__.py rename to neps/optimizers/bayesian_optimization/acquisition_samplers/__init__.py diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_samplers/base_acq_sampler.py b/neps/optimizers/bayesian_optimization/acquisition_samplers/base_acq_sampler.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/acquisition_samplers/base_acq_sampler.py rename to neps/optimizers/bayesian_optimization/acquisition_samplers/base_acq_sampler.py diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_samplers/evolution_sampler.py b/neps/optimizers/bayesian_optimization/acquisition_samplers/evolution_sampler.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/acquisition_samplers/evolution_sampler.py rename to neps/optimizers/bayesian_optimization/acquisition_samplers/evolution_sampler.py diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_samplers/freeze_thaw_sampler.py b/neps/optimizers/bayesian_optimization/acquisition_samplers/freeze_thaw_sampler.py similarity index 71% rename from src/neps/optimizers/bayesian_optimization/acquisition_samplers/freeze_thaw_sampler.py rename to neps/optimizers/bayesian_optimization/acquisition_samplers/freeze_thaw_sampler.py index abae00b1..1fdb01db 100644 --- a/src/neps/optimizers/bayesian_optimization/acquisition_samplers/freeze_thaw_sampler.py +++ b/neps/optimizers/bayesian_optimization/acquisition_samplers/freeze_thaw_sampler.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +import time from ....search_spaces.search_space import SearchSpace from ...multi_fidelity.utils import MFObservedData @@ -13,7 +14,7 @@ class FreezeThawSampler(AcquisitionSampler): - + SAMPLES_TO_DRAW = 100 # number of random samples to draw at lowest fidelity def __init__(self, **kwargs): @@ -106,60 +107,70 @@ def sample( set_new_sample_fidelity: int | float = None, ) -> list(): """Samples a new set and returns the total set of observed + new configs.""" + start = time.time() partial_configs = self.observations.get_partial_configs_at_max_seen() - new_configs = self._sample_new( - index_from=self.observations.next_config_id(), n=n, ignore_fidelity=False - ) - - def __sample_single_new_tabular(index: int): - """ - A function to use in a list comprehension to slightly speed up - the sampling process when self.SAMPLE_TO_DRAW is large - """ - config = self.pipeline_space.sample( - patience=self.patience, user_priors=False, ignore_fidelity=False - ) - config["id"].value = _new_configs[index] - config.fidelity.value = set_new_sample_fidelity - return config + # print("-" * 50) + # print(f"| freeze-thaw:get_partial_at_max_seen(): {time.time()-start:.2f}s") + # print("-" * 50) + _n = n if n is not None else self.SAMPLES_TO_DRAW if self.is_tabular: - _n = n if n is not None else self.SAMPLES_TO_DRAW + # handles tabular data such that the entire unseen set of configs from the + # table is considered to be the new set of candidates _partial_ids = {conf["id"].value for conf in partial_configs} - _all_ids = set(self.pipeline_space.custom_grid_table.index.values) + _all_ids = set(list(self.pipeline_space.custom_grid_table.keys())) # accounting for unseen configs only, samples remaining table if flag is set max_n = len(_all_ids) + 1 if self.sample_full_table else _n _n = min(max_n, len(_all_ids - _partial_ids)) - + + start = time.time() _new_configs = np.random.choice( list(_all_ids - _partial_ids), size=_n, replace=False ) - new_configs = [__sample_single_new_tabular(i) for i in range(_n)] + placeholder_config = self.pipeline_space.sample( + patience=self.patience, user_priors=False, ignore_fidelity=False + ) + _configs = [deepcopy(placeholder_config) for _id in _new_configs] + for _i, val in enumerate(_new_configs): + _configs[_i]["id"].value = val + + # print("-" * 50) + # print(f"| freeze-thaw:sample:new_configs_extraction: {time.time()-start:.2f}s") + # print("-" * 50) new_configs = pd.Series( - new_configs, + _configs, index=np.arange( - len(partial_configs), len(partial_configs) + len(new_configs) + len(partial_configs), len(partial_configs) + len(_new_configs) ), ) + else: + # handles sampling new configurations for continuous spaces + new_configs = self._sample_new( + index_from=self.observations.next_config_id(), n=_n, ignore_fidelity=False + ) + # Continuous benchmarks need to deepcopy individual configs here, + # because in contrast to tabular benchmarks + # they are not reset in every sampling step + partial_configs = pd.Series( + [deepcopy(p_config_) for idx, p_config_ in partial_configs.items()], + index=partial_configs.index + ) - elif set_new_sample_fidelity is not None: + # Updating fidelity values + start = time.time() + if set_new_sample_fidelity is not None: for config in new_configs: config.fidelity.value = set_new_sample_fidelity + # print("-" * 50) + # print(f"| freeze-thaw:sample:new_configs_set_fidelity: {time.time()-start:.2f}s") + # print("-" * 50) - # Deep copy configs for fidelity updates - partial_configs_list = [] - index_list = [] - for idx, config in partial_configs.items(): - _config = deepcopy(config) - partial_configs_list.append(_config) - index_list.append(idx) - - # We build a new series of partial configs to avoid - # incrementing fidelities multiple times due to pass-by-reference - partial_configs = pd.Series(partial_configs_list, index=index_list) - - configs = pd.concat([partial_configs, new_configs]) + start = time.time() + configs = pd.concat([deepcopy(partial_configs), new_configs]) + # print("-" * 50) + # print(f"| freeze-thaw:sample:concat_configs: {time.time()-start:.2f}s") + # print("-" * 50) return configs @@ -180,3 +191,4 @@ def set_state( and self.pipeline_space.custom_grid_table is not None ): self.is_tabular = True + self.set_sample_full_tabular(True) diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_samplers/mutation_sampler.py b/neps/optimizers/bayesian_optimization/acquisition_samplers/mutation_sampler.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/acquisition_samplers/mutation_sampler.py rename to neps/optimizers/bayesian_optimization/acquisition_samplers/mutation_sampler.py diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_samplers/random_sampler.py b/neps/optimizers/bayesian_optimization/acquisition_samplers/random_sampler.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/acquisition_samplers/random_sampler.py rename to neps/optimizers/bayesian_optimization/acquisition_samplers/random_sampler.py diff --git a/src/neps/optimizers/bayesian_optimization/cost_cooling.py b/neps/optimizers/bayesian_optimization/cost_cooling.py similarity index 99% rename from src/neps/optimizers/bayesian_optimization/cost_cooling.py rename to neps/optimizers/bayesian_optimization/cost_cooling.py index 7a565ce8..8077dced 100644 --- a/src/neps/optimizers/bayesian_optimization/cost_cooling.py +++ b/neps/optimizers/bayesian_optimization/cost_cooling.py @@ -2,8 +2,7 @@ from typing import Any -from metahyper import ConfigResult, instance_from_map - +from ...metahyper import ConfigResult, instance_from_map from ...optimizers.bayesian_optimization.acquisition_functions.cost_cooling import ( CostCooler, ) diff --git a/src/neps/optimizers/bayesian_optimization/kernels/__init__.py b/neps/optimizers/bayesian_optimization/kernels/__init__.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/__init__.py rename to neps/optimizers/bayesian_optimization/kernels/__init__.py diff --git a/src/neps/optimizers/bayesian_optimization/kernels/combine_kernels.py b/neps/optimizers/bayesian_optimization/kernels/combine_kernels.py similarity index 99% rename from src/neps/optimizers/bayesian_optimization/kernels/combine_kernels.py rename to neps/optimizers/bayesian_optimization/kernels/combine_kernels.py index 833602e2..849198ca 100644 --- a/src/neps/optimizers/bayesian_optimization/kernels/combine_kernels.py +++ b/neps/optimizers/bayesian_optimization/kernels/combine_kernels.py @@ -51,7 +51,6 @@ def fit_transform( ): N = len(configs) K = torch.zeros(N, N) if self.combined_by == "sum" else torch.ones(N, N) - gr1, x1 = extract_configs(configs) for i, k in enumerate(self.kernels): diff --git a/src/neps/optimizers/bayesian_optimization/kernels/combine_kernels_hierarchy.py b/neps/optimizers/bayesian_optimization/kernels/combine_kernels_hierarchy.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/combine_kernels_hierarchy.py rename to neps/optimizers/bayesian_optimization/kernels/combine_kernels_hierarchy.py diff --git a/src/neps/optimizers/bayesian_optimization/kernels/encoding.py b/neps/optimizers/bayesian_optimization/kernels/encoding.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/encoding.py rename to neps/optimizers/bayesian_optimization/kernels/encoding.py diff --git a/src/neps/optimizers/bayesian_optimization/kernels/get_kernels.py b/neps/optimizers/bayesian_optimization/kernels/get_kernels.py similarity index 97% rename from src/neps/optimizers/bayesian_optimization/kernels/get_kernels.py rename to neps/optimizers/bayesian_optimization/kernels/get_kernels.py index acce8d41..eb4069af 100644 --- a/src/neps/optimizers/bayesian_optimization/kernels/get_kernels.py +++ b/neps/optimizers/bayesian_optimization/kernels/get_kernels.py @@ -1,7 +1,6 @@ from __future__ import annotations -from metahyper import instance_from_map - +from ....metahyper import instance_from_map from ....search_spaces.architecture.core_graph_grammar import CoreGraphGrammar from ....search_spaces.hyperparameters.categorical import CategoricalParameter from ....search_spaces.hyperparameters.float import FloatParameter diff --git a/src/neps/optimizers/bayesian_optimization/kernels/grakel_replace/edge_histogram.py b/neps/optimizers/bayesian_optimization/kernels/grakel_replace/edge_histogram.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/grakel_replace/edge_histogram.py rename to neps/optimizers/bayesian_optimization/kernels/grakel_replace/edge_histogram.py diff --git a/src/neps/optimizers/bayesian_optimization/kernels/grakel_replace/utils.py b/neps/optimizers/bayesian_optimization/kernels/grakel_replace/utils.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/grakel_replace/utils.py rename to neps/optimizers/bayesian_optimization/kernels/grakel_replace/utils.py diff --git a/src/neps/optimizers/bayesian_optimization/kernels/grakel_replace/vertex_histogram.py b/neps/optimizers/bayesian_optimization/kernels/grakel_replace/vertex_histogram.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/grakel_replace/vertex_histogram.py rename to neps/optimizers/bayesian_optimization/kernels/grakel_replace/vertex_histogram.py diff --git a/src/neps/optimizers/bayesian_optimization/kernels/grakel_replace/weisfeiler_lehman.py b/neps/optimizers/bayesian_optimization/kernels/grakel_replace/weisfeiler_lehman.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/grakel_replace/weisfeiler_lehman.py rename to neps/optimizers/bayesian_optimization/kernels/grakel_replace/weisfeiler_lehman.py diff --git a/src/neps/optimizers/bayesian_optimization/kernels/graph_kernel.py b/neps/optimizers/bayesian_optimization/kernels/graph_kernel.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/graph_kernel.py rename to neps/optimizers/bayesian_optimization/kernels/graph_kernel.py diff --git a/src/neps/optimizers/bayesian_optimization/kernels/utils.py b/neps/optimizers/bayesian_optimization/kernels/utils.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/utils.py rename to neps/optimizers/bayesian_optimization/kernels/utils.py diff --git a/src/neps/optimizers/bayesian_optimization/kernels/vectorial_kernels.py b/neps/optimizers/bayesian_optimization/kernels/vectorial_kernels.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/vectorial_kernels.py rename to neps/optimizers/bayesian_optimization/kernels/vectorial_kernels.py diff --git a/src/neps/optimizers/bayesian_optimization/kernels/weisfilerlehman.py b/neps/optimizers/bayesian_optimization/kernels/weisfilerlehman.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/kernels/weisfilerlehman.py rename to neps/optimizers/bayesian_optimization/kernels/weisfilerlehman.py diff --git a/src/neps/optimizers/bayesian_optimization/mf_tpe.py b/neps/optimizers/bayesian_optimization/mf_tpe.py similarity index 99% rename from src/neps/optimizers/bayesian_optimization/mf_tpe.py rename to neps/optimizers/bayesian_optimization/mf_tpe.py index eeb37831..f705f9f0 100644 --- a/src/neps/optimizers/bayesian_optimization/mf_tpe.py +++ b/neps/optimizers/bayesian_optimization/mf_tpe.py @@ -9,8 +9,7 @@ from scipy.stats import spearmanr from typing_extensions import Literal -from metahyper import ConfigResult, instance_from_map - +from ...metahyper import ConfigResult, instance_from_map from ...search_spaces import ( CategoricalParameter, ConstantParameter, diff --git a/neps/optimizers/bayesian_optimization/models/DPL.py b/neps/optimizers/bayesian_optimization/models/DPL.py new file mode 100644 index 00000000..82bd78c1 --- /dev/null +++ b/neps/optimizers/bayesian_optimization/models/DPL.py @@ -0,0 +1,834 @@ +from __future__ import annotations + +from copy import deepcopy +import logging +import os +import time +from typing import List, Tuple, Any, Type + +import numpy as np + +from scipy.stats import norm +from pathlib import Path +import torch +import torch.nn as nn + +from neps.search_spaces.search_space import ( + CategoricalParameter, + FloatParameter, + IntegerParameter, + SearchSpace, +) + +from neps.optimizers.multi_fidelity.utils import MFObservedData + + +# TODO: Move to utils +def get_optimizer_losses(root_directory: Path | str) -> list[float]: + all_losses_file = root_directory / "all_losses_and_configs.txt" + + # Read all losses from the file in the order they are explored + losses = [ + float(line[6:]) + for line in all_losses_file.read_text(encoding="utf-8").splitlines() + if "Loss: " in line + ] + return losses + + +def get_best_loss(root_directory: Path | str) -> float: + root_directory = Path(root_directory) + best_loss_fiel = root_directory / "best_loss_trajectory.txt" + + # Get the best seen loss value + best_loss = float(best_loss_fiel.read_text(encoding="utf-8").splitlines()[-1].strip()) + + return best_loss + + +class ConditionedPowerLaw(nn.Module): + + def __init__( + self, + nr_initial_features=10, + nr_units=200, + nr_layers=3, + use_learning_curve: bool = True, + kernel_size: int = 3, + nr_filters: int = 4, + nr_cnn_layers: int = 2, + ): + """ + Args: + nr_initial_features: int + The number of features per example. + nr_units: int + The number of units for every layer. + nr_layers: int + The number of layers for the neural network. + use_learning_curve: bool + If the learning curve should be use in the network. + kernel_size: int + The size of the kernel that is applied in the cnn layer. + nr_filters: int + The number of filters that are used in the cnn layers. + nr_cnn_layers: int + The number of cnn layers to be used. + """ + super(ConditionedPowerLaw, self).__init__() + + self.use_learning_curve = use_learning_curve + self.kernel_size = kernel_size + self.nr_filters = nr_filters + self.nr_cnn_layers = nr_cnn_layers + + self.act_func = torch.nn.LeakyReLU() + self.last_act_func = torch.nn.GLU() + self.tan_func = torch.nn.Tanh() + self.batch_norm = torch.nn.BatchNorm1d + + layers = [] + # adding one since we concatenate the features with the budget + nr_initial_features = nr_initial_features + if self.use_learning_curve: + nr_initial_features = nr_initial_features + nr_filters + + layers.append(nn.Linear(nr_initial_features, nr_units)) + layers.append(self.act_func) + + for i in range(2, nr_layers + 1): + layers.append(nn.Linear(nr_units, nr_units)) + layers.append(self.act_func) + + last_layer = nn.Linear(nr_units, 3) + layers.append(last_layer) + + self.layers = torch.nn.Sequential(*layers) + + cnn_part = [] + if use_learning_curve: + cnn_part.append( + nn.Conv1d( + in_channels=2, + kernel_size=(self.kernel_size,), + out_channels=self.nr_filters, + ), + ) + for i in range(1, self.nr_cnn_layers): + cnn_part.append(self.act_func) + cnn_part.append( + nn.Conv1d( + in_channels=self.nr_filters, + kernel_size=(self.kernel_size,), + out_channels=self.nr_filters, + ), + ), + cnn_part.append(nn.AdaptiveAvgPool1d(1)) + + self.cnn = nn.Sequential(*cnn_part) + + def forward( + self, + x: torch.Tensor, + predict_budgets: torch.Tensor, + evaluated_budgets: torch.Tensor, + learning_curves: torch.Tensor, + ): + """ + Args: + x: torch.Tensor + The examples. + predict_budgets: torch.Tensor + The budgets for which the performance will be predicted for the + hyperparameter configurations. + evaluated_budgets: torch.Tensor + The budgets for which the hyperparameter configurations have been + evaluated so far. + learning_curves: torch.Tensor + The learning curves for the hyperparameter configurations. + """ + # print(x.shape) + # print(learning_curves.shape) + # x = torch.cat((x, torch.unsqueeze(evaluated_budgets, 1)), dim=1) + if self.use_learning_curve: + lc_features = self.cnn(learning_curves) + # print(lc_features.shape) + # revert the output from the cnn into nr_rows x nr_kernels. + lc_features = torch.squeeze(lc_features, 2) + # print(lc_features) + x = torch.cat((x, lc_features), dim=1) + # print(x.shape) + if torch.any(torch.isnan(x)): + raise ValueError("NaN values in input, the network probably diverged") + x = self.layers(x) + alphas = x[:, 0] + betas = x[:, 1] + gammas = x[:, 2] + # print(x) + output = torch.add( + alphas, + torch.mul( + self.last_act_func(torch.cat((betas, betas))), + torch.pow( + predict_budgets, + torch.mul(self.last_act_func(torch.cat((gammas, gammas))), -1) + ) + ), + ) + + return output + + +ModelClass = ConditionedPowerLaw + +MODEL_MAPPING: dict[str, Type[ModelClass]] = {"power_law": ConditionedPowerLaw} + + +class PowerLawSurrogate: + # defaults to be used for functions + # fit params + default_lr = 0.001 + default_batch_size = 64 + default_nr_epochs = 250 + default_refine_epochs = 20 + default_early_stopping = False + default_early_stopping_patience = 10 + + # init params + default_n_initial_full_trainings = 10 + default_n_models = 5 + default_model_config = dict(nr_units=128, + nr_layers=2, + use_learning_curve=False, + kernel_size=3, + nr_filters=4, + nr_cnn_layers=2) + + # fit+predict params + default_padding_type = "zero" + default_budget_normalize = True + default_use_min_budget = False + default_y_normalize = False + + + # Defined in __init__(...) + default_no_improvement_patience = ... + def __init__( + self, + pipeline_space: SearchSpace, + observed_data: MFObservedData | None = None, + logger=None, + surrogate_model_fit_args: dict | None = None, + # IMPORTANT: Checkpointing does not use file locking, + # IMPORTANT: hence, it is not suitable for multiprocessing settings + # IMPORTANT: For parallel runs lock the checkpoint file during the whole training + checkpointing: bool = False, + root_directory: Path | str | None = None, + # IMPORTANT: For parallel runs use a different checkpoint_file name for each + # IMPORTANT: surrogate. This makes sure that parallel runs don't override each + # IMPORTANT: others saved checkpoint. Although they will still have some conflicts due to + # IMPORTANT: global optimizer step tracking + checkpoint_file: Path | str = "surrogate_checkpoint.pth", + refine_epochs: int = default_refine_epochs, + n_initial_full_trainings: int = default_n_initial_full_trainings, + default_model_class: str = "power_law", + default_model_config: dict[str, Any] = default_model_config, + n_models: int = default_n_models, + model_classes: list[str] | None = None, + model_configs: list[dict[str, Any]] | None = None, + refine_batch_size: int | None = None + ): + if pipeline_space.has_tabular: + self.cover_pipeline_space = pipeline_space + self.real_pipeline_space = pipeline_space.raw_tabular_space + else: + self.cover_pipeline_space = pipeline_space + self.real_pipeline_space = pipeline_space + # self.pipeline_space = pipeline_space + + self.observed_data = observed_data + self.__preprocess_search_space(self.real_pipeline_space) + self.seeds = np.random.choice(100, n_models, replace=False) + self.model_configs = [dict( + nr_initial_features=self.input_size, **default_model_config)] * n_models if not model_configs else model_configs + self.model_classes = [MODEL_MAPPING[default_model_class]] * n_models \ + if not model_classes \ + else [MODEL_MAPPING[m_class] for m_class in model_classes] + self.device = "cpu" + self.models: list[ModelClass] = [self.__initialize_model(config, + self.model_classes[ + index], + self.device) + for index, config in + enumerate(self.model_configs)] + + self.checkpointing = checkpointing + self.refine_epochs = refine_epochs + self.refine_batch_size = refine_batch_size + self.n_initial_full_trainings = n_initial_full_trainings + self.default_no_improvement_patience = int(self.max_fidelity + + 0.2 * self.max_fidelity) + + if checkpointing: + assert ( + root_directory is not None + ), "neps root_directory must be provided for the checkpointing" + self.root_dir = Path(os.getcwd(), root_directory) + self.checkpoint_path = Path(os.getcwd(), root_directory, checkpoint_file) + + self.surrogate_model_fit_args = ( + surrogate_model_fit_args if surrogate_model_fit_args is not None else {} + ) + + if self.surrogate_model_fit_args.get("no_improvement_patience", None) is None: + # To replicate how the original DPL implementation handles the + # no_improvement_threshold + self.surrogate_model_fit_args["no_improvement_patience"] = ( + self.default_no_improvement_patience) + + self.categories_array = np.array(self.categories) + + self.best_state = None + self.prediction_learning_curves = None + + self.criterion = torch.nn.L1Loss() + + self.logger = logger or logging.getLogger("neps") + + def __preprocess_search_space(self, pipeline_space: SearchSpace): + self.categories = [] + self.categorical_hps = [] + + parameter_count = 0 + for hp_name, hp in pipeline_space.items(): + # Collect all categories in a list for the encoder + if hp.is_fidelity: + continue # Ignore fidelity + if isinstance(hp, CategoricalParameter): + self.categorical_hps.append(hp_name) + self.categories.extend(hp.choices) + parameter_count += len(hp.choices) + else: + parameter_count += 1 + + # add 1 for budget + self.input_size = parameter_count + self.continuous_params_size = self.input_size - len(self.categories) + self.min_fidelity = pipeline_space.fidelity.lower + self.max_fidelity = pipeline_space.fidelity.upper + + def __encode_config(self, config: SearchSpace) -> np.ndarray: + categorical_encoding = np.zeros_like(self.categories_array, dtype=np.single) + continuous_values = [] + + for hp_name, hp in config.items(): + if hp.is_fidelity: + continue # Ignore fidelity + if hp_name in self.categorical_hps: + label = hp.value + categorical_encoding[np.argwhere(self.categories_array == label)] = 1 + else: + continuous_values.append(hp.normalized().value) + + continuous_encoding = np.array(continuous_values) + + encoding = np.concatenate([categorical_encoding, continuous_encoding]) + return encoding + + def __normalize_budgets(self, + budgets: np.ndarray, + use_min_budget: bool) -> np.ndarray: + min_budget = self.min_fidelity if use_min_budget else 0 + normalized_budgets = (budgets - min_budget) / ( + self.max_fidelity - min_budget + ) + return normalized_budgets + + def __extract_budgets( + self, x_train: list[SearchSpace], + normalized: bool, + use_min_budget: bool + ) -> np.ndarray: + + budgets = np.array([config.fidelity.value for config in x_train], dtype=np.single) + + if normalized: + budgets = self.__normalize_budgets(budgets, use_min_budget) + return budgets + + def __preprocess_learning_curves( + self, learning_curves: list[list[float]], padding_type: str + ) -> np.ndarray: + # Add padding to the learning curves to make them the same size + existing_values_mask = [] + max_length = self.max_fidelity - 1 + + if padding_type == "last": + init_value = self.__get_mean_initial_value() + else: + init_value = 0.0 + + for lc in learning_curves: + if len(lc) == 0: + padding_value = init_value + elif padding_type == "last": + padding_value = lc[-1] + else: + padding_value = 0.0 + + padding_length = int(max_length - len(lc)) + + mask = [1] * len(lc) + [0] * padding_length + existing_values_mask.append(mask) + + lc.extend([padding_value] * padding_length) + # print(learning_curves) + learning_curves = np.array(learning_curves, dtype=np.single) + existing_values_mask = np.array(existing_values_mask, dtype=np.single) + + learning_curves = np.stack((learning_curves, existing_values_mask), axis=1) + + return learning_curves + + def __reset_xy( + self, + x_train: list[SearchSpace], + y_train: list[float], + learning_curves: list[list[float]], + normalize_y: bool = default_y_normalize, + normalize_budget: bool = default_budget_normalize, + use_min_budget: bool = default_use_min_budget, + padding_type: str = default_padding_type + ): + self.normalize_budget = ( # pylint: disable=attribute-defined-outside-init + normalize_budget + ) + self.use_min_budget = ( # pylint: disable=attribute-defined-outside-init + use_min_budget + ) + self.padding_type = ( # pylint: disable=attribute-defined-outside-init + padding_type + ) + self.normalize_y = normalize_y # pylint: disable=attribute-defined-outside-init + + x_train, train_budgets, learning_curves = self._preprocess_input( + x_train, learning_curves, self.normalize_budget, self.use_min_budget, self.padding_type + ) + + y_train = self._preprocess_y(y_train, normalize_y) + + self.x_train = x_train # pylint: disable=attribute-defined-outside-init + self.train_budgets = ( # pylint: disable=attribute-defined-outside-init + train_budgets + ) + self.learning_curves = ( # pylint: disable=attribute-defined-outside-init + learning_curves + ) + self.y_train = y_train # pylint: disable=attribute-defined-outside-init + + def _preprocess_input( + self, + x: list[SearchSpace], + learning_curves: list[list[float]], + normalize_budget: bool, + use_min_budget: bool, + padding_type: str + ) -> [torch.tensor, torch.tensor, torch.tensor]: + budgets = self.__extract_budgets(x, normalize_budget, use_min_budget) + learning_curves = self.__preprocess_learning_curves(learning_curves, padding_type) + + x = np.array([self.__encode_config(config) for config in x], dtype=np.single) + + x = torch.tensor(x).to(device=self.device) + budgets = torch.tensor(budgets).to(device=self.device) + learning_curves = torch.tensor(learning_curves).to(device=self.device) + + return x, budgets, learning_curves + + def _preprocess_y(self, y_train: list[float], + normalize_y: bool) -> torch.tensor: + y_train_array = np.array(y_train, dtype=np.single) + self.min_y = y_train_array.min() # pylint: disable=attribute-defined-outside-init + self.max_y = y_train_array.max() # pylint: disable=attribute-defined-outside-init + if normalize_y: + y_train_array = (y_train_array - self.min_y) / (self.max_y - self.min_y) + y_train_array = torch.tensor(y_train_array).to(device=self.device) + return y_train_array + + def __is_refine(self, no_improvement_patience: int) -> bool: + losses = get_optimizer_losses(self.root_dir) + + best_loss = get_best_loss(self.root_dir) + + total_optimizer_steps = len(losses) + + # Count the non-improvement + non_improvement_steps = 0 + for loss in reversed(losses): + if np.greater(loss, best_loss): + non_improvement_steps += 1 + else: + break + + self.logger.debug(f"No improvement for: {non_improvement_steps} evaulations") + + return ((non_improvement_steps < no_improvement_patience) + and (self.n_initial_full_trainings <= total_optimizer_steps)) + + def fit( + self, + x_train: list[SearchSpace], + y_train: list[float], + learning_curves: list[list[float]], + ): + self._fit(x_train, y_train, learning_curves, **self.surrogate_model_fit_args) + + def _fit(self, + x_train: list[SearchSpace], + y_train: list[float], + learning_curves: list[list[float]], + nr_epochs: int = default_nr_epochs, + batch_size: int = default_batch_size, + early_stopping: bool = default_early_stopping, + early_stopping_patience: int = default_early_stopping_patience, + no_improvement_patience: int = default_no_improvement_patience, + optimizer_args: dict[str, Any] | None = None, + + normalize_y: bool = default_y_normalize, + normalize_budget: bool = default_budget_normalize, + use_min_budget: bool = default_use_min_budget, + padding_type: str = default_padding_type): + + self.__reset_xy( + x_train, + y_train, + learning_curves, + normalize_y=normalize_y, + normalize_budget=normalize_budget, + use_min_budget=use_min_budget, + padding_type=padding_type, + ) + # check when to refine + if self.checkpointing and self.__is_refine(no_improvement_patience) and self.checkpoint_path.exists(): + # self.__initialize_model() + self.load_state() + weight_new_point = True + nr_epochs = self.refine_epochs + batch_size = self.refine_batch_size if self.refine_batch_size else batch_size + else: + weight_new_point = False + + if optimizer_args is None: + optimizer_args = {"lr": self.default_lr} + + for model_index, model in enumerate(self.models): + self._train_a_model(model_index, + self.x_train, + self.train_budgets, + self.y_train, + self.learning_curves, + nr_epochs=nr_epochs, + batch_size=batch_size, + early_stopping_patience=early_stopping_patience, + early_stopping=early_stopping, + weight_new_point=weight_new_point, + optimizer_args=optimizer_args) + + # save model after training if checkpointing + if self.checkpointing: + self.save_state() + + def _train_a_model(self, + model_index: int, + x_train: torch.tensor, + train_budgets: torch.tensor, + y_train: torch.tensor, + learning_curves: torch.tensor, + nr_epochs: int, + batch_size: int, + early_stopping_patience: int, + early_stopping: bool, + weight_new_point: bool, + optimizer_args: dict[str, Any]): + + # Setting seeds will interfere with SearchSpace random sampling + if self.cover_pipeline_space.has_tabular: + seed = self.seeds[model_index] + torch.manual_seed(seed) + np.random.seed(seed) + + model = self.models[model_index] + + optimizer = ( + torch.optim.Adam(**dict({"params": model.parameters()}, **optimizer_args)) + ) + + count_down = early_stopping_patience + best_loss = np.inf + best_state = deepcopy(model.state_dict()) + + model.train() + + if weight_new_point: + new_x, new_b, new_lc, new_y = self.prep_new_point() + else: + new_x, new_b, new_lc, new_y = [torch.tensor([])] * 4 + + for epoch in range(0, nr_epochs): + + if early_stopping and count_down == 0: + self.logger.info( + f"Epoch: {epoch - 1} surrogate training stops due to early " + f"stopping with the patience: {early_stopping_patience} and " + f"the minimum average loss of {best_loss} and " + f"the final average loss of {best_loss}" + ) + model.load_state_dict(best_state) + break + + n_examples_batch = x_train.size(dim=0) + + # get a random permutation for mini-batches + permutation = torch.randperm(n_examples_batch) + + # optimize over mini-batches + total_scaled_loss = 0.0 + for batch_idx, start_index in enumerate( + range(0, n_examples_batch, batch_size) + ): + end_index = start_index + batch_size + if end_index > n_examples_batch: + end_index = n_examples_batch + indices = permutation[start_index:end_index] + batch_x, batch_budget, batch_lc, batch_y = ( + x_train[indices], + train_budgets[indices], + learning_curves[indices], + y_train[indices], + ) + + minibatch_size = end_index - start_index + + if weight_new_point: + batch_x = torch.cat((batch_x, new_x)) + batch_budget = torch.cat((batch_budget, new_b)) + batch_lc = torch.cat((batch_lc, new_lc)) + batch_y = torch.cat((batch_y, new_y)) + + # increase the batchsize + minibatch_size += new_x.shape[0] + + # if only one example in the batch, skip the batch. + # Otherwise, the code will fail because of batchnorm + if minibatch_size <= 1: + continue + + # Zero backprop gradients + optimizer.zero_grad(set_to_none=True) + + outputs = model(batch_x, batch_budget, batch_budget, + batch_lc) + loss = self.criterion(outputs, batch_y) + loss.backward() + optimizer.step() + + total_scaled_loss += (loss.detach().item() * minibatch_size) + + running_loss = total_scaled_loss / n_examples_batch + + if running_loss < best_loss: + best_loss = running_loss + count_down = early_stopping_patience + best_state = deepcopy(model.state_dict()) + elif early_stopping: + self.logger.debug( + f"No improvement over the minimum loss value of {best_loss} " + f"for the past {early_stopping_patience - count_down} epochs " + f"the training will stop in {count_down} epochs" + ) + count_down -= 1 + if early_stopping: + model.load_state_dict(best_state) + return model + + def set_prediction_learning_curves(self, learning_curves: list[list[float]]): + # pylint: disable=attribute-defined-outside-init + self.prediction_learning_curves = learning_curves + # pylint: enable=attribute-defined-outside-init + + def predict( + self, + x: list[SearchSpace], + learning_curves: list[list[float]] | None = None, + real_budgets: list[int | float] | None = None + ) -> [torch.tensor, torch.tensor]: + # Preprocess input + # [print(_x.hp_values()) for _x in x] + if learning_curves is None: + learning_curves = self.prediction_learning_curves + + if real_budgets is None: + # Get the list of budgets the configs are evaluated for + real_budgets = [len(lc) + 1 for lc in learning_curves] + + x_test, prediction_budgets, learning_curves = self._preprocess_input( + x, learning_curves, self.normalize_budget, self.use_min_budget, self.padding_type + ) + # preprocess the list of budgets the configs are evaluated for + real_budgets = np.array(real_budgets, dtype=np.single) + real_budgets = self.__normalize_budgets(real_budgets, + self.use_min_budget) + real_budgets = torch.tensor(real_budgets).to(self.device) + + all_predictions = [] + for model in self.models: + model.eval() + + preds = model(x_test, prediction_budgets, real_budgets, learning_curves) + all_predictions.append(preds.detach().cpu().numpy()) + + means = torch.tensor(np.mean(all_predictions, axis=0)).cpu() + std_predictions = np.std(all_predictions, axis=0) + cov = torch.diag(torch.tensor(np.power(std_predictions, 2))).cpu() + + return means, cov + + def load_state(self, state: dict[str, int | str | dict[str, Any]] | None = None): + # load and save last evaluated config as well + if state is None: + checkpoint = torch.load(self.checkpoint_path) + else: + checkpoint = state + + self.last_point = checkpoint["last_point"] + + for model_index in range(checkpoint["n_models"]): + self.models[model_index].load_state_dict( + checkpoint[f"model_{model_index}_state_dict"]) + self.models[model_index].to(self.device) + + def get_state(self) -> dict[str, int | str | dict[str, Any]]: + n_models = len(self.models) + model_states = {f"model_{model_index}_state_dict": + deepcopy(self.models[model_index].state_dict()) + for model_index in range(n_models)} + + # get last point + last_point = self.get_last_point() + current_state = dict(n_models=n_models, last_point=last_point, **model_states) + + return current_state + + def __config_ids(self) -> list[str]: + # Parallelization issues + all_losses_file = self.root_dir / "all_losses_and_configs.txt" + + if all_losses_file.exists(): + # Read all losses from the file in the order they are explored + config_ids = [ + str(line[11:]) + for line in all_losses_file.read_text(encoding="utf-8").splitlines() + if "Config ID: " in line + ] + else: + config_ids = [] + + return config_ids + + def save_state(self, state: dict[str, int | str | dict[str, Any]] | None = None): + # TODO: save last evaluated config as well + if state is None: + torch.save( + self.get_state(), + self.checkpoint_path, + ) + else: + assert ("last_point" in state and + "n_models" in state), \ + "The state dictionary is not complete" + torch.save( + state, + self.checkpoint_path, + ) + + def get_last_point(self) -> str: + # Only for single worker case + last_config_id = self.__config_ids()[-1] + # For parallel runs + # get the last config_id that's also in self.observed_configs + return last_config_id + + def get_new_points(self) -> [list[SearchSpace], list[list[float]], list[float]]: + # Get points that haven't been trained on before + + config_ids = self.__config_ids() + + if self.last_point: + index = config_ids.index(self.last_point) + 1 + else: + index = len(config_ids) - 1 + + new_config_indices = [tuple(map(int, config_id.split("_"))) for config_id in + config_ids[index:]] + + # Only include the points that exist in the observed data already + # (not a use case for single worker runs) + existing_index_map = self.observed_data.df.index.isin(new_config_indices) + + new_config_df = self.observed_data.df.loc[existing_index_map, :].copy(deep=True) + + new_configs, new_lcs, new_y = self.observed_data.get_training_data_4DyHPO( + new_config_df, self.cover_pipeline_space) + + return new_configs, new_lcs, new_y + + @staticmethod + def __initialize_model(model_params: dict[str, Any], model_class: Type[ModelClass], + device: str) -> ModelClass: + model = model_class(**model_params) + model.to(device) + return model + + def prep_new_point(self) -> [torch.tensor, torch.tensor, torch.tensor, torch.tensor]: + new_point, new_lc, new_y = self.get_new_points() + + new_x, new_b, new_lc = self._preprocess_input(new_point, + new_lc, + self.normalize_budget, + self.use_min_budget, + self.padding_type + ) + new_y = self._preprocess_y(new_y, self.normalize_y) + + return new_x, new_b, new_lc, new_y + + def __get_mean_initial_value(self): + mean = self.observed_data.get_learning_curves().loc[:, 0].mean() + + return mean + + +if __name__ == "__main__": + max_fidelity = 50 + pipe_space = SearchSpace( + float_=FloatParameter(lower=0.0, upper=5.0), + e=IntegerParameter(lower=1, upper=max_fidelity, is_fidelity=True), + ) + + configs = [pipe_space.sample(ignore_fidelity=False) for _ in range(100)] + + y = np.random.random(100).tolist() + + lcs = [ + np.random.random(size=np.random.randint(low=1, high=max_fidelity)).tolist() + for _ in range(100) + ] + + surrogate = PowerLawSurrogate(pipe_space) + + surrogate.fit(x_train=configs, learning_curves=lcs, y_train=y) + + means, stds = surrogate.predict(configs, lcs) + + print(list(zip(means, y))) + print(stds) diff --git a/neps/optimizers/bayesian_optimization/models/__init__.py b/neps/optimizers/bayesian_optimization/models/__init__.py new file mode 100755 index 00000000..3f9bec50 --- /dev/null +++ b/neps/optimizers/bayesian_optimization/models/__init__.py @@ -0,0 +1,22 @@ +from ....metahyper.utils import MissingDependencyError +from .gp import ComprehensiveGP +from .gp_hierarchy import ComprehensiveGPHierarchy +from .DPL import PowerLawSurrogate + +try: + from .deepGP import DeepGP +except ImportError as e: + DeepGP = MissingDependencyError("gpytorch", e) + +try: + from .pfn import PFN_SURROGATE # only if available locally +except ImportError as e: + PFN_SURROGATE = MissingDependencyError("pfn", e) + +SurrogateModelMapping = { + "deep_gp": DeepGP, + "gp": ComprehensiveGP, + "gp_hierarchy": ComprehensiveGPHierarchy, + "dpl": PowerLawSurrogate, + "pfn": PFN_SURROGATE, +} diff --git a/src/neps/optimizers/bayesian_optimization/models/deepGP.py b/neps/optimizers/bayesian_optimization/models/deepGP.py similarity index 88% rename from src/neps/optimizers/bayesian_optimization/models/deepGP.py rename to neps/optimizers/bayesian_optimization/models/deepGP.py index ee47ce70..862f8a6e 100644 --- a/src/neps/optimizers/bayesian_optimization/models/deepGP.py +++ b/neps/optimizers/bayesian_optimization/models/deepGP.py @@ -17,12 +17,11 @@ SearchSpace, ) +def get_optimizer_losses(root_directory: Path | str) -> list[float]: -def count_non_improvement_steps(root_directory: Path | str) -> int: - root_directory = Path(root_directory) all_losses_file = root_directory / "all_losses_and_configs.txt" - best_loss_fiel = root_directory / "best_loss_trajectory.txt" + # Read all losses from the file in the order they are explored losses = [ @@ -30,18 +29,16 @@ def count_non_improvement_steps(root_directory: Path | str) -> int: for line in all_losses_file.read_text(encoding="utf-8").splitlines() if "Loss: " in line ] + return losses + +def get_best_loss(root_directory: Path | str) -> float: + root_directory = Path(root_directory) + best_loss_fiel = root_directory / "best_loss_trajectory.txt" + # Get the best seen loss value best_loss = float(best_loss_fiel.read_text(encoding="utf-8").splitlines()[-1].strip()) - # Count the non-improvement - count = 0 - for loss in reversed(losses): - if np.greater(loss, best_loss): - count += 1 - else: - break - - return count + return best_loss class NeuralFeatureExtractor(nn.Module): @@ -167,13 +164,19 @@ def __init__( # IMPORTANT: hence, it is not suitable for multiprocessing settings checkpointing: bool = False, root_directory: Path | str | None = None, + # IMPORTANT: For parallel runs use a different checkpoint_file name for each + # IMPORTANT: surrogate. This makes sure that parallel runs don't override each + # IMPORTANT: others saved checkpoint. Although they will still have some conflicts due to + # IMPORTANT: global optimizer step tracking checkpoint_file: Path | str = "surrogate_checkpoint.pth", refine_epochs: int = 50, + n_initial_full_trainings: int = 10, **kwargs, # pylint: disable=unused-argument - ): + ): self.surrogate_model_fit_args = ( surrogate_model_fit_args if surrogate_model_fit_args is not None else {} ) + self.n_initial_full_trainings = n_initial_full_trainings self.checkpointing = checkpointing self.refine_epochs = refine_epochs @@ -204,9 +207,17 @@ def __init__( neural_network_args.get("n_layers", 2) ) + if self.surrogate_model_fit_args.get("perf_patience", -1) is None: + # To replicate how the original DyHPO implementation handles the + # no_improvement_threshold + self.surrogate_model_fit_args["perf_patience"] = int(self.max_fidelity + + 0.2 * self.max_fidelity) + # build the neural network self.nn = NeuralFeatureExtractor(self.input_size, **neural_network_args) + self.best_state = None + self.logger = logger or logging.getLogger("neps") def __initialize_gp_model( @@ -237,6 +248,26 @@ def __initialize_gp_model( mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model).to(self.device) return model, likelihood, mll + def __is_refine(self, perf_patience: int): + losses = get_optimizer_losses(self.root_dir) + + best_loss = get_best_loss(self.root_dir) + + total_optimizer_steps = len(losses) + + # Count the non-improvement + non_improvement_steps = 0 + for loss in reversed(losses): + if np.greater(loss, best_loss): + non_improvement_steps += 1 + else: + break + + self.logger.debug(f"No improvement for: {non_improvement_steps} evaulations") + + return ((non_improvement_steps < perf_patience) + and (self.n_initial_full_trainings <= total_optimizer_steps)) + def __preprocess_search_space(self, pipeline_space: SearchSpace): self.categories = [] self.categorical_hps = [] @@ -258,9 +289,9 @@ def __preprocess_search_space(self, pipeline_space: SearchSpace): self.max_fidelity = pipeline_space.fidelity.upper def __encode_config(self, config: SearchSpace): - categorical_encoding = np.zeros_like(self.categories_array) + categorical_encoding = np.zeros_like(self.categories_array, dtype=np.single) continuous_values = [] - + # print(config.hp_values()) for hp_name, hp in config.items(): if hp.is_fidelity: continue # Ignore fidelity @@ -268,6 +299,7 @@ def __encode_config(self, config: SearchSpace): label = hp.value categorical_encoding[np.argwhere(self.categories_array == label)] = 1 else: + # self.logger.info(f"{hp_name} Value: {hp.value} Normalized {hp.normalized().value}") continuous_values.append(hp.normalized().value) continuous_encoding = np.array(continuous_values) @@ -276,12 +308,16 @@ def __encode_config(self, config: SearchSpace): return encoding def __extract_budgets( - self, x_train: list[SearchSpace], normalized: bool = True + self, x_train: list[SearchSpace], + normalized: bool = True, + use_min_budget: bool = False ) -> np.ndarray: + + min_budget = self.min_fidelity if use_min_budget else 0 budgets = np.array([config.fidelity.value for config in x_train], dtype=np.single) if normalized: - normalized_budgets = (budgets - self.min_fidelity) / ( - self.max_fidelity - self.min_fidelity + normalized_budgets = (budgets - min_budget) / ( + self.max_fidelity - min_budget ) budgets = normalized_budgets return budgets @@ -316,14 +352,18 @@ def __reset_xy( learning_curves: list[list[float]], normalize_y: bool = False, normalize_budget: bool = True, + use_min_budget: bool = False, ): self.normalize_budget = ( # pylint: disable=attribute-defined-outside-init normalize_budget ) + self.use_min_budget = ( # pylint: disable=attribute-defined-outside-init + use_min_budget + ) self.normalize_y = normalize_y # pylint: disable=attribute-defined-outside-init x_train, train_budgets, learning_curves = self._preprocess_input( - x_train, learning_curves, normalize_budget + x_train, learning_curves, normalize_budget, use_min_budget ) y_train = self._preprocess_y(y_train, normalize_y) @@ -342,8 +382,9 @@ def _preprocess_input( x: list[SearchSpace], learning_curves: list[list[float]], normalize_budget: bool = True, + use_min_budget: bool = False ): - budgets = self.__extract_budgets(x, normalize_budget) + budgets = self.__extract_budgets(x, normalize_budget, use_min_budget) learning_curves = self.__preprocess_learning_curves(learning_curves) x = np.array([self.__encode_config(config) for config in x], dtype=np.single) @@ -369,6 +410,7 @@ def fit( y_train: list[float], learning_curves: list[list[float]], ): + self.logger.info(f"FIT ARGS: {self.surrogate_model_fit_args}") self._fit(x_train, y_train, learning_curves, **self.surrogate_model_fit_args) def _fit( @@ -378,6 +420,7 @@ def _fit( learning_curves: list[list[float]], normalize_y: bool = False, normalize_budget: bool = True, + use_min_budget: bool = False, n_epochs: int = 1000, batch_size: int = 64, optimizer_args: dict | None = None, @@ -391,6 +434,7 @@ def _fit( learning_curves, normalize_y=normalize_y, normalize_budget=normalize_budget, + use_min_budget=use_min_budget ) self.model, self.likelihood, self.mll = self.__initialize_gp_model(len(y_train)) self.nn = NeuralFeatureExtractor(self.input_size, **self.nn_args) @@ -399,12 +443,10 @@ def _fit( self.nn.to(self.device) if self.checkpointing and self.checkpoint_path.exists(): - non_improvement_steps = count_non_improvement_steps(self.root_dir) # If checkpointing and patience is not exhausted load a partial model - if non_improvement_steps < perf_patience: + if self.__is_refine(perf_patience): n_epochs = self.refine_epochs self.load_checkpoint() - self.logger.debug(f"No improvement for: {non_improvement_steps} evaulations") self.logger.debug(f"N Epochs for the full training: {n_epochs}") initial_state = self.get_state() @@ -421,7 +463,7 @@ def _fit( patience=patience, ) if self.checkpointing: - self.save_checkpoint() + self.save_checkpoint(self.best_state) except gpytorch.utils.errors.NotPSDError: self.logger.info("Model training failed loading the untrained model") self.load_checkpoint(initial_state) @@ -467,6 +509,7 @@ def __train_model( f"the minimum average loss of {min_avg_loss_val} and " f"the final average loss of {average_loss}" ) + self.load_checkpoint(self.best_state) break n_examples_batch = x_train.size(dim=0) @@ -530,6 +573,7 @@ def __train_model( if average_loss < min_avg_loss_val: min_avg_loss_val = average_loss count_down = patience + self.best_state = self.get_state() elif early_stopping: self.logger.debug( f"No improvement over the minimum loss value of {min_avg_loss_val} " @@ -558,7 +602,7 @@ def predict( if learning_curves is None: learning_curves = self.prediction_learning_curves x_test, test_budgets, learning_curves = self._preprocess_input( - x, learning_curves, self.normalize_budget + x, learning_curves, self.normalize_budget, self.use_min_budget ) self.model.eval() diff --git a/src/neps/optimizers/bayesian_optimization/models/gp.py b/neps/optimizers/bayesian_optimization/models/gp.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/models/gp.py rename to neps/optimizers/bayesian_optimization/models/gp.py diff --git a/src/neps/optimizers/bayesian_optimization/models/gp_hierarchy.py b/neps/optimizers/bayesian_optimization/models/gp_hierarchy.py similarity index 100% rename from src/neps/optimizers/bayesian_optimization/models/gp_hierarchy.py rename to neps/optimizers/bayesian_optimization/models/gp_hierarchy.py diff --git a/src/neps/optimizers/bayesian_optimization/optimizer.py b/neps/optimizers/bayesian_optimization/optimizer.py similarity index 99% rename from src/neps/optimizers/bayesian_optimization/optimizer.py rename to neps/optimizers/bayesian_optimization/optimizer.py index e598b3df..6c47ac8b 100644 --- a/src/neps/optimizers/bayesian_optimization/optimizer.py +++ b/neps/optimizers/bayesian_optimization/optimizer.py @@ -5,8 +5,7 @@ from typing_extensions import Literal -from metahyper import ConfigResult, instance_from_map - +from ...metahyper import ConfigResult, instance_from_map from ...search_spaces.hyperparameters.categorical import ( CATEGORICAL_CONFIDENCE_SCORES, CategoricalParameter, diff --git a/src/neps/optimizers/default_searchers/asha.yaml b/neps/optimizers/default_searchers/asha.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/asha.yaml rename to neps/optimizers/default_searchers/asha.yaml diff --git a/src/neps/optimizers/default_searchers/asha_prior.yaml b/neps/optimizers/default_searchers/asha_prior.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/asha_prior.yaml rename to neps/optimizers/default_searchers/asha_prior.yaml diff --git a/src/neps/optimizers/default_searchers/bayesian_optimization.yaml b/neps/optimizers/default_searchers/bayesian_optimization.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/bayesian_optimization.yaml rename to neps/optimizers/default_searchers/bayesian_optimization.yaml diff --git a/src/neps/optimizers/default_searchers/hyperband.yaml b/neps/optimizers/default_searchers/hyperband.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/hyperband.yaml rename to neps/optimizers/default_searchers/hyperband.yaml diff --git a/src/neps/optimizers/default_searchers/mobster.yaml b/neps/optimizers/default_searchers/mobster.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/mobster.yaml rename to neps/optimizers/default_searchers/mobster.yaml diff --git a/src/neps/optimizers/default_searchers/pibo.yaml b/neps/optimizers/default_searchers/pibo.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/pibo.yaml rename to neps/optimizers/default_searchers/pibo.yaml diff --git a/src/neps/optimizers/default_searchers/priorband.yaml b/neps/optimizers/default_searchers/priorband.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/priorband.yaml rename to neps/optimizers/default_searchers/priorband.yaml diff --git a/src/neps/optimizers/default_searchers/priorband_bo.yaml b/neps/optimizers/default_searchers/priorband_bo.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/priorband_bo.yaml rename to neps/optimizers/default_searchers/priorband_bo.yaml diff --git a/src/neps/optimizers/default_searchers/random_search.yaml b/neps/optimizers/default_searchers/random_search.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/random_search.yaml rename to neps/optimizers/default_searchers/random_search.yaml diff --git a/src/neps/optimizers/default_searchers/regularized_evolution.yaml b/neps/optimizers/default_searchers/regularized_evolution.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/regularized_evolution.yaml rename to neps/optimizers/default_searchers/regularized_evolution.yaml diff --git a/src/neps/optimizers/default_searchers/successive_halving.yaml b/neps/optimizers/default_searchers/successive_halving.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/successive_halving.yaml rename to neps/optimizers/default_searchers/successive_halving.yaml diff --git a/src/neps/optimizers/default_searchers/successive_halving_prior.yaml b/neps/optimizers/default_searchers/successive_halving_prior.yaml similarity index 100% rename from src/neps/optimizers/default_searchers/successive_halving_prior.yaml rename to neps/optimizers/default_searchers/successive_halving_prior.yaml diff --git a/src/neps/optimizers/grid_search/optimizer.py b/neps/optimizers/grid_search/optimizer.py similarity index 96% rename from src/neps/optimizers/grid_search/optimizer.py rename to neps/optimizers/grid_search/optimizer.py index 1924ec62..1c988926 100644 --- a/src/neps/optimizers/grid_search/optimizer.py +++ b/neps/optimizers/grid_search/optimizer.py @@ -2,8 +2,7 @@ import random -from metahyper.api import ConfigResult - +from ...metahyper import ConfigResult from ...search_spaces.search_space import SearchSpace from ..base_optimizer import BaseOptimizer diff --git a/src/neps/optimizers/info.py b/neps/optimizers/info.py similarity index 100% rename from src/neps/optimizers/info.py rename to neps/optimizers/info.py diff --git a/src/neps/optimizers/multi_fidelity/_dyhpo.py b/neps/optimizers/multi_fidelity/_dyhpo.py similarity index 99% rename from src/neps/optimizers/multi_fidelity/_dyhpo.py rename to neps/optimizers/multi_fidelity/_dyhpo.py index b4d972a1..1c948cea 100644 --- a/src/neps/optimizers/multi_fidelity/_dyhpo.py +++ b/neps/optimizers/multi_fidelity/_dyhpo.py @@ -3,8 +3,7 @@ import numpy as np -from metahyper import ConfigResult - +from ...metahyper import ConfigResult from ...search_spaces.search_space import FloatParameter, IntegerParameter, SearchSpace from ..base_optimizer import BaseOptimizer from ..bayesian_optimization.acquisition_functions.base_acquisition import BaseAcquisition diff --git a/src/neps/optimizers/multi_fidelity/dyhpo.py b/neps/optimizers/multi_fidelity/dyhpo.py similarity index 83% rename from src/neps/optimizers/multi_fidelity/dyhpo.py rename to neps/optimizers/multi_fidelity/dyhpo.py index 75f374b4..5e540bc3 100755 --- a/src/neps/optimizers/multi_fidelity/dyhpo.py +++ b/neps/optimizers/multi_fidelity/dyhpo.py @@ -5,9 +5,12 @@ from typing import Any import numpy as np +import pandas as pd +import time -from metahyper import ConfigResult, instance_from_map +from ...utils.common import EvaluationData, SimpleCSVWriter +from ...metahyper import ConfigResult, instance_from_map from ...search_spaces.search_space import FloatParameter, IntegerParameter, SearchSpace from ..base_optimizer import BaseOptimizer from ..bayesian_optimization.acquisition_functions import AcquisitionMapping @@ -23,6 +26,15 @@ from .utils import MFObservedData +class AcqWriter(SimpleCSVWriter): + def set_data(self, sample_configs: pd.Series, acq_vals: pd.Series): + config_vals = pd.DataFrame([config.hp_values() for config in sample_configs], index=sample_configs.index) + if isinstance(acq_vals, pd.Series): + acq_vals.name = "Acq Value" + self.df = config_vals.join(acq_vals) + self.df = self.df.sort_values(by="Acq Value") + + class MFEIBO(BaseOptimizer): """Base class for MF-BO algorithms that use DyHPO-like acquisition and budgeting.""" @@ -124,7 +136,7 @@ def __init__( self._prep_model_args(self.hp_kernels, self.graph_kernels, pipeline_space) # TODO: Better solution than branching based on the surrogate name is needed - if surrogate_model in ["deep_gp", "gp"]: + if surrogate_model in ["deep_gp", "gp", "dpl"]: model_policy = FreezeThawModel elif surrogate_model == "pfn": model_policy = PFNSurrogate @@ -164,6 +176,8 @@ def __init__( ) self.count = 0 + self.evaluation_data = EvaluationData() + def _prep_model_args(self, hp_kernels, graph_kernels, pipeline_space): if self.surrogate_model_name in ["gp", "gp_hierarchy"]: # setup for GP implemented in NePS @@ -264,7 +278,7 @@ def total_budget_spent(self) -> int | float: return total_budget_spent - def is_init_phase(self, budget_based: bool = True) -> bool: + def is_init_phase(self, budget_based: bool = False) -> bool: if budget_based: # Check if we are still in the initial design phase based on # either the budget spent so far or the number of configurations evaluated @@ -290,11 +304,11 @@ def load_results( previous_results (dict[str, ConfigResult]): [description] pending_evaluations (dict[str, ConfigResult]): [description] """ + start = time.time() self.observed_configs = MFObservedData( columns=["config", "perf", "learning_curves"], index_names=["config_id", "budget_id"], ) - # previous optimization run exists and needs to be loaded self._load_previous_observations(previous_results) self.total_fevals = len(previous_results) + len(pending_evaluations) @@ -306,7 +320,6 @@ def load_results( self.observed_configs.df.sort_index( level=self.observed_configs.df.index.names, inplace=True ) - # TODO: can we do better than keeping a copy of the observed configs? # TODO: can we not hide this in load_results and have something that pops out # more, like a set_state or policy_args @@ -315,6 +328,9 @@ def load_results( init_phase = self.is_init_phase() if not init_phase: self._fit_models() + # print("-" * 50) + # print(f"| Total time for `load_results()`: {time.time()-start:.2f}s") + # print("-" * 50) @classmethod def _get_config_id_split(cls, config_id: str) -> tuple[str, str]: @@ -420,7 +436,7 @@ def get_config_and_ids( # pylint: disable=no-self-use ) config.fidelity.value = self.min_budget _config_id = self.observed_configs.next_config_id() - elif self.is_init_phase(budget_based=True) or self._model_update_failed: + elif self.is_init_phase() or self._model_update_failed: # promote a config randomly if initial design size is satisfied but the # initial design budget has not been exhausted self.logger.info("promoting...") @@ -433,32 +449,84 @@ def get_config_and_ids( # pylint: disable=no-self-use # main acquisition call here after initial design is turned off self.logger.info("acquiring...") # generates candidate samples for acquisition calculation + start = time.time() samples = self.acquisition_sampler.sample( set_new_sample_fidelity=self.pipeline_space.fidelity.lower ) # fidelity values here should be the observations or min. fidelity + # print("-" * 50) + # print(f"| Total time for acq. sampling: {time.time()-start:.2f}s") + # print("-" * 50) + + start = time.time() # calculating acquisition function values for the candidate samples acq, _samples = self.acquisition.eval( # type: ignore[attr-defined] x=samples, asscalar=True ) + acq = pd.Series(acq, index=_samples.index) + + # print("-" * 50) + # print(f"| Total time for acq. eval: {time.time()-start:.2f}s") + # print("-" * 50) # maximizing acquisition function - _idx = np.argsort(acq)[-1] + best_idx = acq.sort_values().index[-1] # extracting the config ID for the selected maximizer - _config_id = samples.index[_samples.index.values[_idx]] + _config_id = best_idx # samples.index[_samples.index.values[_idx]] # `_samples` should have new configs with fidelities set to as required # NOTE: len(samples) need not be equal to len(_samples) as `samples` contain # all (partials + new) configurations obtained from the sampler, but # in `_samples`, configs are removed that have reached maximum epochs allowed # NOTE: `samples` and `_samples` should share the same index values, hence, - # avoid using `.iloc` and work with `.loc` on pandas DataFrame/Series - - # Is this "config = _samples.loc[_config_id]"? + # avoid using `.iloc` and work with `.loc` on these pandas DataFrame/Series + + acq_writer = AcqWriter("Acq_values") + # Writes extra information into Acq_values.csv + # if hasattr(self.acquisition, "mu"): + # # collect prediction learning_curves + # lcs = [] + # # and tabular ids + # tabular_ids = [] + # for idx in _samples.index: + # if self.acquisition_sampler.is_tabular: + # tabular_ids.append(samples[idx]["id"].value) + # if idx in self.observed_configs.df.index.levels[0]: + # # budget_level = self.get_budget_level(_samples[idx]) + # # extracting the available/observed learning curve + # lc = self.observed_configs.extract_learning_curve(idx, budget_id=None) + # else: + # # initialize a learning curve with a placeholder + # # This is later padded accordingly for the Conv1D layer + # lc = [] + # lcs.append(lc) + # + # data = {"Acq Value": acq.values, + # "preds": self.acquisition.mu, + # "incumbents": self.acquisition.mu_star, + # "std": self.acquisition.std, + # "pred_learning_curves": lcs} + # if self.acquisition_sampler.is_tabular: + # data["tabular_ids"] = tabular_ids + # + # acq = pd.DataFrame(data, index=_samples.index) + acq_writer.set_data(_samples, acq) + self.evaluation_data.data_dict["acq"] = acq_writer + + # assigning config hyperparameters config = samples.loc[_config_id] - config.fidelity.value = _samples.loc[_config_id].fidelity.value + # IMPORTANT: setting the fidelity appropriately + + config.fidelity.value = ( + config.fidelity.lower + if best_idx > max(self.observed_configs.seen_config_ids) + else ( + self.get_budget_value( + self.observed_configs.get_max_observed_fidelity_level_per_config().loc[best_idx] + ) + self.step_size # ONE-STEP FIDELITY QUERY + ) + ) # generating correct IDs if _config_id in self.observed_configs.seen_config_ids: config_id = f"{_config_id}_{self.get_budget_level(config)}" previous_config_id = f"{_config_id}_{self.get_budget_level(config) - 1}" else: config_id = f"{self.observed_configs.next_config_id()}_{self.get_budget_level(config)}" - return config.hp_values(), config_id, previous_config_id diff --git a/src/neps/optimizers/multi_fidelity/hyperband.py b/neps/optimizers/multi_fidelity/hyperband.py similarity index 99% rename from src/neps/optimizers/multi_fidelity/hyperband.py rename to neps/optimizers/multi_fidelity/hyperband.py index 4c368bfe..86ff2f5f 100644 --- a/src/neps/optimizers/multi_fidelity/hyperband.py +++ b/neps/optimizers/multi_fidelity/hyperband.py @@ -9,8 +9,7 @@ import numpy as np from typing_extensions import Literal -from metahyper import ConfigResult - +from ...metahyper import ConfigResult from ...search_spaces.search_space import SearchSpace from ..bayesian_optimization.acquisition_functions.base_acquisition import BaseAcquisition from ..bayesian_optimization.acquisition_samplers.base_acq_sampler import ( diff --git a/src/neps/optimizers/multi_fidelity/mf_bo.py b/neps/optimizers/multi_fidelity/mf_bo.py similarity index 90% rename from src/neps/optimizers/multi_fidelity/mf_bo.py rename to neps/optimizers/multi_fidelity/mf_bo.py index 3cf191bd..c825120b 100755 --- a/src/neps/optimizers/multi_fidelity/mf_bo.py +++ b/neps/optimizers/multi_fidelity/mf_bo.py @@ -7,8 +7,7 @@ import pandas as pd import torch -from metahyper import instance_from_map - +from ...metahyper import instance_from_map from ..bayesian_optimization.models import SurrogateModelMapping from ..multi_fidelity.utils import normalize_vectorize_config from ..multi_fidelity_prior.utils import calc_total_resources_spent, update_fidelity @@ -199,6 +198,11 @@ def __init__( ) if self.surrogate_model_name in ["deep_gp", "pfn"]: self.surrogate_model_args.update({"pipeline_space": pipeline_space}) + elif self.surrogate_model_name == "dpl": + self.surrogate_model_args.update( + {"pipeline_space": self.pipeline_space, + "observed_data": self.observed_configs} + ) # instantiate the surrogate model self.surrogate_model = instance_from_map( @@ -234,7 +238,7 @@ def _fantasize_pending(self, train_x, train_y, pending_x): def _fit(self, train_x, train_y, train_lcs): if self.surrogate_model_name in ["gp", "gp_hierarchy"]: self.surrogate_model.fit(train_x, train_y) - elif self.surrogate_model_name in ["deep_gp", "pfn"]: + elif self.surrogate_model_name in ["deep_gp", "pfn", "dpl"]: self.surrogate_model.fit(train_x, train_y, train_lcs) else: # check neps/optimizers/bayesian_optimization/models/__init__.py for options @@ -245,7 +249,7 @@ def _fit(self, train_x, train_y, train_lcs): def _predict(self, test_x, test_lcs): if self.surrogate_model_name in ["gp", "gp_hierarchy"]: return self.surrogate_model.predict(test_x) - elif self.surrogate_model_name in ["deep_gp", "pfn"]: + elif self.surrogate_model_name in ["deep_gp", "pfn", "dpl"]: return self.surrogate_model.predict(test_x, test_lcs) else: # check neps/optimizers/bayesian_optimization/models/__init__.py for options @@ -263,12 +267,29 @@ def set_state( self.surrogate_model_args = ( surrogate_model_args if surrogate_model_args is not None else {} ) + if self.surrogate_model_name == "dpl": + self.surrogate_model_args.update( + {"pipeline_space": self.pipeline_space, + "observed_data": self.observed_configs} + ) + self.surrogate_model = instance_from_map( + SurrogateModelMapping, + self.surrogate_model_name, + name="surrogate model", + kwargs=self.surrogate_model_args, + ) + # only to handle tabular spaces if self.pipeline_space.has_tabular: if self.surrogate_model_name in ["deep_gp", "pfn"]: self.surrogate_model_args.update( {"pipeline_space": self.pipeline_space.raw_tabular_space} ) + elif self.surrogate_model_name == "dpl": + self.surrogate_model_args.update( + {"pipeline_space": self.pipeline_space, + "observed_data": self.observed_configs} + ) # instantiate the surrogate model, again, with the new pipeline space self.surrogate_model = instance_from_map( SurrogateModelMapping, @@ -276,6 +297,17 @@ def set_state( name="surrogate model", kwargs=self.surrogate_model_args, ) + elif self.surrogate_model_name == "dpl": + self.surrogate_model_args.update( + {"pipeline_space": self.pipeline_space, + "observed_data": self.observed_configs} + ) + self.surrogate_model = instance_from_map( + SurrogateModelMapping, + self.surrogate_model_name, + name="surrogate model", + kwargs=self.surrogate_model_args, + ) def update_model(self, train_x=None, train_y=None, pending_x=None, decay_t=None): if train_x is None: @@ -323,6 +355,8 @@ def preprocess_training_set(self): configs, idxs, performances = self.observed_configs.get_tokenized_data( self.observed_configs.df.copy().assign(config=_configs) ) + idxs = idxs.astype(float) + idxs[:, 1] = idxs[:, 1] / _configs[0].fidelity.upper # TODO: account for fantasization self.train_x = torch.Tensor(np.hstack([idxs, configs])).to(device) self.train_y = torch.Tensor(performances).to(device) diff --git a/src/neps/optimizers/multi_fidelity/promotion_policy.py b/neps/optimizers/multi_fidelity/promotion_policy.py similarity index 100% rename from src/neps/optimizers/multi_fidelity/promotion_policy.py rename to neps/optimizers/multi_fidelity/promotion_policy.py diff --git a/src/neps/optimizers/multi_fidelity/sampling_policy.py b/neps/optimizers/multi_fidelity/sampling_policy.py similarity index 99% rename from src/neps/optimizers/multi_fidelity/sampling_policy.py rename to neps/optimizers/multi_fidelity/sampling_policy.py index 5b1a8529..fc8075e4 100644 --- a/src/neps/optimizers/multi_fidelity/sampling_policy.py +++ b/neps/optimizers/multi_fidelity/sampling_policy.py @@ -10,8 +10,7 @@ import pandas as pd import torch -from metahyper import instance_from_map - +from ...metahyper import instance_from_map from ...search_spaces.search_space import SearchSpace from ..bayesian_optimization.acquisition_functions import AcquisitionMapping from ..bayesian_optimization.acquisition_functions.base_acquisition import BaseAcquisition diff --git a/src/neps/optimizers/multi_fidelity/successive_halving.py b/neps/optimizers/multi_fidelity/successive_halving.py similarity index 99% rename from src/neps/optimizers/multi_fidelity/successive_halving.py rename to neps/optimizers/multi_fidelity/successive_halving.py index 54fa09db..a3145dc2 100644 --- a/src/neps/optimizers/multi_fidelity/successive_halving.py +++ b/neps/optimizers/multi_fidelity/successive_halving.py @@ -10,8 +10,7 @@ import pandas as pd from typing_extensions import Literal -from metahyper import ConfigResult - +from ...metahyper import ConfigResult from ...search_spaces.hyperparameters.categorical import ( CATEGORICAL_CONFIDENCE_SCORES, CategoricalParameter, diff --git a/src/neps/optimizers/multi_fidelity/utils.py b/neps/optimizers/multi_fidelity/utils.py similarity index 81% rename from src/neps/optimizers/multi_fidelity/utils.py rename to neps/optimizers/multi_fidelity/utils.py index f1d359a6..80a6a230 100644 --- a/src/neps/optimizers/multi_fidelity/utils.py +++ b/neps/optimizers/multi_fidelity/utils.py @@ -5,8 +5,11 @@ import numpy as np import pandas as pd +import time import torch +from copy import deepcopy + from ...optimizers.utils import map_real_hyperparameters_from_tabular_ids from ...search_spaces.search_space import SearchSpace @@ -55,6 +58,7 @@ class MFObservedData: default_config_col = "config" default_perf_col = "perf" default_lc_col = "learning_curves" + # TODO: deepcopy all the mutable outputs from the dataframe def __init__( self, @@ -79,6 +83,7 @@ def __init__( self.config_idx = index_names[0] self.budget_idx = index_names[1] + self.index_names = index_names index = pd.MultiIndex.from_tuples([], names=index_names) @@ -129,8 +134,9 @@ def add_data( data_list = data if not self.df.index.isin(index_list).any(): - _df = pd.DataFrame(data_list, columns=self.df.columns, index=index_list) - self.df = pd.concat((self.df, _df)) + index = pd.MultiIndex.from_tuples(index_list, names=self.index_names) + _df = pd.DataFrame(data_list, columns=self.df.columns, index=index) + self.df = _df.copy() if self.df.empty else pd.concat((self.df, _df)) elif error: raise ValueError( f"Data with at least one of the given indices already " @@ -178,7 +184,7 @@ def get_incumbents_for_budgets(self, maximize: bool = False): Returns a series object with the best partial configuration for each budget id Note: this will always map the best lowest ID if two configurations - has the same performance at the same fidelity + have the same performance at the same fidelity """ learning_curves = self.get_learning_curves() if maximize: @@ -205,6 +211,16 @@ def get_best_performance_for_each_budget(self, maximize: bool = False): return performance + def get_budget_level_for_best_performance(self, maximize: bool = False) -> int: + """Returns the lowest budget level at which the highest performance was recorded. + """ + perf_per_z = self.get_best_performance_for_each_budget(maximize=maximize) + y_star = self.get_best_seen_performance(maximize=maximize) + # uses the minimum of the budget that see the maximum obseved score + op = max if maximize else min + z_inc = int(op([_z for _z, _y in perf_per_z.items() if _y == y_star])) + return z_inc + def get_best_learning_curve_id(self, maximize: bool = False): """ Returns a single configuration id of the best observed performance @@ -240,7 +256,17 @@ def reduce_to_max_seen_budgets(self): def get_partial_configs_at_max_seen(self): return self.reduce_to_max_seen_budgets()[self.config_col] - def extract_learning_curve(self, config_id: int, budget_id: int) -> list[float]: + def extract_learning_curve( + self, config_id: int, budget_id: int | None = None + ) -> list[float]: + if budget_id is None: + # budget_id only None when predicting + # extract full observed learning curve for prediction pipeline + budget_id = max(self.df.loc[config_id].index.get_level_values("budget_id").values) + 1 + + # For the first epoch we have no learning curve available + if budget_id == 0: + return [] # reduce budget_id to discount the current validation loss # both during training and prediction phase budget_id = max(0, budget_id - 1) @@ -249,11 +275,12 @@ def extract_learning_curve(self, config_id: int, budget_id: int) -> list[float]: else: lcs = self.get_learning_curves() lc = lcs.loc[config_id, :budget_id].values.flatten().tolist() - return lc + return deepcopy(lc) def get_training_data_4DyHPO( self, df: pd.DataFrame, pipeline_space: SearchSpace | None = None ): + start = time.time() configs = [] learning_curves = [] performance = [] @@ -268,8 +295,34 @@ def get_training_data_4DyHPO( configs.append(row[self.config_col]) performance.append(row[self.perf_col]) learning_curves.append(self.extract_learning_curve(config_id, budget_id)) + # print("-" * 50) + # print(f"| Time for `get_training_data_4DyHPO()`: {time.time()-start:.2f}s") + # print("-" * 50) return configs, learning_curves, performance + def get_best_performance_per_config(self, maximize: bool = False) -> pd.Series: + """Returns the best score recorded per config across fidelities seen. + """ + op = np.max if maximize else np.min + perf = ( + self.df + .sort_values("budget_id", ascending=False) # sorts with largest budget first + .groupby("config_id") # retains only config_id + .first() # retrieves the largest budget seen for each config_id + .learning_curves # extracts all values seen till largest budget for a config + .apply(op) # finds the minimum over per-config learning curve + ) + return perf + + def get_max_observed_fidelity_level_per_config(self) -> pd.Series: + """Returns the highest fidelity level recorded per config seen. + """ + max_z_observed = { + _id: self.df.loc[_id,:].index.sort_values()[-1] + for _id in self.df.index.get_level_values("config_id").sort_values() + } + return pd.Series(max_z_observed) + def get_tokenized_data(self, df: pd.DataFrame): idxs = df.index.values idxs = np.array([list(idx) for idx in idxs]) diff --git a/src/neps/optimizers/multi_fidelity_prior/__init__.py b/neps/optimizers/multi_fidelity_prior/__init__.py similarity index 100% rename from src/neps/optimizers/multi_fidelity_prior/__init__.py rename to neps/optimizers/multi_fidelity_prior/__init__.py diff --git a/src/neps/optimizers/multi_fidelity_prior/async_priorband.py b/neps/optimizers/multi_fidelity_prior/async_priorband.py similarity index 99% rename from src/neps/optimizers/multi_fidelity_prior/async_priorband.py rename to neps/optimizers/multi_fidelity_prior/async_priorband.py index 9527c97d..c932d45d 100644 --- a/src/neps/optimizers/multi_fidelity_prior/async_priorband.py +++ b/neps/optimizers/multi_fidelity_prior/async_priorband.py @@ -5,8 +5,7 @@ import numpy as np from typing_extensions import Literal -from metahyper import ConfigResult - +from ...metahyper import ConfigResult from ...search_spaces.search_space import SearchSpace from ..bayesian_optimization.acquisition_functions.base_acquisition import BaseAcquisition from ..bayesian_optimization.acquisition_samplers.base_acq_sampler import ( diff --git a/src/neps/optimizers/multi_fidelity_prior/priorband.py b/neps/optimizers/multi_fidelity_prior/priorband.py similarity index 100% rename from src/neps/optimizers/multi_fidelity_prior/priorband.py rename to neps/optimizers/multi_fidelity_prior/priorband.py diff --git a/src/neps/optimizers/multi_fidelity_prior/utils.py b/neps/optimizers/multi_fidelity_prior/utils.py similarity index 100% rename from src/neps/optimizers/multi_fidelity_prior/utils.py rename to neps/optimizers/multi_fidelity_prior/utils.py diff --git a/src/neps/optimizers/multiple_knowledge_sources/prototype_optimizer.py b/neps/optimizers/multiple_knowledge_sources/prototype_optimizer.py similarity index 98% rename from src/neps/optimizers/multiple_knowledge_sources/prototype_optimizer.py rename to neps/optimizers/multiple_knowledge_sources/prototype_optimizer.py index eb399b06..9ab96b1a 100644 --- a/src/neps/optimizers/multiple_knowledge_sources/prototype_optimizer.py +++ b/neps/optimizers/multiple_knowledge_sources/prototype_optimizer.py @@ -3,8 +3,7 @@ import logging from typing import Any -from metahyper.api import ConfigResult - +from ...metahyper import ConfigResult from ...search_spaces.search_space import SearchSpace from ...utils.data_loading import read_tasks_and_dev_stages_from_disk from .. import BaseOptimizer diff --git a/src/neps/optimizers/random_search/optimizer.py b/neps/optimizers/random_search/optimizer.py similarity index 96% rename from src/neps/optimizers/random_search/optimizer.py rename to neps/optimizers/random_search/optimizer.py index 2d23fe19..4b84838d 100644 --- a/src/neps/optimizers/random_search/optimizer.py +++ b/neps/optimizers/random_search/optimizer.py @@ -1,7 +1,6 @@ from __future__ import annotations -from metahyper.api import ConfigResult - +from ...metahyper import ConfigResult from ...search_spaces.search_space import SearchSpace from ..base_optimizer import BaseOptimizer diff --git a/src/neps/optimizers/regularized_evolution/optimizer.py b/neps/optimizers/regularized_evolution/optimizer.py similarity index 100% rename from src/neps/optimizers/regularized_evolution/optimizer.py rename to neps/optimizers/regularized_evolution/optimizer.py diff --git a/neps/optimizers/utils.py b/neps/optimizers/utils.py new file mode 100644 index 00000000..757dd0b3 --- /dev/null +++ b/neps/optimizers/utils.py @@ -0,0 +1,30 @@ +import pandas as pd + +from ..search_spaces.search_space import SearchSpace + + +def map_real_hyperparameters_from_tabular_ids( + x: pd.Series, pipeline_space: SearchSpace +) -> pd.Series: + """ Maps the tabular IDs to the actual HPs from the pipeline space. + + Args: + x (pd.Series): A pandas series with the tabular IDs. + TODO: Mention expected format of the series. + pipeline_space (SearchSpace): The pipeline space. + + Returns: + pd.Series: A pandas series with the actual HPs. + TODO: Mention expected format of the series. + """ + if len(x) == 0: + return x + # copying hyperparameter configs based on IDs + _x = pd.Series( + [pipeline_space.custom_grid_table[x.loc[idx]["id"].value] for idx in x.index.values], + index=x.index + ) + # setting the passed fidelities for the corresponding IDs + for idx in _x.index.values: + _x.loc[idx].fidelity.value = x.loc[idx].fidelity.value + return _x diff --git a/src/neps/plot/__init__.py b/neps/plot/__init__.py similarity index 100% rename from src/neps/plot/__init__.py rename to neps/plot/__init__.py diff --git a/src/neps/plot/__main__.py b/neps/plot/__main__.py similarity index 100% rename from src/neps/plot/__main__.py rename to neps/plot/__main__.py diff --git a/src/neps/plot/plot.py b/neps/plot/plot.py similarity index 100% rename from src/neps/plot/plot.py rename to neps/plot/plot.py diff --git a/src/neps/plot/plotting.py b/neps/plot/plotting.py similarity index 100% rename from src/neps/plot/plotting.py rename to neps/plot/plotting.py diff --git a/src/neps/plot/read_results.py b/neps/plot/read_results.py similarity index 100% rename from src/neps/plot/read_results.py rename to neps/plot/read_results.py diff --git a/src/neps/plot/tensorboard_eval.py b/neps/plot/tensorboard_eval.py similarity index 96% rename from src/neps/plot/tensorboard_eval.py rename to neps/plot/tensorboard_eval.py index 445406cb..6463563e 100644 --- a/src/neps/plot/tensorboard_eval.py +++ b/neps/plot/tensorboard_eval.py @@ -2,15 +2,22 @@ import math import os +import warnings from pathlib import Path import numpy as np import torch + +# Remove this once we support pytorch > 2.1.0 +# https://github.com/automl/neps/issues/26 +warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="torch.utils.tensorboard" +) + from torch.utils.tensorboard import SummaryWriter from torch.utils.tensorboard.summary import hparams -from metahyper.api import ConfigInRun - +from ..metahyper.api import ConfigInRun from ..status.status import get_summary_dict from ..utils.common import get_initial_directory @@ -306,7 +313,7 @@ def _write_image_config( if resize_images is None: resize_images = [32, 32] - if tblogger.current_epoch and tblogger.current_epoch % counter == 0: + if tblogger.current_epoch >= 0 and tblogger.current_epoch % counter == 0: # Log every multiple of "counter" if num_images > len(image): @@ -473,10 +480,12 @@ def log( loss (float): Current loss value. current_epoch (int): Current epoch of the experiment (used as the global step). - writer_scalar (bool, optional): Displaying the loss or accuracy + writer_config_scalar (bool, optional): Displaying the loss or accuracy curve on tensorboard (default: True) - writer_hparam (bool, optional): Write hyperparameters logging of + writer_config_hparam (bool, optional): Write hyperparameters logging of the configs (default: True). + write_summary_incumbent (bool, optional): Set to `True` for a live + incumbent trajectory. extra_data (dict, optional): Additional experiment data for logging. """ if tblogger.disable_logging: diff --git a/neps/py.typed b/neps/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/src/neps/search_spaces/__init__.py b/neps/search_spaces/__init__.py similarity index 100% rename from src/neps/search_spaces/__init__.py rename to neps/search_spaces/__init__.py diff --git a/src/neps/search_spaces/architecture/api.py b/neps/search_spaces/architecture/api.py similarity index 100% rename from src/neps/search_spaces/architecture/api.py rename to neps/search_spaces/architecture/api.py diff --git a/src/neps/search_spaces/architecture/cfg.py b/neps/search_spaces/architecture/cfg.py similarity index 100% rename from src/neps/search_spaces/architecture/cfg.py rename to neps/search_spaces/architecture/cfg.py diff --git a/src/neps/search_spaces/architecture/cfg_variants/cfg_resolution.py b/neps/search_spaces/architecture/cfg_variants/cfg_resolution.py similarity index 100% rename from src/neps/search_spaces/architecture/cfg_variants/cfg_resolution.py rename to neps/search_spaces/architecture/cfg_variants/cfg_resolution.py diff --git a/src/neps/search_spaces/architecture/cfg_variants/constrained_cfg.py b/neps/search_spaces/architecture/cfg_variants/constrained_cfg.py similarity index 100% rename from src/neps/search_spaces/architecture/cfg_variants/constrained_cfg.py rename to neps/search_spaces/architecture/cfg_variants/constrained_cfg.py diff --git a/src/neps/search_spaces/architecture/core_graph_grammar.py b/neps/search_spaces/architecture/core_graph_grammar.py similarity index 100% rename from src/neps/search_spaces/architecture/core_graph_grammar.py rename to neps/search_spaces/architecture/core_graph_grammar.py diff --git a/src/neps/search_spaces/architecture/crossover.py b/neps/search_spaces/architecture/crossover.py similarity index 100% rename from src/neps/search_spaces/architecture/crossover.py rename to neps/search_spaces/architecture/crossover.py diff --git a/src/neps/search_spaces/architecture/graph.py b/neps/search_spaces/architecture/graph.py similarity index 99% rename from src/neps/search_spaces/architecture/graph.py rename to neps/search_spaces/architecture/graph.py index 35d2ad01..42b2c388 100644 --- a/src/neps/search_spaces/architecture/graph.py +++ b/neps/search_spaces/architecture/graph.py @@ -12,7 +12,7 @@ import networkx as nx import torch from networkx.algorithms.dag import lexicographical_topological_sort -from path import Path +from pathlib import Path from torch import nn from ...utils.common import AttrDict diff --git a/src/neps/search_spaces/architecture/graph_grammar.py b/neps/search_spaces/architecture/graph_grammar.py similarity index 100% rename from src/neps/search_spaces/architecture/graph_grammar.py rename to neps/search_spaces/architecture/graph_grammar.py diff --git a/src/neps/search_spaces/architecture/mutations.py b/neps/search_spaces/architecture/mutations.py similarity index 100% rename from src/neps/search_spaces/architecture/mutations.py rename to neps/search_spaces/architecture/mutations.py diff --git a/src/neps/search_spaces/architecture/primitives.py b/neps/search_spaces/architecture/primitives.py similarity index 100% rename from src/neps/search_spaces/architecture/primitives.py rename to neps/search_spaces/architecture/primitives.py diff --git a/src/neps/search_spaces/architecture/topologies.py b/neps/search_spaces/architecture/topologies.py similarity index 100% rename from src/neps/search_spaces/architecture/topologies.py rename to neps/search_spaces/architecture/topologies.py diff --git a/src/neps/search_spaces/hyperparameters/categorical.py b/neps/search_spaces/hyperparameters/categorical.py similarity index 98% rename from src/neps/search_spaces/hyperparameters/categorical.py rename to neps/search_spaces/hyperparameters/categorical.py index 39547321..6b5674b2 100644 --- a/src/neps/search_spaces/hyperparameters/categorical.py +++ b/neps/search_spaces/hyperparameters/categorical.py @@ -2,11 +2,10 @@ import random from copy import copy, deepcopy -from typing import Iterable +from typing import Iterable, Literal import numpy as np import numpy.typing as npt -from typing_extensions import Literal from ..parameter import Parameter @@ -32,9 +31,7 @@ def __init__( self.upper = default self.default_confidence_score = CATEGORICAL_CONFIDENCE_SCORES[default_confidence] self.has_prior = self.default is not None - self.is_fidelity = is_fidelity - self.choices = list(choices) self.num_choices = len(self.choices) self.probabilities: list[npt.NDArray] = list( diff --git a/src/neps/search_spaces/hyperparameters/constant.py b/neps/search_spaces/hyperparameters/constant.py similarity index 100% rename from src/neps/search_spaces/hyperparameters/constant.py rename to neps/search_spaces/hyperparameters/constant.py diff --git a/src/neps/search_spaces/hyperparameters/float.py b/neps/search_spaces/hyperparameters/float.py similarity index 99% rename from src/neps/search_spaces/hyperparameters/float.py rename to neps/search_spaces/hyperparameters/float.py index c35cafc2..2c8eaeb4 100644 --- a/src/neps/search_spaces/hyperparameters/float.py +++ b/neps/search_spaces/hyperparameters/float.py @@ -2,10 +2,10 @@ import math from copy import deepcopy +from typing import Literal import numpy as np import scipy.stats -from typing_extensions import Literal from .numerical import NumericalParameter @@ -37,7 +37,6 @@ def __init__( if self.lower >= self.upper: raise ValueError("Float parameter: bounds error (lower >= upper).") - self.log = log if self.log: diff --git a/src/neps/search_spaces/hyperparameters/integer.py b/neps/search_spaces/hyperparameters/integer.py similarity index 100% rename from src/neps/search_spaces/hyperparameters/integer.py rename to neps/search_spaces/hyperparameters/integer.py diff --git a/src/neps/search_spaces/hyperparameters/numerical.py b/neps/search_spaces/hyperparameters/numerical.py similarity index 100% rename from src/neps/search_spaces/hyperparameters/numerical.py rename to neps/search_spaces/hyperparameters/numerical.py diff --git a/src/neps/search_spaces/parameter.py b/neps/search_spaces/parameter.py similarity index 100% rename from src/neps/search_spaces/parameter.py rename to neps/search_spaces/parameter.py diff --git a/src/neps/search_spaces/search_space.py b/neps/search_spaces/search_space.py similarity index 71% rename from src/neps/search_spaces/search_space.py rename to neps/search_spaces/search_space.py index 48d0b252..d74fac11 100644 --- a/src/neps/search_spaces/search_space.py +++ b/neps/search_spaces/search_space.py @@ -6,10 +6,12 @@ from collections import OrderedDict from copy import deepcopy from itertools import product +from pathlib import Path import ConfigSpace as CS import numpy as np import pandas as pd +import yaml from ..utils.common import has_instance from . import ( @@ -21,6 +23,10 @@ ) from .architecture.graph import Graph from .parameter import Parameter +from .yaml_search_space_utils import ( + SearchSpaceFromYamlFileError, + deduce_and_validate_param_type, +) def pipeline_space_from_configspace( @@ -61,6 +67,121 @@ def pipeline_space_from_configspace( return pipeline_space +def pipeline_space_from_yaml( + yaml_file_path: str | Path, +) -> dict[ + str, FloatParameter | IntegerParameter | CategoricalParameter | ConstantParameter +]: + """ + Reads configuration details from a YAML file and constructs a pipeline space + dictionary. + + This function extracts parameter configurations from a YAML file, validating and + translating them into corresponding parameter objects. The resulting dictionary + maps parameter names to their respective configuration objects. + + Args: + yaml_file_path (str | Path): Path to the YAML file containing parameter + configurations. + + Returns: + dict: A dictionary where keys are parameter names and values are parameter + objects (like IntegerParameter, FloatParameter, etc.). + + Raises: + SearchSpaceFromYamlFileError: This custom exception is raised if there are issues + with the YAML file's format or contents. It encapsulates underlying exceptions + (KeyError, TypeError, ValueError) that occur during the processing of the YAML + file. This approach localizes error handling, providing clearer context and + diagnostics. The raised exception includes the type of the original error and + a descriptive message. + + Note: + The YAML file should be properly structured with valid keys and values as per the + expected parameter types. The function employs modular validation and type + deduction logic, ensuring each parameter's configuration adheres to expected + formats and constraints. Any deviation results in an appropriately raised error, + which is then captured by SearchSpaceFromYamlFileError for streamlined error + handling. + + Example: + To use this function with a YAML file 'config.yaml', you can do: + pipeline_space = pipeline_space_from_yaml('config.yaml') + """ + try: + # try to load the YAML file + try: + with open(yaml_file_path) as file: + config = yaml.safe_load(file) + except yaml.YAMLError as e: + raise ValueError( + f"The file at {str(yaml_file_path)} is not a valid YAML file." + ) from e + + # check for init key pipeline_space + if "pipeline_space" not in config: + raise KeyError( + "The YAML file is incorrectly constructed: the 'pipeline_space:' " + "reference is missing at the top of the file." + ) + + # Initialize the pipeline space + pipeline_space = {} + + # Iterate over the items in the YAML configuration + for name, details in config["pipeline_space"].items(): + # get parameter type + param_type = deduce_and_validate_param_type(name, details) + + # init parameter by checking type + if param_type in ("int", "integer"): + # Integer Parameter + pipeline_space[name] = IntegerParameter( + lower=details["lower"], + upper=details["upper"], + log=details.get("log", False), + is_fidelity=details.get("is_fidelity", False), + default=details.get("default", None), + default_confidence=details.get("default_confidence", "low"), + ) + elif param_type == "float": + # Float Parameter + pipeline_space[name] = FloatParameter( + lower=details["lower"], + upper=details["upper"], + log=details.get("log", False), + is_fidelity=details.get("is_fidelity", False), + default=details.get("default", None), + default_confidence=details.get("default_confidence", "low"), + ) + elif param_type in ("cat", "categorical"): + # Categorical parameter + pipeline_space[name] = CategoricalParameter( + choices=details["choices"], + is_fidelity=details.get("is_fidelity", False), + default=details.get("default", None), + default_confidence=details.get("default_confidence", "low"), + ) + elif param_type in ("const", "constant"): + # Constant parameter + pipeline_space[name] = ConstantParameter( + value=details["value"], is_fidelity=details.get("is_fidelity", False) + ) + else: + # Handle unknown parameter type + raise TypeError( + f"Unsupported parameter type{details['type']} for '{name}'.\n" + f"Supported Types for argument type are:\n" + "For integer parameter: int, integer\n" + "For float parameter: float\n" + "For categorical parameter: cat, categorical\n" + "For constant parameter: const, constant\n" + ) + except (KeyError, TypeError, ValueError) as e: + raise SearchSpaceFromYamlFileError(e) from e + return pipeline_space + + class SearchSpace(collections.abc.Mapping): def __init__(self, **hyperparameters): self.hyperparameters = OrderedDict() @@ -83,12 +204,17 @@ def __init__(self, **hyperparameters): ) self.fidelity = hyperparameter - # Check if defaults exists to construct prior from - if hasattr(hyperparameter, "default") and hyperparameter.default is not None: + # Check if defaults exists to construct prior from, except of + # ConstantParameter because default gets init always by the given value + if ( + hasattr(hyperparameter, "default") + and hyperparameter.default is not None + and not isinstance(hyperparameter, ConstantParameter) + ): self.has_prior = True elif hasattr(hyperparameter, "has_prior") and hyperparameter.has_prior: self.has_prior = True - + # Variables for tabular bookkeeping self.custom_grid_table = None self.raw_tabular_space = None @@ -97,29 +223,40 @@ def __init__(self, **hyperparameters): def set_custom_grid_space( self, grid_table: pd.Series | pd.DataFrame, - raw_space: SearchSpace | CS.ConfigurationSpace + raw_space: SearchSpace | CS.ConfigurationSpace, ): """Set a custom grid space for the search space. - This function is used to set a custom grid space for the pipeline space. - NOTE: Only to be used if a custom set of hyperparameters from the search space - is to be sampled or used for acquisition functions. - WARNING: The type check and the table format requirement is loose and + This function is used to set a custom grid space for the pipeline space. + NOTE: Only to be used if a custom set of hyperparameters from the search space + is to be sampled or used for acquisition functions. + WARNING: The type check and the table format requirement is loose and can break certain components. """ self.custom_grid_table: pd.DataFrame | pd.Series = grid_table self.raw_tabular_space = ( SearchSpace(**raw_space) - if not isinstance(raw_space, SearchSpace) else raw_space + if not isinstance(raw_space, SearchSpace) + else raw_space ) if self.custom_grid_table is None or self.raw_tabular_space is None: raise ValueError( "Both grid_table and raw_space must be set!\n" - "A table or list of fixed configs must be supported with a " + "A table or list of fixed configs must be supported with a " "continuous space representing the type and bounds of each " "hyperparameter for accurate modeling." ) self.has_tabular = True + # Updating `custom_grid_table` as a map for quick lookup with placeholder fidelity + placeholder_config = self.raw_tabular_space.sample(ignore_fidelity=True) # sets fidelity as None + # `placeholder_config` allows to store map values as NePS SearchSpace type + # and also create a placeholder for fideity value + _map = { + idx: deepcopy(placeholder_config) + for idx in self.custom_grid_table.index.values + } + _ = [v.load_from(self.custom_grid_table.loc[k].to_dict()) for k, v in _map.items()] + self.custom_grid_table = _map @property def has_fidelity(self): @@ -201,7 +338,7 @@ def _smbo_mutation(self, patience=50, **kwargs): new_config[hp_name] = hp.mutate(**kwargs) break except Exception as e: - self.logger.warning(f"{hp_name} FAILED!") + self.logger.warning(f"{hp_name} FAILED! Error: {e}") continue return new_config @@ -356,7 +493,14 @@ def load_from(self, config: dict): self.hyperparameters[name].load_from(config[name]) def copy(self): - return deepcopy(self) + _copy = deepcopy(self) + + if _copy.has_tabular: + # each configuration does not need to carry the tabular data + _copy.has_tabular = False + _copy.custom_grid_table = None + _copy.raw_tabular_space = None + return _copy def sample_default_configuration( self, patience: int = 1, ignore_fidelity=True, ignore_missing_defaults=False @@ -440,8 +584,8 @@ def __str__(self): return pprint.pformat(self.hyperparameters) def is_equal_value(self, other, include_fidelity=True, on_decimal=8): - # This does NOT check that the entire SearchSpace is equal (and thus it is not a dunder method), - # but only checks the configuration + # This does NOT check that the entire SearchSpace is equal (and thus it is + # not a dunder method), but only checks the configuration if self.hyperparameters.keys() != other.hyperparameters.keys(): return False diff --git a/neps/search_spaces/yaml_search_space_utils.py b/neps/search_spaces/yaml_search_space_utils.py new file mode 100644 index 00000000..e0efe616 --- /dev/null +++ b/neps/search_spaces/yaml_search_space_utils.py @@ -0,0 +1,462 @@ +from __future__ import annotations + +import re + + +def convert_scientific_notation(value: str | int | float, show_usage_flag=False) \ + -> float | (float, bool): + """ + Convert a given value to a float if it's a string that matches scientific e notation. + This is especially useful for numbers like "3.3e-5" which YAML parsers may not + directly interpret as floats. + + If the 'show_usage_flag' is set to True, the function returns a tuple of the float + conversion and a boolean flag indicating whether scientific notation was detected. + + Args: + value (str | int | float): The value to convert. Can be an integer, float, + or a string representing a number, possibly in + scientific notation. + show_usage_flag (bool): Optional; defaults to False. If True, the function + also returns a flag indicating whether scientific + notation was detected in the string. + + Returns: + float: The value converted to float if 'show_usage_flag' is False. + (float, bool): A tuple containing the value converted to float and a flag + indicating scientific notation detection if 'show_usage_flag' + is True. + + Raises: + ValueError: If the value is a string and does not represent a valid number. + """ + + e_notation_pattern = r"^-?\d+(\.\d+)?[eE]-?\d+$" + + flag = False # Flag if e notation was detected + + if isinstance(value, str): + # Remove all whitespace from the string + value_no_space = value.replace(" ", "") + + # check for e notation + if re.match(e_notation_pattern, value_no_space): + flag = True + + if show_usage_flag is True: + return float(value), flag + else: + return float(value) + + +class SearchSpaceFromYamlFileError(Exception): + """ + Exception raised for errors occurring during the initialization of the search space + from a YAML file. + + Attributes: + exception_type (str): The type of the original exception. + message (str): A detailed message that includes the type of the original exception + and the error description. + + Args: + exception (Exception): The original exception that was raised during the + initialization of the search space from the YAML file. + + Example Usage: + try: + # Code to initialize search space from YAML file + except (KeyError, TypeError, ValueError) as e: + raise SearchSpaceFromYamlFileError(e) + """ + + def __init__(self, exception): + self.exception_type = type(exception).__name__ + self.message = ( + f"Error occurred during initialization of search space from " + f"YAML file.\n {self.exception_type}: {exception}" + ) + super().__init__(self.message) + + +def deduce_and_validate_param_type( + name: str, details: dict[str, str | int | float] +) -> str: + """ + Deduces the parameter type from details and validates them. + + Args: + name (str): The name of the parameter. + details (dict): A dictionary containing parameter specifications. + + Returns: + str: The deduced parameter type ('int', 'float', 'categorical', or 'constant'). + + Raises: + TypeError: If the type cannot be deduced or the details don't align with expected + constraints. + """ + # Deduce type + if "type" in details: + param_type = details["type"].lower() + else: + # Logic to infer type if not explicitly provided + param_type = deduce_param_type(name, details) + + # Validate details of a parameter based on (deduced) type + validate_param_details(name, param_type, details) + + return param_type + + +def deduce_param_type(name: str, details: dict[str, int | str | float]) -> str: + """Deduces the parameter type based on the provided details. + + The function interprets the 'details' dictionary to determine the parameter type. + The dictionary should include key-value pairs that describe the parameter's + characteristics, such as lower, upper, default value, or possible choices. + + + Args: + name (str): The name of the parameter. + details ((dict[str, int | str | float])): A dictionary containing parameter + specifications. + + Returns: + str: The deduced parameter type ('int', 'float', 'categorical', or 'constant'). + + Raises: + TypeError: If the parameter type cannot be deduced from the details, or if the + provided details have inconsistent types for expected keys. + + Example: + param_type = deduce_param_type('example_param', {'lower': 0, 'upper': 10})""" + # Logic to deduce type from details + + # check for int and float conditions + if "lower" in details and "upper" in details: + # Determine if it's an integer or float range parameter + if isinstance(details["lower"], int) and isinstance(details["upper"], int): + param_type = "int" + elif isinstance(details["lower"], float) and isinstance(details["upper"], float): + param_type = "float" + else: + try: + details["lower"], flag_lower = convert_scientific_notation( + details["lower"], show_usage_flag=True + ) + details["upper"], flag_upper = convert_scientific_notation( + details["upper"], show_usage_flag=True + ) + except ValueError as e: + raise TypeError( + f"Inconsistent types for 'lower' and 'upper' in '{name}'. " + f"Both must be either integers or floats." + ) from e + + # check if one value is e notation and if so convert it to float + if flag_lower or flag_upper: + param_type = "float" + else: + raise TypeError( + f"Inconsistent types for 'lower' and 'upper' in '{name}'. " + f"Both must be either integers or floats." + ) + # check for categorical condition + elif "choices" in details: + param_type = "categorical" + + # check for constant condition + elif "value" in details: + param_type = "constant" + else: + raise KeyError( + f"Unable to deduce parameter type from {name} " + f"with details {details}\n" + "Supported parameters:\n" + "Float and Integer: Expected keys: 'lower', 'upper'\n" + "Categorical: Expected keys: 'choices'\n" + "Constant: Expected keys: 'value'" + ) + return param_type + + +def validate_param_details( + name: str, param_type: str, details: dict[str, int | str | float] +): + """ + Validates the details of a parameter based on its type. + + This function checks the format and type-specific details of a parameter + specified in a YAML file. It ensures that the 'name' of the parameter is a string + and its 'details' are provided as a dictionary. Depending on the parameter type, + it delegates the validation to the appropriate type-specific validation function. + + Parameters: + name (str): The name of the parameter. It should be a string. + param_type (str): The type of the parameter. Supported types are 'int' (or 'integer'), + 'float', 'cat' (or 'categorical'), and 'const' (or 'constant'). + details (dict): The detailed configuration of the parameter, which includes its + attributes like 'lower', 'upper', 'default', etc. + + Raises: + KeyError: If the 'name' is not a string or 'details' is not a dictionary, or if + the necessary keys in the 'details' are missing based on the parameter type. + TypeError: If the 'param_type' is not one of the supported types. + + Example Usage: + validate_param_details("learning_rate", "float", {"lower": 0.01, "upper": 0.1, + "default": 0.05}) + """ + if not (isinstance(name, str) and isinstance(details, dict)): + raise KeyError( + f"Invalid format for {name} in YAML file. " + f"Expected 'name' as string and corresponding 'details' as a " + f"dictionary. Found 'name' type: {type(name).__name__}, 'details' " + f"type: {type(details).__name__}." + ) + param_type = param_type.lower() + # init parameter by checking type + if param_type in ("int", "integer"): + validate_integer_parameter(name, details) + + elif param_type == "float": + validate_float_parameter(name, details) + + elif param_type in ("cat", "categorical"): + validate_categorical_parameter(name, details) + + elif param_type in ("const", "constant"): + validate_constant_parameter(name, details) + else: + # Handle unknown parameter types + raise TypeError( + f"Unsupported parameter type'{details['type']}' for '{name}'.\n" + f"Supported Types for argument type are:\n" + "For integer parameter: int, integer\n" + "For float parameter: float\n" + "For categorical parameter: cat, categorical\n" + "For constant parameter: const, constant\n" + ) + + +def validate_integer_parameter(name: str, details: dict[str, str | int | float]): + """ + Validates and processes an integer parameter's details, converting scientific + notation to integers where necessary. + + This function checks the type of 'lower' and 'upper', and the 'default' + value (if present) for an integer parameter. It also handles conversion of values + in scientific notation (e.g., 1e2) to integers. + + Args: + name (str): The name of the integer parameter. + details (dict[str, str | int | float]): A dictionary containing the parameter's + specifications. Expected keys include + 'lower', 'upper', and optionally 'default', + among others. + + Raises: + TypeError: If 'lower', 'upper', or 'default' are not valid integers or cannot + be converted from scientific notation to integers. + """ + # check if all keys are allowed to use and if the mandatory ones are provided + check_keys( + name, + details, + {"lower", "upper", "type", "log", "is_fidelity", "default", "default_confidence"}, + {"lower", "upper"}, + ) + + if not isinstance(details["lower"], int) or not isinstance(details["upper"], int): + try: + # for numbers like 1e2 and 10^ + lower, flag_lower = convert_scientific_notation( + details["lower"], show_usage_flag=True + ) + upper, flag_upper = convert_scientific_notation( + details["upper"], show_usage_flag=True + ) + # check if one value format is e notation and if its an integer + if flag_lower or flag_upper: + if lower == int(lower) and upper == int(upper): + details["lower"] = int(lower) + details["upper"] = int(upper) + else: + raise TypeError() + else: + raise TypeError() + except (ValueError, TypeError) as e: + raise TypeError( + f"'lower' and 'upper' must be integer for " f"integer parameter '{name}'." + ) from e + if "default" in details: + if not isinstance(details["default"], int): + try: + # convert value can raise ValueError + default = convert_scientific_notation(details["default"]) + if default == int(default): + details["default"] = int(default) + else: + raise TypeError() # type of value is not int + except (ValueError, TypeError) as e: + raise TypeError( + f"default value {details['default']} " + f"must be integer for integer parameter {name}" + ) from e + + +def validate_float_parameter(name: str, details: dict[str, str | int | float]): + """ + Validates and processes a float parameter's details, converting scientific + notation values to float where necessary. + + This function checks the type of 'lower' and 'upper', and the 'default' + value (if present) for a float parameter. It handles conversion of values in + scientific notation (e.g., 1e-5) to float. + + Args: + name: The name of the float parameter. + details: A dictionary containing the parameter's specifications. Expected keys + include 'lower', 'upper', and optionally 'default', among others. + + Raises: + TypeError: If 'lower', 'upper', or 'default' are not valid floats or cannot + be converted from scientific notation to floats. + """ + # check if all keys are allowed to use and if the mandatory ones are provided + check_keys( + name, + details, + {"lower", "upper", "type", "log", "is_fidelity", "default", "default_confidence"}, + {"lower", "upper"}, + ) + + if not isinstance(details["lower"], float) or not isinstance(details["upper"], float): + try: + # for numbers like 1e-5 and 10^ + details["lower"] = convert_scientific_notation(details["lower"]) + details["upper"] = convert_scientific_notation(details["upper"]) + except ValueError as e: + raise TypeError( + f"'lower' and 'upper' must be integer for " f"integer parameter '{name}'." + ) from e + if "default" in details: + if not isinstance(details["default"], float): + try: + details["default"] = convert_scientific_notation(details["default"]) + except ValueError as e: + raise TypeError( + f" default'{details['default']}' must be float for float " + f"parameter {name} " + ) from e + + +def validate_categorical_parameter(name: str, details: dict[str, str | int | float]): + """ + Validates a categorical parameter, including conversion of scientific notation + values to floats within the choices. + + This function ensures that the 'choices' key in the details is a list and attempts + to convert any elements in scientific notation to floats. It also handles the + 'default' value, converting it from scientific notation if necessary. + + Args: + name: The name of the categorical parameter. + details: A dictionary containing the parameter's specifications. Required key + is 'choices', with 'default' being optional. + + Raises: + TypeError: If 'choices' is not a list + """ + # check if all keys are allowed to use and if the mandatory ones are provided + check_keys( + name, + details, + {"choices", "type", "is_fidelity", "default", "default_confidence"}, + {"choices"}, + ) + + if not isinstance(details["choices"], list): + raise TypeError(f"The 'choices' for '{name}' must be a list.") + for i, element in enumerate(details["choices"]): + try: + converted_value, e_flag = convert_scientific_notation( + element, show_usage_flag=True + ) + if e_flag: + details["choices"][ + i + ] = converted_value # Replace the element at the same position + except ValueError: + pass # If a ValueError occurs, simply continue to the next element + if "default" in details: + e_flag = False + try: + # check if e notation, if then convert to number + default, e_flag = convert_scientific_notation( + details["default"], show_usage_flag=True + ) + except ValueError: + pass # if default value is not in a numeric format, Value Error occurs + if e_flag is True: + details["default"] = default + + +def validate_constant_parameter(name: str, details: dict[str, str | int | float]): + """ + Validates a constant parameter, including conversion of values in scientific + notation to floats. + + This function checks the 'value' key in the details dictionary and converts any + value expressed in scientific notation to a float. It ensures that the mandatory + 'value' key is provided and appropriately formatted. + + Args: + name: The name of the constant parameter. + details: A dictionary containing the parameter's specifications. The required + key is 'value'. + """ + # check if all keys are allowed to use and if the mandatory ones are provided + check_keys(name, details, {"value", "type", "is_fidelity"}, {"value"}) + + # check for e notation and convert it to float + e_flag = False + try: + converted_value, e_flag = convert_scientific_notation( + details["value"], show_usage_flag=True + ) + except ValueError: + # if the value is not able to convert to float a ValueError get raised by + # convert_scientific_notation function + pass + if e_flag: + details["value"] = converted_value + + +def check_keys( + name: str, + details: dict[str, str | int | float], + allowed_keys: set, + mandatory_keys: set, +): + """ + Checks if all keys in 'my_dict' are contained in the set 'allowed_keys' and + if all keys in 'mandatory_keys' are present in 'my_dict'. + Raises an exception if an unallowed key is found or if a mandatory key is missing. + """ + # Check for unallowed keys + unallowed_keys = [key for key in details if key not in allowed_keys] + if unallowed_keys: + unallowed_keys_str = ", ".join(unallowed_keys) + raise KeyError( + f"Unallowed key(s) '{unallowed_keys_str}' found for parameter '" f"{name}'." + ) + + # Check for missing mandatory keys + missing_mandatory_keys = [key for key in mandatory_keys if key not in details] + if missing_mandatory_keys: + missing_keys_str = ", ".join(missing_mandatory_keys) + raise KeyError( + f"Missing mandatory key(s) '{missing_keys_str}' for parameter '" f"{name}'." + ) diff --git a/src/neps/status/__main__.py b/neps/status/__main__.py similarity index 100% rename from src/neps/status/__main__.py rename to neps/status/__main__.py diff --git a/src/neps/status/status.py b/neps/status/status.py similarity index 93% rename from src/neps/status/status.py rename to neps/status/status.py index 90832427..3d45d2fe 100644 --- a/src/neps/status/status.py +++ b/neps/status/status.py @@ -3,13 +3,13 @@ import logging from pathlib import Path from typing import Any +import shutil +import zipfile import pandas as pd -from metahyper import read -from metahyper._locker import Locker -from metahyper.api import ConfigResult - +from ..metahyper import ConfigResult, read +from ..metahyper._locker import Locker from ..search_spaces.search_space import SearchSpace from ..utils.result_utils import get_loss @@ -206,8 +206,8 @@ def _get_dataframes_from_summary( config_data_pending.append(config_pending) # Creating the dataframe for previous config results. - df_previous = pd.DataFrame({"Config_id": indices_prev}) - df_previous["Status"] = "Complete" + df_previous = pd.DataFrame({"config_id": indices_prev}) + df_previous["status"] = "complete" df_previous = pd.concat( [df_previous, pd.json_normalize(config_data_prev).add_prefix("config.")], axis=1 ) @@ -220,8 +220,8 @@ def _get_dataframes_from_summary( ) # Creating dataframe for pending configs. - df_pending = pd.DataFrame({"Config_id": indices_pending}) - df_pending["Status"] = "Pending" + df_pending = pd.DataFrame({"config_id": indices_pending}) + df_pending["status"] = "pending" df_pending = pd.concat( [df_pending, pd.json_normalize(config_data_pending).add_prefix("config.")], axis=1, @@ -295,7 +295,7 @@ def _save_data_to_csv( run_data_df.index == "num_evaluated_configs", "value" ] # checks if the current worker has more evaluated configs than the previous - if int(num_evaluated_configs_csv) < int(num_evaluated_configs_run): + if int(num_evaluated_configs_csv) < int(num_evaluated_configs_run.iloc[0]): config_data_df = config_data_df.sort_values( by="result.loss", ascending=True ) @@ -318,6 +318,17 @@ def _save_data_to_csv( def post_run_csv(root_directory: str | Path, logger=None) -> None: + + root_directory = Path(root_directory) + zip_filename = Path(root_directory / "results.zip") + base_result_directory =root_directory / "results" + + # Extract previous results to load if it exists + if zip_filename.exists(): + # and not any(Path(base_result_directory).iterdir()): + shutil.unpack_archive(zip_filename, base_result_directory, "zip") + zip_filename.unlink() + if logger is None: logger = logging.getLogger("neps_status") @@ -339,3 +350,6 @@ def post_run_csv(root_directory: str | Path, logger=None) -> None: df_config_data, df_run_data, ) + +def get_run_summary_csv(root_directory: str | Path): + post_run_csv(root_directory=root_directory) diff --git a/src/neps/utils/common.py b/neps/utils/common.py similarity index 82% rename from src/neps/utils/common.py rename to neps/utils/common.py index 62b95e2a..2780a7ca 100644 --- a/src/neps/utils/common.py +++ b/neps/utils/common.py @@ -1,16 +1,17 @@ from __future__ import annotations import glob +import json import os import random from pathlib import Path +from typing import Any import numpy as np import torch import yaml -from metahyper.api import ConfigInRun - +from ..metahyper.api import ConfigInRun from ..optimizers.info import SearcherConfigs @@ -231,6 +232,17 @@ def get_searcher_data(searcher: str, searcher_path: Path | str | None = None) -> return data +def get_value(obj: Any): + if isinstance(obj, (str, int, float, bool, type(None))): + return obj + elif isinstance(obj, dict): + return {key: get_value(value) for key, value in obj.items()} + elif isinstance(obj, list): + return [get_value(item) for item in obj] + else: + return obj.__name__ + + def has_instance(collection, *types): return any([isinstance(el, typ) for el in collection for typ in types]) @@ -274,3 +286,60 @@ class AttrDict(dict): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__dict__ = self + + +class DataWriter: + """ + A class to specify how to save/write a data to the folder by + implementing your own write_data function. + Use the set_attributes function to set all your necessary attributes and the data + and then write_data will be called with only the directory path as argument + during the write process + """ + def __init__(self, name: str): + self.name = name + + def set_attributes(self, attribute_dict: dict[str, Any]): + for attribute_name, attribute in attribute_dict.items(): + setattr(self, attribute_name, attribute) + + def write_data(self, to_directory: Path): + raise NotImplementedError + + +class EvaluationData: + """ + A class to store some data for a single evaluation (configuration) + and write that data to its corresponding config folder + """ + def __init__(self): + self.data_dict: dict[str, DataWriter] = {} + + def write_all(self, directory: Path): + for _, data_writer in self.data_dict.items(): + data_writer.write_data(directory) + + +class SimpleCSVWriter(DataWriter): + def write_data(self, to_directory: Path): + # self.df: pd.DataFrame = pd.DataFrame() + path = to_directory / str(self.name + ".csv") + self.df.to_csv(path, float_format="%g") + + +class SimpleJSONWriter(DataWriter): + def __init__(self): + self.data: dict[str, Any] = {} + + def write_data(self, to_directory: Path): + # self.df: pd.DataFrame = pd.DataFrame() + path = to_directory / str(self.name + ".json") + with open(path, "w") as file: + json.dump(self.data, file) + + + + + + + diff --git a/src/neps/utils/data_loading.py b/neps/utils/data_loading.py similarity index 97% rename from src/neps/utils/data_loading.py rename to neps/utils/data_loading.py index ed2712bc..996d1583 100644 --- a/src/neps/utils/data_loading.py +++ b/neps/utils/data_loading.py @@ -9,8 +9,7 @@ import numpy as np import yaml -import metahyper.api - +from ..metahyper import read from .result_utils import get_loss @@ -40,7 +39,7 @@ def read_tasks_and_dev_stages_from_disk( continue dev_id = get_id_from_path(dev_dir_path) # TODO: Perhaps use 2nd and 3rd argument as well - result, _, _ = metahyper.api.read(dev_dir_path) + result, _, _ = read(dev_dir_path) results[task_id][dev_id] = result return results @@ -65,7 +64,7 @@ def read_user_prior_results_from_disk(path: str): continue # get name of the directory name = os.path.basename(prior_dir_path) - results[name], _, _ = metahyper.api.read(prior_dir_path) + results[name], _, _ = read(prior_dir_path) return results @@ -163,7 +162,7 @@ def summarize_results( # TOOD: only use IDs if provided final_results = results[final_task_id][final_dev_id] else: - final_results, _, _ = metahyper.api.read(seed_dir_path) + final_results, _, _ = read(seed_dir_path) # This part is copied from neps.status() best_loss = float("inf") diff --git a/src/neps/utils/result_utils.py b/neps/utils/result_utils.py similarity index 100% rename from src/neps/utils/result_utils.py rename to neps/utils/result_utils.py diff --git a/neps_examples/README.md b/neps_examples/README.md index f2173a54..4161d7ec 100644 --- a/neps_examples/README.md +++ b/neps_examples/README.md @@ -1,11 +1,11 @@ # Neural Pipeline Search Examples -1. Examples for the [basic usage](basic_usage) showing how to perform HPO / NAS / JAHS and how to analyse runs. +1. Navigate to [basic_usage](basic_usage) for demonstrations on fundamental usage. Learn how to perform Hyperparameter Optimization (HPO), Neural Architecture Search (NAS), and Joint Architecture and Hyperparameter Search (JAHS). Understand how to analyze runs on a basic level, emphasizing that no neural network training is involved at this stage; the search is performed on functions to introduce NePS. -2. Examples for how to [boost efficiency](efficiency) with expert_priors and/or multi fidelity and/or parallelization. +2. Navigate to [efficiency](efficiency) examples showcasing how to enhance efficiency in NePS. Learn about expert priors, multi-fidelity, and parallelization to streamline your pipeline and optimize search processes. -3. Examples showcasing some [utilities for your convenience](convenience). +3. Navigate to [convenience](convenience) for examples highlighting utilities that add extra features to NePS. Discover tensorboard compatibility and its integration, explore the compatibility with PyTorch Lightning, and understand file management within the run pipeline function used in NePS. -4. [Experimental examples](experimental) which are mostly useful for NePS contributors. +4. Navigate to into [experimental](experimental) examples tailored for NePS contributors. These examples provide insights and practices for experimental scenarios. -5. [template](template) which is a template for creating a neps program. +5. Navigate to [template](template) to find a basic fill-in template to kickstart your hyperparameter search with NePS. Use this template as a foundation for your projects, saving time and ensuring a structured starting point. diff --git a/neps_examples/basic_usage/defining_search_space/hpo_usage_example.py b/neps_examples/basic_usage/defining_search_space/hpo_usage_example.py new file mode 100644 index 00000000..a121d4df --- /dev/null +++ b/neps_examples/basic_usage/defining_search_space/hpo_usage_example.py @@ -0,0 +1,37 @@ +import logging +import time + +import numpy as np + +import neps + + +def run_pipeline( + float_name1, + float_name2, + categorical_name1, + categorical_name2, + integer_name1, + integer_name2, +): + # neps optimize to find values that maximizes sum, for demonstration only + loss = -float( + np.sum( + [float_name1, float_name2, categorical_name1, integer_name1, integer_name2] + ) + ) + if categorical_name2 == "a": + loss += 1 + + time.sleep(2) # For demonstration purposes only + return loss + + +logging.basicConfig(level=logging.INFO) +neps.run( + run_pipeline=run_pipeline, + pipeline_space="search_space_example.yaml", + root_directory="results/hyperparameters_example", + post_run_summary=True, + max_evaluations_total=15, +) diff --git a/neps_examples/basic_usage/defining_search_space/search_space_example.yaml b/neps_examples/basic_usage/defining_search_space/search_space_example.yaml new file mode 100644 index 00000000..27bd37c1 --- /dev/null +++ b/neps_examples/basic_usage/defining_search_space/search_space_example.yaml @@ -0,0 +1,26 @@ +search_space: + float_name1: + lower: 3e-5 + upper: 0.1 + + float_name2: + type: "float" # Optional, as neps infers type from 'lower' and 'upper' + lower: 1.7 + upper: 42.0 + log: true # Optional, default: False + + categorical_name1: + choices: [0, 1] + + categorical_name2: + type: cat + choices: ["a", "b", "c"] + + integer_name1: + lower: 32 + upper: 128 + is_fidelity: True # Optional, default: False + + integer_name2: + lower: -5 + upper: 5 diff --git a/neps_examples/convenience/neps_tblogger_tutorial.py b/neps_examples/convenience/neps_tblogger_tutorial.py index ea02a8dd..a70cc494 100644 --- a/neps_examples/convenience/neps_tblogger_tutorial.py +++ b/neps_examples/convenience/neps_tblogger_tutorial.py @@ -353,7 +353,7 @@ def run_pipeline(lr, optim, weight_decay): run_args = dict( run_pipeline=run_pipeline, pipeline_space=pipeline_space(), - root_directory="output", + root_directory="results/neps_tblogger_example", searcher="random_search", ) diff --git a/neps_examples/efficiency/multi_fidelity.py b/neps_examples/efficiency/multi_fidelity.py index 4239aee7..0731b1b5 100644 --- a/neps_examples/efficiency/multi_fidelity.py +++ b/neps_examples/efficiency/multi_fidelity.py @@ -37,6 +37,11 @@ def get_model_and_optimizer(learning_rate): return model, optimizer +# Important: Include the "pipeline_directory" and "previous_pipeline_directory" arguments +# in your run_pipeline function. This grants access to NePS's folder system and is +# critical for leveraging efficient multi-fidelity optimization strategies. + + def run_pipeline(pipeline_directory, previous_pipeline_directory, learning_rate, epoch): model, optimizer = get_model_and_optimizer(learning_rate) checkpoint_name = "checkpoint.pth" diff --git a/neps_examples/experimental/hierarchical_architecture_hierarchical_GP.py b/neps_examples/experimental/hierarchical_architecture_hierarchical_GP.py index 6fea9195..3db93bde 100644 --- a/neps_examples/experimental/hierarchical_architecture_hierarchical_GP.py +++ b/neps_examples/experimental/hierarchical_architecture_hierarchical_GP.py @@ -135,6 +135,7 @@ def run_pipeline(architecture): pipeline_space=pipeline_space, root_directory="results/hierarchical_architecture_example_new", max_evaluations_total=15, + searcher="bayesian_optimization", surrogate_model=surrogate_model, surrogate_model_args=surrogate_model_args, ) diff --git a/pyproject.toml b/pyproject.toml index 19c4336f..2867b998 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "neural-pipeline-search" -version = "v0.10.0" +version = "v0.11.1" description = "Neural Pipeline Search helps deep learning experts find the best neural pipeline." authors = [ "Danny Stoll ", @@ -27,6 +27,7 @@ classifiers = [ "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", "Operating System :: POSIX :: Linux", "Operating System :: Microsoft :: Windows", "Operating System :: MacOS", @@ -38,48 +39,53 @@ classifiers = [ "Topic :: System :: Distributed Computing", ] packages = [ - { include = "neps", from = "src" }, - { include = "metahyper", from = "src" }, + { include = "neps" }, + { include = "metahyper", from = "neps"}, { include = "neps_examples" }, ] - [tool.poetry.dependencies] python = ">=3.8,<3.12" -ConfigSpace = "^0.4.19" -grakel = "^0.1.9" -numpy = "^1.23.0" -pandas = "^1.3.1" +ConfigSpace = "^0.7" +grakel = "^0.1" +numpy = "^1" +pandas = "^2" networkx = "^2.6.3" nltk = "^3.6.4" -path = "^16.2.0" -termcolor = "^1.1.0" -scipy = "^1.8" +#path = "^16.2.0" +#termcolor = "^1.1.0" +scipy = "^1" torch = ">=1.7.0,<=2.1, !=2.0.1, !=2.1.0" # fix from: https://stackoverflow.com/a/76647180 # torch = [ # {version = ">=1.7.0,<=2.1", markers = "sys_platform == 'darwin'"}, # Segfaults for macOS on github actions # {version = ">=1.7.0,<=2.1", markers = "sys_platform != 'darwin'"}, # ] -matplotlib = "^3.4" -statsmodels = "^0.13.2" -more-itertools = "^9.0.0" -portalocker = "^2.6.0" -seaborn = "^0.12.1" -pyyaml = "^6.0" -tensorboard = "^2.13" -cython = "^3.0.4" -torchvision = "<0.16.0" +matplotlib = "^3" +# statsmodels = "^0.13.2" +more-itertools = "^10" +portalocker = "^2" +seaborn = "^0.13" +pyyaml = "^6" +tensorboard = "^2" +# cython = "^3.0.4" [tool.poetry.group.dev.dependencies] -pre-commit = "^2.10" -mypy = "^0.930" -pytest = "^6.2.5" -types-PyYAML = "^6.0.12" -typing-extensions = "^4.0.1" -types-termcolor = "^1.1.2" +pre-commit = "^3" +mypy = "^1" +pytest = "^7" +types-PyYAML = "^6" +typing-extensions = "^4" +#types-termcolor = "^1.1.2" # jahs-bench = {git = "https://github.com/automl/jahs_bench_201.git", rev = "v1.0.2"} mkdocs-material = "^8.1.3" mike = "^1.1.2" +torchvision = "<0.16.0" # Used in examples + + +[tool.poetry.group.experimental] +optional = true + +[tool.poetry.group.experimental.dependencies] gpytorch = "1.8.0" [build-system] @@ -95,8 +101,8 @@ profile = 'black' line_length = 90 [tool.pytest.ini_options] -addopts = "--basetemp ./tests_tmpdir -m 'core_examples or yaml_api'" -markers = ["all_examples", "core_examples", "regression_all", "metahyper", "yaml_api", "summary_csv"] +addopts = "--basetemp ./tests_tmpdir -m 'neps_api or core_examples'" +markers = ["all_examples", "core_examples", "regression_all", "metahyper", "neps_api", "summary_csv"] filterwarnings = "ignore::DeprecationWarning:torch.utils.tensorboard.*:" [tool.mypy] diff --git a/src/metahyper/__init__.py b/src/metahyper/__init__.py deleted file mode 100644 index 4bcb998e..00000000 --- a/src/metahyper/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .api import ConfigResult, Sampler, read, run -from .utils import instance_from_map diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py b/src/neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py deleted file mode 100644 index aedb483f..00000000 --- a/src/neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -from functools import partial -from typing import Callable - -from .ei import ComprehensiveExpectedImprovement -from .mf_ei import MFEI -from .ucb import UpperConfidenceBound, MF_UCB - - -AcquisitionMapping: dict[str, Callable] = { - "EI": partial( - ComprehensiveExpectedImprovement, - in_fill="best", - augmented_ei=False, - ), - "LogEI": partial( - ComprehensiveExpectedImprovement, - in_fill="best", - augmented_ei=False, - log_ei=True, - ), - # # Uses the augmented EI heuristic and changed the in-fill criterion to the best test location with - # # the highest *posterior mean*, which are preferred when the optimisation is noisy. - "AEI": partial( - ComprehensiveExpectedImprovement, - in_fill="posterior", - augmented_ei=True, - ), - "MFEI": partial( - MFEI, - in_fill="best", - augmented_ei=False, - ), - "UCB": partial( - UpperConfidenceBound, - maximize=False, - ), - "MF-UCB": partial( - MF_UCB, - maximize=False, - ), -} diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py b/src/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py deleted file mode 100644 index f677f894..00000000 --- a/src/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py +++ /dev/null @@ -1,205 +0,0 @@ -# type: ignore -from typing import Any, Iterable, Tuple, Union - -import numpy as np -import pandas as pd -import torch -from torch.distributions import Normal - -from ....optimizers.utils import map_real_hyperparameters_from_tabular_ids -from ....search_spaces.search_space import SearchSpace -from ...multi_fidelity.utils import MFObservedData -from .ei import ComprehensiveExpectedImprovement - - -class MFEI(ComprehensiveExpectedImprovement): - def __init__( - self, - pipeline_space: SearchSpace, - surrogate_model_name: str = None, - augmented_ei: bool = False, - xi: float = 0.0, - in_fill: str = "best", - log_ei: bool = False, - ): - super().__init__(augmented_ei, xi, in_fill, log_ei) - self.pipeline_space = pipeline_space - self.surrogate_model_name = surrogate_model_name - self.surrogate_model = None - self.observations = None - self.b_step = None - - def get_budget_level(self, config) -> int: - return int((config.fidelity.value - config.fidelity.lower) / self.b_step) - - def preprocess(self, x: pd.Series) -> Tuple[Iterable, Iterable]: - """Prepares the configurations for appropriate EI calculation. - - Takes a set of points and computes the budget and incumbent for each point, as - required by the multi-fidelity Expected Improvement acquisition function. - """ - budget_list = [] - - if self.pipeline_space.has_tabular: - # preprocess tabular space differently - # expected input: IDs pertaining to the tabular data - # expected output: IDs pertaining to current observations and set of HPs - x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) - indices_to_drop = [] - for i, config in x.items(): - target_fidelity = config.fidelity.lower - if i <= max(self.observations.seen_config_ids): - # IMPORTANT to set the fidelity at which EI will be calculated only for - # the partial configs that have been observed already - target_fidelity = config.fidelity.value + self.b_step - - if np.less_equal(target_fidelity, config.fidelity.upper): - # only consider the configs with fidelity lower than the max fidelity - config.fidelity.value = target_fidelity - budget_list.append(self.get_budget_level(config)) - else: - # if the target_fidelity higher than the max drop the configuration - indices_to_drop.append(i) - else: - config.fidelity.value = target_fidelity - budget_list.append(self.get_budget_level(config)) - - # Drop unused configs - x.drop(labels=indices_to_drop, inplace=True) - - performances = self.observations.get_best_performance_for_each_budget() - inc_list = [] - for budget_level in budget_list: - if budget_level in performances.index: - inc = performances[budget_level] - else: - inc = self.observations.get_best_seen_performance() - inc_list.append(inc) - - return x, torch.Tensor(inc_list) - - def preprocess_gp(self, x: Iterable) -> Tuple[Iterable, Iterable]: - x, inc_list = self.preprocess(x) - return x.values.tolist(), inc_list - - def preprocess_deep_gp(self, x: Iterable) -> Tuple[Iterable, Iterable]: - x, inc_list = self.preprocess(x) - x_lcs = [] - for idx in x.index: - if idx in self.observations.df.index.levels[0]: - budget_level = self.get_budget_level(x[idx]) - lc = self.observations.extract_learning_curve(idx, budget_level) - else: - # initialize a learning curve with a place holder - # This is later padded accordingly for the Conv1D layer - lc = [0.0] - x_lcs.append(lc) - self.surrogate_model.set_prediction_learning_curves(x_lcs) - return x.values.tolist(), inc_list - - def preprocess_pfn(self, x: Iterable) -> Tuple[Iterable, Iterable, Iterable]: - """Prepares the configurations for appropriate EI calculation. - - Takes a set of points and computes the budget and incumbent for each point, as - required by the multi-fidelity Expected Improvement acquisition function. - """ - _x, inc_list = self.preprocess(x.copy()) - _x_tok = self.observations.tokenize(_x, as_tensor=True) - len_partial = len(self.observations.seen_config_ids) - z_min = x[0].fidelity.lower - # converting fidelity to the discrete budget level - # STRICT ASSUMPTION: fidelity is the first dimension - _x_tok[:len_partial, 0] = ( - _x_tok[:len_partial, 0] + self.b_step - z_min - ) / self.b_step - return _x_tok, _x, inc_list - - def eval(self, x: pd.Series, asscalar: bool = False) -> Tuple[np.ndarray, pd.Series]: - # _x = x.copy() # preprocessing needs to change the reference x Series so we don't copy here - if self.surrogate_model_name == "pfn": - _x_tok, _x, inc_list = self.preprocess_pfn( - x.copy() - ) # IMPORTANT change from vanilla-EI - ei = self.eval_pfn_ei(_x_tok, inc_list) - elif self.surrogate_model_name == "deep_gp": - _x, inc_list = self.preprocess_deep_gp( - x.copy() - ) # IMPORTANT change from vanilla-EI - ei = self.eval_gp_ei(_x, inc_list) - _x = pd.Series(_x, index=np.arange(len(_x))) - else: - _x, inc_list = self.preprocess_gp( - x.copy() - ) # IMPORTANT change from vanilla-EI - ei = self.eval_gp_ei(_x, inc_list) - _x = pd.Series(_x, index=np.arange(len(_x))) - - if ei.is_cuda: - ei = ei.cpu() - if len(x) > 1 and asscalar: - return ei.detach().numpy(), _x - else: - return ei.detach().numpy().item(), _x - - def eval_pfn_ei( - self, x: Iterable, inc_list: Iterable - ) -> Union[np.ndarray, torch.Tensor, float]: - """PFN-EI modified to preprocess samples and accept list of incumbents.""" - # x, inc_list = self.preprocess(x) # IMPORTANT change from vanilla-EI - # _x = x.copy() - ei = self.surrogate_model.get_ei(x.to(self.surrogate_model.device), inc_list) - if len(ei.shape) == 2: - ei = ei.flatten() - return ei - - def eval_gp_ei( - self, x: Iterable, inc_list: Iterable - ) -> Union[np.ndarray, torch.Tensor, float]: - """Vanilla-EI modified to preprocess samples and accept list of incumbents.""" - # x, inc_list = self.preprocess(x) # IMPORTANT change from vanilla-EI - _x = x.copy() - try: - mu, cov = self.surrogate_model.predict(_x) - except ValueError as e: - raise e - # return -1.0 # in case of error. return ei of -1 - std = torch.sqrt(torch.diag(cov)) - - mu_star = inc_list.to(mu.device) # IMPORTANT change from vanilla-EI - - gauss = Normal(torch.zeros(1, device=mu.device), torch.ones(1, device=mu.device)) - # u = (mu - mu_star - self.xi) / std - # ei = std * updf + (mu - mu_star - self.xi) * ucdf - if self.log_ei: - # we expect that f_min is in log-space - f_min = mu_star - self.xi - v = (f_min - mu) / std - ei = torch.exp(f_min) * gauss.cdf(v) - torch.exp( - 0.5 * torch.diag(cov) + mu - ) * gauss.cdf(v - std) - else: - u = (mu_star - mu - self.xi) / std - ucdf = gauss.cdf(u) - updf = torch.exp(gauss.log_prob(u)) - ei = std * updf + (mu_star - mu - self.xi) * ucdf - if self.augmented_ei: - sigma_n = self.surrogate_model.likelihood - ei *= 1.0 - torch.sqrt(torch.tensor(sigma_n, device=mu.device)) / torch.sqrt( - sigma_n + torch.diag(cov) - ) - return ei - - def set_state( - self, - pipeline_space: SearchSpace, - surrogate_model: Any, - observations: MFObservedData, - b_step: Union[int, float], - **kwargs, - ): - # overload to select incumbent differently through observations - self.pipeline_space = pipeline_space - self.surrogate_model = surrogate_model - self.observations = observations - self.b_step = b_step - return diff --git a/src/neps/optimizers/bayesian_optimization/models/__init__.py b/src/neps/optimizers/bayesian_optimization/models/__init__.py deleted file mode 100755 index 799545cd..00000000 --- a/src/neps/optimizers/bayesian_optimization/models/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from .deepGP import DeepGP -from .gp import ComprehensiveGP -from .gp_hierarchy import ComprehensiveGPHierarchy - - -SurrogateModelMapping = { - "deep_gp": DeepGP, - "gp": ComprehensiveGP, - "gp_hierarchy": ComprehensiveGPHierarchy, -} - -try: - from .pfn import PFN_SURROGATE # only if available locally - SurrogateModelMapping.update({"pfn": PFN_SURROGATE}) -except: - pass diff --git a/src/neps/optimizers/utils.py b/src/neps/optimizers/utils.py deleted file mode 100644 index c203f4db..00000000 --- a/src/neps/optimizers/utils.py +++ /dev/null @@ -1,47 +0,0 @@ -import pandas as pd - -from ..search_spaces.search_space import SearchSpace - - -# def map_real_hyperparameters_from_tabular_ids( -# ids: pd.Series, pipeline_space: SearchSpace -# ) -> pd.Series: -# return x - - -def map_real_hyperparameters_from_tabular_ids( - x: pd.Series, pipeline_space: SearchSpace -) -> pd.Series: - """ Maps the tabular IDs to the actual HPs from the pipeline space. - - Args: - x (pd.Series): A pandas series with the tabular IDs. - TODO: Mention expected format of the series. - pipeline_space (SearchSpace): The pipeline space. - - Returns: - pd.Series: A pandas series with the actual HPs. - TODO: Mention expected format of the series. - """ - if len(x) == 0: - return x - # extract fid name - _x = x.iloc[0].hp_values() - _x.pop("id") - fid_name = list(_x.keys())[0] - for i in x.index.values: - # extracting actual HPs from the tabular space - _config = pipeline_space.custom_grid_table.loc[x.loc[i]["id"].value].to_dict() - # updating fidelities as per the candidate set passed - _config.update({fid_name: x.loc[i][fid_name].value}) - # placeholder config from the raw tabular space - config = pipeline_space.raw_tabular_space.sample( - patience=100, - user_priors=True, - ignore_fidelity=True # True allows fidelity to appear in the sample - ) - # copying values from table to placeholder config of type SearchSpace - config.load_from(_config) - # replacing the ID in the candidate set with the actual HPs of the config - x.loc[i] = config - return x diff --git a/tests/test_metahyper/test_locking.py b/tests/test_metahyper/test_locking.py index 2cd09499..dfdfcb05 100644 --- a/tests/test_metahyper/test_locking.py +++ b/tests/test_metahyper/test_locking.py @@ -8,7 +8,7 @@ from more_itertools import first_true -def launch_example_processes(n_workers: int=3) -> list: +def launch_example_processes(n_workers: int = 3) -> list: processes = [] for _ in range(n_workers): processes.append( @@ -90,7 +90,7 @@ def test_summary_csv(): config_data_df = pd.read_csv(summary_dir / "config_data.csv") assert config_data_df.shape[0] == 15 - assert (config_data_df["Status"] == "Complete").all() + assert (config_data_df["status"] == "complete").all() except Exception as e: raise e finally: diff --git a/tests/test_neps_api/solution_yamls/bo_custom_created.yaml b/tests/test_neps_api/solution_yamls/bo_custom_created.yaml new file mode 100644 index 00000000..c808c699 --- /dev/null +++ b/tests/test_neps_api/solution_yamls/bo_custom_created.yaml @@ -0,0 +1,5 @@ +searcher_name: baseoptimizer +searcher_alg: BayesianOptimization +searcher_selection: user-instantiation +neps_decision_tree: false +searcher_args: {} diff --git a/tests/test_neps_api/solution_yamls/bo_neps_decided.yaml b/tests/test_neps_api/solution_yamls/bo_neps_decided.yaml new file mode 100644 index 00000000..76935d6c --- /dev/null +++ b/tests/test_neps_api/solution_yamls/bo_neps_decided.yaml @@ -0,0 +1,13 @@ +searcher_name: bayesian_optimization +searcher_alg: bayesian_optimization +searcher_selection: neps-default +neps_decision_tree: true +searcher_args: + initial_design_size: 10 + surrogate_model: gp + acquisition: EI + log_prior_weighted: false + acquisition_sampler: mutation + random_interleave_prob: 0.0 + disable_priors: true + sample_default_first: false diff --git a/tests/test_neps_api/solution_yamls/bo_user_decided.yaml b/tests/test_neps_api/solution_yamls/bo_user_decided.yaml new file mode 100644 index 00000000..c87b923a --- /dev/null +++ b/tests/test_neps_api/solution_yamls/bo_user_decided.yaml @@ -0,0 +1,29 @@ +searcher_name: bayesian_optimization +searcher_alg: bayesian_optimization +searcher_selection: neps-default +neps_decision_tree: false +searcher_args: + initial_design_size: 10 + surrogate_model: ComprehensiveGPHierarchy + acquisition: EI + log_prior_weighted: false + acquisition_sampler: mutation + random_interleave_prob: 0.0 + disable_priors: true + sample_default_first: false + surrogate_model_args: + graph_kernels: + - WeisfeilerLehman + - WeisfeilerLehman + - WeisfeilerLehman + - WeisfeilerLehman + - WeisfeilerLehman + hp_kernels: [] + verbose: false + hierarchy_consider: + - 0 + - 1 + - 2 + - 3 + d_graph_features: 0 + vectorial_features: null diff --git a/tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml b/tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml new file mode 100644 index 00000000..602b965c --- /dev/null +++ b/tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml @@ -0,0 +1,5 @@ +searcher_name: baseoptimizer +searcher_alg: Hyperband +searcher_selection: user-instantiation +neps_decision_tree: false +searcher_args: {} diff --git a/tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml b/tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml new file mode 100644 index 00000000..29bf8dec --- /dev/null +++ b/tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml @@ -0,0 +1,11 @@ +searcher_name: hyperband +searcher_alg: hyperband +searcher_selection: neps-default +neps_decision_tree: true +searcher_args: + eta: 3 + initial_design_type: max_budget + use_priors: false + random_interleave_prob: 0.0 + sample_default_first: false + sample_default_at_target: false diff --git a/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml b/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml new file mode 100644 index 00000000..dce4d40e --- /dev/null +++ b/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml @@ -0,0 +1,14 @@ +searcher_name: pibo +searcher_alg: bayesian_optimization +searcher_selection: neps-default +neps_decision_tree: true +searcher_args: + initial_design_size: 10 + surrogate_model: gp + acquisition: EI + log_prior_weighted: false + acquisition_sampler: mutation + random_interleave_prob: 0.0 + disable_priors: false + prior_confidence: medium + sample_default_first: false diff --git a/tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml b/tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml new file mode 100644 index 00000000..cd7c82ec --- /dev/null +++ b/tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml @@ -0,0 +1,23 @@ +searcher_name: priorband_bo +searcher_alg: priorband +searcher_selection: neps-default +neps_decision_tree: false +searcher_args: + eta: 3 + initial_design_type: max_budget + prior_confidence: medium + random_interleave_prob: 0.0 + sample_default_first: true + sample_default_at_target: false + prior_weight_type: geometric + inc_sample_type: mutation + inc_mutation_rate: 0.5 + inc_mutation_std: 0.25 + inc_style: dynamic + model_based: true + modelling_type: joint + initial_design_size: 5 + surrogate_model: gp + acquisition: EI + log_prior_weighted: false + acquisition_sampler: mutation diff --git a/tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml b/tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml new file mode 100644 index 00000000..eb3b0179 --- /dev/null +++ b/tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml @@ -0,0 +1,17 @@ +searcher_name: priorband +searcher_alg: priorband +searcher_selection: neps-default +neps_decision_tree: true +searcher_args: + eta: 3 + initial_design_type: max_budget + prior_confidence: medium + random_interleave_prob: 0.0 + sample_default_first: true + sample_default_at_target: false + prior_weight_type: geometric + inc_sample_type: mutation + inc_mutation_rate: 0.5 + inc_mutation_std: 0.25 + inc_style: dynamic + model_based: false diff --git a/tests/test_neps_api/solution_yamls/user_yaml_bo.yaml b/tests/test_neps_api/solution_yamls/user_yaml_bo.yaml new file mode 100644 index 00000000..156d67e4 --- /dev/null +++ b/tests/test_neps_api/solution_yamls/user_yaml_bo.yaml @@ -0,0 +1,14 @@ +searcher_name: optimizer_test +searcher_alg: bayesian_optimization +searcher_selection: user-yaml +neps_decision_tree: false +searcher_args: + initial_design_size: 5 + surrogate_model: gp + acquisition: EI + log_prior_weighted: false + acquisition_sampler: random + random_interleave_prob: 0.1 + disable_priors: false + prior_confidence: high + sample_default_first: false diff --git a/tests/test_neps_api/test_api.py b/tests/test_neps_api/test_api.py new file mode 100644 index 00000000..a50b91d1 --- /dev/null +++ b/tests/test_neps_api/test_api.py @@ -0,0 +1,130 @@ +import logging +import os +import runpy +from pathlib import Path + +import pytest +import yaml + + +# To change the working directly into the tmp_path when testing function +@pytest.fixture(autouse=True) +def use_tmpdir(tmp_path, request): + os.chdir(tmp_path) + yield + os.chdir(request.config.invocation_dir) + + +# https://stackoverflow.com/a/59745629 +# Fail tests if there is a logging.error +@pytest.fixture(autouse=True) +def no_logs_gte_error(caplog): + yield + errors = [ + record for record in caplog.get_records("call") if record.levelno >= logging.ERROR + ] + assert not errors + + +testing_scripts = [ + "default_neps", + "baseoptimizer_neps", + "user_yaml_neps", +] + +examples_folder = Path(__file__, "..", "testing_scripts").resolve() +solution_folder = Path(__file__, "..", "solution_yamls").resolve() +neps_api_example_script = [ + examples_folder / f"{example}.py" for example in testing_scripts +] + + +@pytest.mark.neps_api +def test_default_examples(tmp_path): + # Running the example files holding multiple neps.run commands. + + runpy.run_path( + neps_api_example_script[0], + run_name="__main__", + ) + + # Testing each folder with its corresponding expected dictionary + for folder_name in os.listdir(tmp_path): + folder_path = os.path.join(tmp_path, folder_name) + + assert os.path.exists(folder_path), f"Directory does not exist: {folder_path}" + + info_yaml_path = os.path.join(folder_path, ".optimizer_info.yaml") + + assert os.path.exists( + str(info_yaml_path) + ), f"File does not exist: {info_yaml_path}" + + # Load the YAML file + with open(str(info_yaml_path)) as yaml_config: + loaded_data = yaml.safe_load(yaml_config) + + with open(str(solution_folder / (folder_name + ".yaml"))) as solution_yaml: + expected_data = yaml.safe_load(solution_yaml) + + assert loaded_data == expected_data + + +@pytest.mark.neps_api +def test_baseoptimizer_examples(tmp_path): + # Running the example files holding multiple neps.run commands. + + runpy.run_path( + neps_api_example_script[1], + run_name="__main__", + ) + + # Testing each folder with its corresponding expected dictionary + for folder_name in os.listdir(tmp_path): + folder_path = os.path.join(tmp_path, folder_name) + + assert os.path.exists(folder_path), f"Directory does not exist: {folder_path}" + + info_yaml_path = os.path.join(folder_path, ".optimizer_info.yaml") + + assert os.path.exists( + str(info_yaml_path) + ), f"File does not exist: {info_yaml_path}" + + # Load the YAML file + with open(str(info_yaml_path)) as yaml_config: + loaded_data = yaml.safe_load(yaml_config) + + with open(str(solution_folder / (folder_name + ".yaml"))) as solution_yaml: + expected_data = yaml.safe_load(solution_yaml) + + assert loaded_data == expected_data + + +@pytest.mark.neps_api +def test_user_created_yaml_examples(tmp_path): + runpy.run_path( + neps_api_example_script[2], + run_name="__main__", + ) + + # Testing each folder with its corresponding expected dictionary + for folder_name in os.listdir(tmp_path): + folder_path = os.path.join(tmp_path, folder_name) + + assert os.path.exists(folder_path), f"Directory does not exist: {folder_path}" + + info_yaml_path = os.path.join(folder_path, ".optimizer_info.yaml") + + assert os.path.exists( + str(info_yaml_path) + ), f"File does not exist: {info_yaml_path}" + + # Load the YAML file + with open(str(info_yaml_path)) as yaml_config: + loaded_data = yaml.safe_load(yaml_config) + + with open(str(solution_folder / (folder_name + ".yaml"))) as solution_yaml: + expected_data = yaml.safe_load(solution_yaml) + + assert loaded_data == expected_data diff --git a/tests/test_neps_api/test_yaml_api.py b/tests/test_neps_api/test_yaml_api.py deleted file mode 100644 index 364dc49b..00000000 --- a/tests/test_neps_api/test_yaml_api.py +++ /dev/null @@ -1,98 +0,0 @@ -import logging -import os -import runpy -from pathlib import Path - -import pytest -import yaml - - -# To change the working directly into the tmp_path when testing function -@pytest.fixture(autouse=True) -def use_tmpdir(tmp_path, request): - os.chdir(tmp_path) - yield - os.chdir(request.config.invocation_dir) - - -# https://stackoverflow.com/a/59745629 -# Fail tests if there is a logging.error -@pytest.fixture(autouse=True) -def no_logs_gte_error(caplog): - yield - errors = [ - record for record in caplog.get_records("call") if record.levelno >= logging.ERROR - ] - assert not errors - - -# Expected outcomes of the optimizer YAML according to different cases of neps.run -# all based on the examples in tests/test_neps_api/examples_test_api.py -expected_dicts = { - "priorband_bo_user_decided": { - "searcher_name": "priorband_bo", - "searcher_alg": "priorband", - "user_defined_searcher": True, - "searcher_args_user_modified": True, - }, - "priorband_neps_decided": { - "searcher_name": "priorband", - "searcher_alg": "priorband", - "user_defined_searcher": False, - "searcher_args_user_modified": False, - }, - "bo_neps_decided": { - "searcher_name": "bayesian_optimization", - "searcher_alg": "bayesian_optimization", - "user_defined_searcher": False, - "searcher_args_user_modified": False, - }, - "pibo_neps_decided": { - "searcher_name": "pibo", - "searcher_alg": "bayesian_optimization", - "user_defined_searcher": False, - "searcher_args_user_modified": False, - }, - "hyperband_neps_decided": { - "searcher_name": "hyperband", - "searcher_alg": "hyperband", - "user_defined_searcher": False, - "searcher_args_user_modified": False, - }, - "bo_custom_created": { - "searcher_name": "custom", - "searcher_alg": "BayesianOptimization", - "user_defined_searcher": True, - "searcher_args_user_modified": False, - }, -} - - -yaml_api_example = "examples_test_api" # Run locally and on github actions - -examples_folder = Path(__file__, "..").resolve() -yaml_api_example_script = examples_folder / f"{yaml_api_example}.py" - - -@pytest.mark.yaml_api -def test_core_examples(tmp_path): - # Running the example files holding multiple neps.run commands. - runpy.run_path(yaml_api_example_script, run_name="__main__") - - # Testing each folder with its corresponding expected dictionary - for folder_name in os.listdir(tmp_path): - folder_path = os.path.join(tmp_path, folder_name) - - assert os.path.exists(folder_path), f"Directory does not exist: {folder_path}" - - info_yaml_path = os.path.join(folder_path, ".optimizer_info.yaml") - - assert os.path.exists( - str(info_yaml_path) - ), f"File does not exist: {info_yaml_path}" - - # Load the YAML file - with open(str(info_yaml_path)) as yaml_config: - loaded_data = yaml.safe_load(yaml_config) - - assert loaded_data == expected_dicts[folder_name] diff --git a/tests/test_neps_api/testing_scripts/baseoptimizer_neps.py b/tests/test_neps_api/testing_scripts/baseoptimizer_neps.py new file mode 100644 index 00000000..1fe9a219 --- /dev/null +++ b/tests/test_neps_api/testing_scripts/baseoptimizer_neps.py @@ -0,0 +1,51 @@ +import logging + +import neps +from neps.optimizers.bayesian_optimization.optimizer import BayesianOptimization +from neps.optimizers.multi_fidelity.hyperband import Hyperband +from neps.search_spaces.search_space import SearchSpace + +pipeline_space_fidelity = dict( + val1=neps.FloatParameter(lower=-10, upper=10), + val2=neps.IntegerParameter(lower=1, upper=5, is_fidelity=True), +) + +pipeline_space = dict( + val1=neps.FloatParameter(lower=-10, upper=10), + val2=neps.IntegerParameter(lower=1, upper=5), +) + + +def run_pipeline(val1, val2): + loss = val1 * val2 + return loss + + +def run_pipeline_fidelity(val1, val2): + loss = val1 * val2 + return {"loss": loss, "cost": 1} + + +logging.basicConfig(level=logging.INFO) + +# Case 1: Testing BaseOptimizer as searcher with Bayesian Optimization +search_space = SearchSpace(**pipeline_space) +my_custom_searcher_1 = BayesianOptimization( + pipeline_space=search_space, initial_design_size=5 +) +neps.run( + run_pipeline=run_pipeline, + root_directory="bo_custom_created", + max_evaluations_total=1, + searcher=my_custom_searcher_1, +) + +# Case 2: Testing BaseOptimizer as searcher with Hyperband +search_space_fidelity = SearchSpace(**pipeline_space_fidelity) +my_custom_searcher_2 = Hyperband(pipeline_space=search_space_fidelity, budget=1) +neps.run( + run_pipeline=run_pipeline_fidelity, + root_directory="hyperband_custom_created", + max_cost_total=1, + searcher=my_custom_searcher_2, +) diff --git a/tests/test_neps_api/examples_test_api.py b/tests/test_neps_api/testing_scripts/default_neps.py similarity index 67% rename from tests/test_neps_api/examples_test_api.py rename to tests/test_neps_api/testing_scripts/default_neps.py index 8738fac2..5384042a 100644 --- a/tests/test_neps_api/examples_test_api.py +++ b/tests/test_neps_api/testing_scripts/default_neps.py @@ -1,8 +1,10 @@ import logging import neps -from neps.optimizers.bayesian_optimization.optimizer import BayesianOptimization -from neps.search_spaces.search_space import SearchSpace +from neps.optimizers.bayesian_optimization.kernels import GraphKernelMapping +from neps.optimizers.bayesian_optimization.models.gp_hierarchy import ( + ComprehensiveGPHierarchy, +) pipeline_space_fidelity_priors = dict( val1=neps.FloatParameter(lower=-10, upper=10, default=1), @@ -34,6 +36,9 @@ def run_pipeline(val1, val2): # Testing user input "priorband_bo" with argument changes that should be # accepted in the run. + +# Case 1: Choosing priorband + neps.run( run_pipeline=run_pipeline, pipeline_space=pipeline_space_fidelity_priors, @@ -44,6 +49,39 @@ def run_pipeline(val1, val2): eta=3, ) +# Case 2: Choosing Bayesian optimization + +early_hierarchies_considered = "0_1_2_3" +hierarchy_considered = [int(hl) for hl in early_hierarchies_considered.split("_")] +graph_kernels = ["wl"] * (len(hierarchy_considered) + 1) +wl_h = [2, 1] + [2] * (len(hierarchy_considered) - 1) +graph_kernels = [ + GraphKernelMapping[kernel]( + h=wl_h[j], + oa=False, + se_kernel=None, + ) + for j, kernel in enumerate(graph_kernels) +] +surrogate_model = ComprehensiveGPHierarchy +surrogate_model_args = { + "graph_kernels": graph_kernels, + "hp_kernels": [], + "verbose": False, + "hierarchy_consider": hierarchy_considered, + "d_graph_features": 0, + "vectorial_features": None, +} +neps.run( + run_pipeline=run_pipeline, + pipeline_space=pipeline_space_not_fidelity, + root_directory="bo_user_decided", + max_evaluations_total=1, + searcher="bayesian_optimization", + surrogate_model=surrogate_model, + surrogate_model_args=surrogate_model_args, +) + # Testing neps decision tree on deciding the searcher and rejecting the # additional arguments. @@ -82,16 +120,3 @@ def run_pipeline(val1, val2): max_evaluations_total=1, eta=2, ) - -# Testing neps when the user creates their own custom searcher -search_space = SearchSpace(**pipeline_space_fidelity) -my_custom_searcher = BayesianOptimization( - pipeline_space=search_space, initial_design_size=5 -) -neps.run( - run_pipeline=run_pipeline, - pipeline_space=pipeline_space_not_fidelity, - root_directory="bo_custom_created", - max_evaluations_total=1, - searcher=my_custom_searcher, -) diff --git a/tests/test_neps_api/testing_scripts/user_yaml_neps.py b/tests/test_neps_api/testing_scripts/user_yaml_neps.py new file mode 100644 index 00000000..6a93c4bd --- /dev/null +++ b/tests/test_neps_api/testing_scripts/user_yaml_neps.py @@ -0,0 +1,31 @@ +import logging +import os + +import neps + +pipeline_space = dict( + val1=neps.FloatParameter(lower=-10, upper=10), + val2=neps.IntegerParameter(lower=1, upper=5), +) + + +def run_pipeline(val1, val2): + loss = val1 * val2 + return loss + + +logging.basicConfig(level=logging.INFO) + +# Testing using created yaml with api +script_directory = os.path.dirname(os.path.abspath(__file__)) +parent_directory = os.path.join(script_directory, os.pardir) +searcher_path = os.path.join(parent_directory, "testing_yaml") +neps.run( + run_pipeline=run_pipeline, + pipeline_space=pipeline_space, + root_directory="user_yaml_bo", + max_evaluations_total=1, + searcher="optimizer_test", + searcher_path=searcher_path, + initial_design_size=5, +) diff --git a/tests/test_neps_api/testing_yaml/optimizer_test.yaml b/tests/test_neps_api/testing_yaml/optimizer_test.yaml new file mode 100644 index 00000000..cad2221d --- /dev/null +++ b/tests/test_neps_api/testing_yaml/optimizer_test.yaml @@ -0,0 +1,12 @@ +searcher_init: + algorithm: bayesian_optimization +searcher_kwargs: # Specific arguments depending on the searcher + initial_design_size: 7 + surrogate_model: gp + acquisition: EI + log_prior_weighted: false + acquisition_sampler: random + random_interleave_prob: 0.1 + disable_priors: false + prior_confidence: high + sample_default_first: false \ No newline at end of file diff --git a/tests/test_yaml_search_space/config_including_unknown_types.yaml b/tests/test_yaml_search_space/config_including_unknown_types.yaml new file mode 100644 index 00000000..790f473d --- /dev/null +++ b/tests/test_yaml_search_space/config_including_unknown_types.yaml @@ -0,0 +1,20 @@ +pipeline_space: + learning_rate: + type: numerical + lower: 0.00001 + upper: 0.1 + log: true + + num_epochs: + type: numerical + lower: 3 + upper: 30 + is_fidelity: True + + optimizer: + type: numerical + choices: ["adam", "sgd", "rmsprop"] + + dropout_rate: + type: numerical + value: 0.5 diff --git a/tests/test_yaml_search_space/config_including_wrong_types.yaml b/tests/test_yaml_search_space/config_including_wrong_types.yaml new file mode 100644 index 00000000..df3a4602 --- /dev/null +++ b/tests/test_yaml_search_space/config_including_wrong_types.yaml @@ -0,0 +1,20 @@ +pipeline_space: + learning_rate: + type: int + lower: 0.00001 + upper: 0.1 + log: true + + num_epochs: + type: int + lower: 3 + upper: 30 + is_fidelity: True + + optimizer: + type: cat + choices: ["adam", "sgd", "rmsprop"] + + dropout_rate: + type: const + value: 0.5 diff --git a/tests/test_yaml_search_space/correct_config.yaml b/tests/test_yaml_search_space/correct_config.yaml new file mode 100644 index 00000000..895efa38 --- /dev/null +++ b/tests/test_yaml_search_space/correct_config.yaml @@ -0,0 +1,32 @@ +pipeline_space: + param_float1: + lower: 0.00001 + upper: 0.1 + log: TRUE + is_fidelity: off + + param_int1: + lower: -3 + upper: 30 + log: false + is_fidelity: on + param_int2: + type: int + lower: 1E2 + upper: 3e4 + log: ON + is_fidelity: FALSE + + param_float2: + lower: 3.3e-5 + upper: 1.5E-1 + + param_cat: + choices: [2, "sgd", 10e-3] + + param_const1: + value: 0.5 + + param_const2: + value: 1e3 + is_fidelity: TRUE diff --git a/tests/test_yaml_search_space/correct_config_including_priors.yml b/tests/test_yaml_search_space/correct_config_including_priors.yml new file mode 100644 index 00000000..a73f2157 --- /dev/null +++ b/tests/test_yaml_search_space/correct_config_including_priors.yml @@ -0,0 +1,22 @@ +pipeline_space: + learning_rate: + lower: 0.00001 + upper: 0.1 + log: true + default: 3.3E-2 + default_confidence: high + + num_epochs: + lower: 3 + upper: 30 + is_fidelity: True + default: 1e1 + + optimizer: + choices: [adam, 90E-3, rmsprop] + default: 90E-3 + default_confidence: "medium" + + dropout_rate: + value: 1E3 + is_fidelity: true diff --git a/tests/test_yaml_search_space/correct_config_including_types.yaml b/tests/test_yaml_search_space/correct_config_including_types.yaml new file mode 100644 index 00000000..60fc4995 --- /dev/null +++ b/tests/test_yaml_search_space/correct_config_including_types.yaml @@ -0,0 +1,37 @@ +pipeline_space: + param_float1: + type: float + lower: 0.00001 + upper: 1e-1 + log: true + + param_int1: + type: integer + lower: -3 + upper: 30 + is_fidelity: True + + param_int2: + type: "int" + lower: 1e2 + upper: 3E4 + log: true + is_fidelity: false + + param_float2: + type: "float" + lower: 3.3e-5 + upper: 1.5E-1 + + param_cat: + type: cat + choices: [2, "sgd", 10E-3] + + param_const1: + type: const + value: 0.5 + + param_const2: + type: const + value: 1e3 + is_fidelity: true diff --git a/tests/test_yaml_search_space/inconsistent_types_config.yml b/tests/test_yaml_search_space/inconsistent_types_config.yml new file mode 100644 index 00000000..797658c4 --- /dev/null +++ b/tests/test_yaml_search_space/inconsistent_types_config.yml @@ -0,0 +1,17 @@ +pipeline_space: + learning_rate: + lower: "string" # Lower is now a string + upper: 1e3 + log: true + + num_epochs: + lower: 3 + upper: 30 + is_fidelity: True + + optimizer: + choices: ["adam", "sgd", "rmsprop"] + + dropout_rate: + value: 0.5 + is_fidelity: True diff --git a/tests/test_yaml_search_space/inconsistent_types_config2.yml b/tests/test_yaml_search_space/inconsistent_types_config2.yml new file mode 100644 index 00000000..d126b52e --- /dev/null +++ b/tests/test_yaml_search_space/inconsistent_types_config2.yml @@ -0,0 +1,18 @@ +pipeline_space: + learning_rate: + type: int + lower: 2.3 # float + upper: 1e3 + log: true + + num_epochs: + lower: 3 + upper: 30 + is_fidelity: True + + optimizer: + choices: ["adam", "sgd", "rmsprop"] + + dropout_rate: + value: 0.5 + is_fidelity: True diff --git a/tests/test_yaml_search_space/incorrect_config.txt b/tests/test_yaml_search_space/incorrect_config.txt new file mode 100644 index 00000000..cff7baa4 --- /dev/null +++ b/tests/test_yaml_search_space/incorrect_config.txt @@ -0,0 +1,5 @@ +pipeline_space # : is missing + learning_rate: + lower: 0.00001 + upper: 0.1 + log: true diff --git a/tests/test_yaml_search_space/missing_key_config.yml b/tests/test_yaml_search_space/missing_key_config.yml new file mode 100644 index 00000000..db0f6361 --- /dev/null +++ b/tests/test_yaml_search_space/missing_key_config.yml @@ -0,0 +1,15 @@ +pipeline_space: + learning_rate: + lower: 0.00001 + log: true + + num_epochs: + lower: 3 + upper: 30 + is_fidelity: True + + optimizer: + choices: ["adam", "sgd", "rmsprop"] + + dropout_rate: + value: 0.5 diff --git a/tests/test_yaml_search_space/not_allowed_key_config.yml b/tests/test_yaml_search_space/not_allowed_key_config.yml new file mode 100644 index 00000000..59887401 --- /dev/null +++ b/tests/test_yaml_search_space/not_allowed_key_config.yml @@ -0,0 +1,26 @@ +pipeline_space: + float_name1: + lower: 3e-5 + upper: 0.1 + + float_name2: + type: "float" # Optional, as neps infers type from 'lower' and 'upper' + lower: 1.7 + upper: 42.0 + log: true + + categorical_name1: + choices: [0, 1] + + categorical_name2: + type: cat + choices: ["a", "b", "c"] + + integer_name1: + lower: 32 + upper: 128 + fidelity: True # error, fidelity instead of is_fidelity + + integer_name2: + lower: -5 + upper: 5 diff --git a/tests/test_yaml_search_space/test_search_space.py b/tests/test_yaml_search_space/test_search_space.py new file mode 100644 index 00000000..c5cfed06 --- /dev/null +++ b/tests/test_yaml_search_space/test_search_space.py @@ -0,0 +1,159 @@ +from pathlib import Path + +import pytest +from neps.search_spaces.search_space import ( + SearchSpaceFromYamlFileError, + pipeline_space_from_yaml, +) + +from neps import CategoricalParameter, ConstantParameter, FloatParameter, IntegerParameter + + +@pytest.mark.neps_api +def test_correct_yaml_files(): + def test_correct_yaml_file(path): + """Test the function with a correctly formatted YAML file.""" + pipeline_space = pipeline_space_from_yaml(path) + assert isinstance(pipeline_space, dict) + assert isinstance(pipeline_space["param_float1"], FloatParameter) + assert pipeline_space["param_float1"].lower == 0.00001 + assert pipeline_space["param_float1"].upper == 0.1 + assert pipeline_space["param_float1"].log is True + assert pipeline_space["param_float1"].is_fidelity is False + assert pipeline_space["param_float1"].default is None + assert pipeline_space["param_float1"].default_confidence_score == 0.5 + assert isinstance(pipeline_space["param_int1"], IntegerParameter) + assert pipeline_space["param_int1"].lower == -3 + assert pipeline_space["param_int1"].upper == 30 + assert pipeline_space["param_int1"].log is False + assert pipeline_space["param_int1"].is_fidelity is True + assert pipeline_space["param_int1"].default is None + assert pipeline_space["param_int1"].default_confidence_score == 0.5 + assert isinstance(pipeline_space["param_int2"], IntegerParameter) + assert pipeline_space["param_int2"].lower == 100 + assert pipeline_space["param_int2"].upper == 30000 + assert pipeline_space["param_int2"].log is True + assert pipeline_space["param_int2"].is_fidelity is False + assert pipeline_space["param_int2"].default is None + assert pipeline_space["param_int2"].default_confidence_score == 0.5 + assert isinstance(pipeline_space["param_float2"], FloatParameter) + assert pipeline_space["param_float2"].lower == 3.3e-5 + assert pipeline_space["param_float2"].upper == 0.15 + assert pipeline_space["param_float2"].log is False + assert pipeline_space["param_float2"].is_fidelity is False + assert pipeline_space["param_float2"].default is None + assert pipeline_space["param_float2"].default_confidence_score == 0.5 + assert isinstance(pipeline_space["param_cat"], CategoricalParameter) + assert pipeline_space["param_cat"].choices == [2, "sgd", 10e-3] + assert pipeline_space["param_cat"].is_fidelity is False + assert pipeline_space["param_cat"].default is None + assert pipeline_space["param_cat"].default_confidence_score == 2 + assert isinstance(pipeline_space["param_const1"], ConstantParameter) + assert pipeline_space["param_const1"].value == 0.5 + assert pipeline_space["param_const1"].is_fidelity is False + assert isinstance(pipeline_space["param_const2"], ConstantParameter) + assert pipeline_space["param_const2"].value == 1e3 + assert pipeline_space["param_const2"].is_fidelity is True + + test_correct_yaml_file("tests/test_yaml_search_space/correct_config.yaml") + test_correct_yaml_file( + "tests/test_yaml_search_space/correct_config_including_types" ".yaml" + ) + + +@pytest.mark.neps_api +def test_correct_including_priors_yaml_file(): + """Test the function with a correctly formatted YAML file.""" + pipeline_space = pipeline_space_from_yaml( + "tests/test_yaml_search_space/correct_config_including_priors.yml" + ) + assert isinstance(pipeline_space, dict) + assert isinstance(pipeline_space["learning_rate"], FloatParameter) + assert pipeline_space["learning_rate"].lower == 0.00001 + assert pipeline_space["learning_rate"].upper == 0.1 + assert pipeline_space["learning_rate"].log is True + assert pipeline_space["learning_rate"].is_fidelity is False + assert pipeline_space["learning_rate"].default == 3.3e-2 + assert pipeline_space["learning_rate"].default_confidence_score == 0.125 + assert isinstance(pipeline_space["num_epochs"], IntegerParameter) + assert pipeline_space["num_epochs"].lower == 3 + assert pipeline_space["num_epochs"].upper == 30 + assert pipeline_space["num_epochs"].log is False + assert pipeline_space["num_epochs"].is_fidelity is True + assert pipeline_space["num_epochs"].default == 10 + assert pipeline_space["num_epochs"].default_confidence_score == 0.5 + assert isinstance(pipeline_space["optimizer"], CategoricalParameter) + assert pipeline_space["optimizer"].choices == ["adam", 90e-3, "rmsprop"] + assert pipeline_space["optimizer"].is_fidelity is False + assert pipeline_space["optimizer"].default == 90e-3 + assert pipeline_space["optimizer"].default_confidence_score == 4 + assert isinstance(pipeline_space["dropout_rate"], ConstantParameter) + assert pipeline_space["dropout_rate"].value == 1e3 + assert pipeline_space["dropout_rate"].default == 1e3 + + +@pytest.mark.neps_api +def test_incorrect_yaml_file(): + """Test the function with an incorrectly formatted YAML file.""" + with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: + pipeline_space_from_yaml( + Path("tests/test_yaml_search_space/incorrect_config.txt") + ) + assert excinfo.value.exception_type == "ValueError" + + +@pytest.mark.neps_api +def test_yaml_file_with_missing_key(): + """Test the function with a YAML file missing a required key.""" + with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: + pipeline_space_from_yaml("tests/test_yaml_search_space/missing_key_config.yml") + assert excinfo.value.exception_type == "KeyError" + + +@pytest.mark.neps_api +def test_yaml_file_with_inconsistent_types(): + """Test the function with a YAML file having inconsistent types for + 'lower' and 'upper'.""" + with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: + pipeline_space_from_yaml( + "tests/test_yaml_search_space/inconsistent_types_config.yml" + ) + assert str(excinfo.value.exception_type == "TypeError") + with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: + pipeline_space_from_yaml( + Path("tests/test_yaml_search_space/inconsistent_types_config2.yml") + ) + assert excinfo.value.exception_type == "TypeError" + + +@pytest.mark.neps_api +def test_yaml_file_including_wrong_types(): + """Test the function with a YAML file that defines the wrong but existing type + int to float as an optional argument""" + with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: + pipeline_space_from_yaml( + "tests/test_yaml_search_space/config_including_wrong_types.yaml" + ) + assert excinfo.value.exception_type == "TypeError" + + +@pytest.mark.neps_api +def test_yaml_file_including_unkown_types(): + """Test the function with a YAML file that defines an unknown type as an optional + argument""" + with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: + pipeline_space_from_yaml( + "tests/test_yaml_search_space/config_including_unknown_types.yaml" + ) + assert excinfo.value.exception_type == "TypeError" + + +@pytest.mark.neps_api +def test_yaml_file_including_not_allowed_parameter_keys(): + """Test the function with a YAML file that defines an unknown type as an optional + argument""" + with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: + pipeline_space_from_yaml( + "tests/test_yaml_search_space/not_allowed_key_config.yml" + ) + assert excinfo.value.exception_type == "KeyError"