Renaming several functions

docs/doc_yamls/run_pipeline.py

@@ -2,6 +2,6 @@
 
 def example_pipeline(learning_rate, optimizer, epochs):
     model = initialize_model()
-    training_loss = train_model(model, optimizer, learning_rate, epochs)
-    evaluation_loss = evaluate_model(model)
-    return {"loss": evaluation_loss, "training_loss": training_loss}
+    training_objective_to_minimize = train_model(model, optimizer, learning_rate, epochs)
+    evaluation_objective_to_minimize = evaluate_model(model)
+    return {"objective_to_minimize": evaluation_objective_to_minimize, "training_objective_to_minimize": training_objective_to_minimize}

docs/doc_yamls/run_pipeline_architecture.py

@@ -19,6 +19,6 @@ def example_pipeline(architecture, optimizer, learning_rate):
         nn.Flatten(),
         nn.Linear(base_channels * out_channels_factor, n_classes),
     )
-    training_loss = train_model(model, optimizer, learning_rate)
-    evaluation_loss = evaluate_model(model)
-    return {"loss": evaluation_loss, "training_loss": training_loss}
+    training_objective_to_minimize = train_model(model, optimizer, learning_rate)
+    evaluation_objective_to_minimize = evaluate_model(model)
+    return {"objective_to_minimize": evaluation_objective_to_minimize, "training_objective_to_minimize": training_objective_to_minimize}

docs/doc_yamls/run_pipeline_big_search_space.py

@@ -1,6 +1,6 @@
 
 def example_pipeline(learning_rate, optimizer, epochs, batch_size, dropout_rate):
     model = initialize_model(dropout_rate)
-    training_loss = train_model(model, optimizer, learning_rate, epochs, batch_size)
-    evaluation_loss = evaluate_model(model)
-    return {"loss": evaluation_loss, "training_loss": training_loss}
+    training_objective_to_minimize = train_model(model, optimizer, learning_rate, epochs, batch_size)
+    evaluation_objective_to_minimize = evaluate_model(model)
+    return {"objective_to_minimize": evaluation_objective_to_minimize, "training_objective_to_minimize": training_objective_to_minimize}

docs/doc_yamls/run_pipeline_extended.py

@@ -1,6 +1,6 @@
 
 def example_pipeline(learning_rate, optimizer, epochs, batch_size):
     model = initialize_model()
-    training_loss = train_model(model, optimizer, learning_rate, epochs, batch_size)
-    evaluation_loss = evaluate_model(model)
-    return {"loss": evaluation_loss, "training_loss": training_loss}
+    training_objective_to_minimize = train_model(model, optimizer, learning_rate, epochs, batch_size)
+    evaluation_objective_to_minimize = evaluate_model(model)
+    return {"objective_to_minimize": evaluation_objective_to_minimize, "training_objective_to_minimize": training_objective_to_minimize}

neps/api.py

@@ -26,7 +26,7 @@
 
 
 def run(
-    run_pipeline: Callable | None = Default(None),
+    evaluate_pipeline: Callable | None = Default(None),
     root_directory: str | Path | None = Default(None),
     pipeline_space: (
         dict[str, Parameter] | str | Path | CS.ConfigurationSpace | None
@@ -41,7 +41,7 @@ def run(
     continue_until_max_evaluation_completed: bool = Default(False),
     max_cost_total: int | float | None = Default(None),
     ignore_errors: bool = Default(False),
-    loss_value_on_error: None | float = Default(None),
+    objective_to_minimize_value_on_error: None | float = Default(None),
     cost_value_on_error: None | float = Default(None),
     pre_load_hooks: Iterable | None = Default(None),
     searcher: (
@@ -68,14 +68,14 @@ def run(
         the multiple calls to run(.) will be independent.
 
     Args:
-        run_pipeline: The objective function to minimize.
+        evaluate_pipeline: The objective function to minimize.
         pipeline_space: The search space to minimize over.
         root_directory: The directory to save progress to. This is also used to
             synchronize multiple calls to run(.) for parallelization.
         run_args: An option for providing the optimization settings e.g.
             max_evaluations_total in a YAML file.
         overwrite_working_directory: If true, delete the working directory at the start of
-            the run. This is, e.g., useful when debugging a run_pipeline function.
+            the run. This is, e.g., useful when debugging a evaluate_pipeline function.
         post_run_summary: If True, creates a csv file after each worker is done,
             holding summary information about the configs and results.
         development_stage_id: ID for the current development stage. Only needed if
@@ -89,13 +89,13 @@ def run(
             max_evaluations_total have been completed. This is only relevant in the
             parallel setting.
         max_cost_total: No new evaluations will start when this cost is exceeded. Requires
-            returning a cost in the run_pipeline function, e.g.,
+            returning a cost in the evaluate_pipeline function, e.g.,
             `return dict(loss=loss, cost=cost)`.
         ignore_errors: Ignore hyperparameter settings that threw an error and do not raise
             an error. Error configs still count towards max_evaluations_total.
-        loss_value_on_error: Setting this and cost_value_on_error to any float will
-            supress any error and will use given loss value instead. default: None
-        cost_value_on_error: Setting this and loss_value_on_error to any float will
+        objective_to_minimize_value_on_error: Setting this and cost_value_on_error to any float will
+            supress any error and will use given objective_to_minimize value instead. default: None
+        cost_value_on_error: Setting this and objective_to_minimize_value_on_error to any float will
             supress any error and will use given cost value instead. default: None
         pre_load_hooks: List of functions that will be called before load_results().
         searcher: Which optimizer to use. Can be a string identifier, an
@@ -111,15 +111,15 @@ def run(
     Example:
         >>> import neps
 
-        >>> def run_pipeline(some_parameter: float):
+        >>> def evaluate_pipeline(some_parameter: float):
         >>>    validation_error = -some_parameter
         >>>    return validation_error
 
         >>> pipeline_space = dict(some_parameter=neps.Float(lower=0, upper=1))
 
         >>> logging.basicConfig(level=logging.INFO)
         >>> neps.run(
-        >>>    run_pipeline=run_pipeline,
+        >>>    evaluate_pipeline=evaluate_pipeline,
         >>>    pipeline_space=pipeline_space,
         >>>    root_directory="usage_example",
         >>>    max_evaluations_total=5,
@@ -193,7 +193,7 @@ def run(
             pipeline_space=settings.pipeline_space,
             max_cost_total=settings.max_cost_total,
             ignore_errors=settings.ignore_errors,
-            loss_value_on_error=settings.loss_value_on_error,
+            objective_to_minimize_value_on_error=settings.objective_to_minimize_value_on_error,
             cost_value_on_error=settings.cost_value_on_error,
             searcher=settings.searcher,
             **settings.searcher_kwargs,
@@ -223,15 +223,15 @@ def run(
         )
 
     _launch_runtime(
-        evaluation_fn=settings.run_pipeline,
+        evaluation_fn=settings.evaluate_pipeline,
         optimizer=searcher_instance,
         optimizer_info=searcher_info,
         max_cost_total=settings.max_cost_total,
         optimization_dir=Path(settings.root_directory),
         max_evaluations_total=settings.max_evaluations_total,
         max_evaluations_for_worker=settings.max_evaluations_per_run,
         continue_until_max_evaluation_completed=settings.continue_until_max_evaluation_completed,
-        loss_value_on_error=settings.loss_value_on_error,
+        objective_to_minimize_value_on_error=settings.objective_to_minimize_value_on_error,
         cost_value_on_error=settings.cost_value_on_error,
         ignore_errors=settings.ignore_errors,
         overwrite_optimization_dir=settings.overwrite_working_directory,
@@ -266,7 +266,7 @@ def _run_args(
     ) = None,
     max_cost_total: int | float | None = None,
     ignore_errors: bool = False,
-    loss_value_on_error: None | float = None,
+    objective_to_minimize_value_on_error: None | float = None,
     cost_value_on_error: None | float = None,
     searcher: (
         Literal[
@@ -406,7 +406,7 @@ def _run_args(
 
     searcher_config.update(
         {
-            "loss_value_on_error": loss_value_on_error,
+            "objective_to_minimize_value_on_error": objective_to_minimize_value_on_error,
             "cost_value_on_error": cost_value_on_error,
             "ignore_errors": ignore_errors,
         }
@@ -416,7 +416,7 @@ def _run_args(
         SearcherMapping, searcher_alg, "searcher", as_class=True
     )(
         pipeline_space=pipeline_space,
-        budget=max_cost_total,  # TODO: use max_cost_total everywhere
+        max_cost_total=max_cost_total,  # TODO: use max_cost_total everywhere
         **searcher_config,
     )
 

neps/optimizers/base_optimizer.py

@@ -14,29 +14,30 @@
     from neps.utils.types import ERROR, ResultDict
 
 
-def _get_loss(
+def _get_objective_to_minimize(
     result: ERROR | ResultDict | float,
-    loss_value_on_error: float | None = None,
+    objective_to_minimize_value_on_error: float | None = None,
     *,
     ignore_errors: bool = False,
 ) -> ERROR | float:
     if result == "error":
         if ignore_errors:
             return "error"
 
-        if loss_value_on_error is not None:
-            return loss_value_on_error
+        if objective_to_minimize_value_on_error is not None:
+            return objective_to_minimize_value_on_error
 
         raise ValueError(
-            "An error happened during the execution of your run_pipeline function."
+            "An error happened during the execution of your evaluate_pipeline function."
             " You have three options: 1. If the error is expected and corresponds to"
-            " a loss value in your application (e.g., 0% accuracy), you can set"
-            " loss_value_on_error to some float. 2. If sometimes your pipeline"
-            " crashes randomly, you can set ignore_errors=True. 3. Fix your error."
+            " an objective_to_minimize value in your application (e.g., 0% accuracy),"
+            " you can set objective_to_minimize_value_on_error to some float. 2. If "
+            " sometimes your pipeline crashes randomly, you can set ignore_errors=True."
+            " 3. Fix your error."
         )
 
     if isinstance(result, dict):
-        return float(result["loss"])
+        return float(result["objective_to_minimize"])
 
     assert isinstance(result, float)
     return float(result)
@@ -54,9 +55,9 @@ def _get_cost(
 
         if cost_value_on_error is None:
             raise ValueError(
-                "An error happened during the execution of your run_pipeline function."
-                " You have three options: 1. If the error is expected and corresponds to"
-                " a cost value in your application, you can set"
+                "An error happened during the execution of your evaluate_pipeline"
+                " function. You have three options: 1. If the error is expected and"
+                " corresponds to a cost value in your application, you can set"
                 " cost_value_on_error to some float. 2. If sometimes your pipeline"
                 " crashes randomly, you can set ignore_errors=True. 3. Fix your error."
             )
@@ -88,20 +89,20 @@ def __init__(
         pipeline_space: SearchSpace,
         patience: int = 50,
         logger: logging.Logger | None = None,
-        budget: int | float | None = None,
-        loss_value_on_error: float | None = None,
+        max_cost_total: int | float | None = None,
+        objective_to_minimize_value_on_error: float | None = None,
         cost_value_on_error: float | None = None,
         learning_curve_on_error: float | list[float] | None = None,
         ignore_errors: bool = False,
     ) -> None:
         if patience < 1:
             raise ValueError("Patience should be at least 1")
 
-        self.budget = budget
+        self.max_cost_total = max_cost_total
         self.pipeline_space = pipeline_space
         self.patience = patience
         self.logger = logger or logging.getLogger("neps")
-        self.loss_value_on_error = loss_value_on_error
+        self.objective_to_minimize_value_on_error = objective_to_minimize_value_on_error
         self.cost_value_on_error = cost_value_on_error
         self.learning_curve_on_error = learning_curve_on_error
         self.ignore_errors = ignore_errors
@@ -110,34 +111,41 @@ def __init__(
     def ask(
         self,
         trials: Mapping[str, Trial],
-        budget_info: BudgetInfo | None,
+        max_cost_total_info: BudgetInfo | None,
     ) -> SampledConfig:
         """Sample a new configuration.
 
         Args:
             trials: All of the trials that are known about.
-            budget_info: information about the budget
+            max_cost_total_info: information about the max_cost_total
 
         Returns:
             SampledConfig: a sampled configuration
             dict: state the optimizer would like to keep between calls
         """
         ...
 
-    def get_loss(self, result: ERROR | ResultDict | float | Report) -> float | ERROR:
-        """Calls result.utils.get_loss() and passes the error handling through.
-        Please use self.get_loss() instead of get_loss() in all optimizer classes.
+    def get_objective_to_minimize(
+        self, result: ERROR | ResultDict | float | Report
+    ) -> float | ERROR:
+        """Calls result.utils.get_objective_to_minimize() and passes the error handling
+        through. Please use self.get_objective_to_minimize() instead of
+        get_objective_to_minimize() in all optimizer classes.
         """
         # TODO(eddiebergman): This is a forward change for whenever we can have optimizers
         # use `Trial` and `Report`, they already take care of this and save having to do
-        # this `_get_loss` at every call. We can also then just use `None` instead of
-        # the string `"error"`
+        # this `_get_objective_to_minimize` at every call. We can also then just use
+        # `None` instead of the string `"error"`
         if isinstance(result, Report):
-            return result.loss if result.loss is not None else "error"
+            return (
+                result.objective_to_minimize
+                if result.objective_to_minimize is not None
+                else "error"
+            )
 
-        return _get_loss(
+        return _get_objective_to_minimize(
             result,
-            loss_value_on_error=self.loss_value_on_error,
+            objective_to_minimize_value_on_error=self.objective_to_minimize_value_on_error,
             ignore_errors=self.ignore_errors,
         )
 
@@ -147,9 +155,13 @@ def get_cost(self, result: ERROR | ResultDict | float | Report) -> float | ERROR
         """
         # TODO(eddiebergman): This is a forward change for whenever we can have optimizers
         # use `Trial` and `Report`, they already take care of this and save having to do
-        # this `_get_loss` at every call
+        # this `_get_objective_to_minimize` at every call
         if isinstance(result, Report):
-            return result.loss if result.loss is not None else "error"
+            return (
+                result.objective_to_minimize
+                if result.objective_to_minimize is not None
+                else "error"
+            )
 
         return _get_cost(
             result,

neps/optimizers/bayesian_optimization/acquisition_functions/cost_cooling.py

@@ -41,14 +41,14 @@ def apply_cost_cooling(
 def cost_cooled_acq(
     acq_fn: AcquisitionFunction,
     model: GPyTorchModel,
-    used_budget_percentage: float,
+    used_max_cost_total_percentage: float,
 ) -> WeightedAcquisition:
-    assert 0 <= used_budget_percentage <= 1
+    assert 0 <= used_max_cost_total_percentage <= 1
     return WeightedAcquisition(
         acq=acq_fn,
         apply_weight=partial(
             apply_cost_cooling,
             cost_model=model,
-            alpha=1 - used_budget_percentage,
+            alpha=1 - used_max_cost_total_percentage,
         ),
     )

neps/optimizers/bayesian_optimization/models/ftpfn.py

@@ -164,7 +164,7 @@ def encode_ftpfn(
         device=device,
         dtype=dtype,
     )
-    train_budgets = budget_domain.cast(
+    train_max_cost_total = budget_domain.cast(
         train_fidelities, frm=space.fidelity.domain, dtype=dtype
     )
 
@@ -174,7 +174,11 @@ def encode_ftpfn(
         [
             pending_value
             if trial.report is None
-            else (error_value if trial.report.loss is None else trial.report.loss)
+            else (
+                error_value
+                if trial.report.objective_to_minimize is None
+                else trial.report.objective_to_minimize
+            )
             for trial in trials.values()
         ],
         device=device,
@@ -188,7 +192,7 @@ def encode_ftpfn(
         )
     maximize_ys = 1 - minimize_ys
     x_train = torch.cat(
-        [ids.unsqueeze(1), train_budgets.unsqueeze(1), train_configs], dim=1
+        [ids.unsqueeze(1), train_max_cost_total.unsqueeze(1), train_configs], dim=1
     )
     return x_train, maximize_ys
 

neps/optimizers/bayesian_optimization/models/gp.py

@@ -243,8 +243,10 @@ def encode_trials_for_gp(
 
         train_configs.append(trial.config)
 
-        loss = trial.report.loss
-        train_losses.append(torch.nan if loss is None else loss)
+        objective_to_minimize = trial.report.objective_to_minimize
+        train_losses.append(
+            torch.nan if objective_to_minimize is None else objective_to_minimize
+        )
 
         cost = trial.report.cost
         train_costs.append(torch.nan if cost is None else cost)
@@ -399,7 +401,7 @@ def fit_and_acquire_from_gp(
         acquisition = cost_cooled_acq(
             acq_fn=acquisition,
             model=cost_gp,
-            used_budget_percentage=cost_percentage_used,
+            used_max_cost_total_percentage=cost_percentage_used,
         )
 
     _n = n_candidates_required if n_candidates_required is not None else 1