Clean internal optimization within Kriging (SMTorg#505)

* test TNC

* cobyla for sgp

* fix for gekpls

* fix gradient algorithm

* pl_exp_3

smt/applications/mfkpls.py

@@ -34,6 +34,13 @@ def _initialize(self):
             desc="Correlation function type",
             types=(str),
         )
+        declare(
+            "hyper_opt",
+            "Cobyla",
+            values=("Cobyla"),
+            desc="Optimiser for hyperparameters optimisation",
+            types=str,
+        )
         declare("n_comp", 1, types=int, desc="Number of principal components")
         self.name = "MFKPLS"
 

smt/applications/mfkplsk.py

@@ -26,6 +26,13 @@ def _initialize(self):
             desc="Correlation function type",
             types=(str),
         )
+        declare(
+            "hyper_opt",
+            "Cobyla",
+            values=("Cobyla"),
+            desc="Optimiser for hyperparameters optimisation",
+            types=str,
+        )
         self.name = "MFKPLSK"
 
     def _componentwise_distance(self, dx, opt=0):

smt/applications/mixed_integer.py

@@ -97,7 +97,6 @@ def __init__(
         """
         super().__init__()
         self._surrogate = surrogate
-
         if isinstance(self._surrogate, KrgBased):
             raise ValueError(
                 "Using MixedIntegerSurrogateModel integer model with "
@@ -109,7 +108,6 @@ def __init__(
         self._input_in_folded_space = input_in_folded_space
         self.supports = self._surrogate.supports
         self.options["print_global"] = False
-
         if "poly" in self._surrogate.options:
             if self._surrogate.options["poly"] != "constant":
                 raise ValueError("constant regression must be used with mixed integer")
@@ -198,6 +196,7 @@ def __init__(
                 + " is not supported. Please use MixedIntegerSurrogateModel instead."
             )
         self.options["design_space"] = self._surrogate.design_space
+        self._surrogate.options["hyper_opt"] = "Cobyla"
 
         self._input_in_folded_space = input_in_folded_space
         self.supports = self._surrogate.supports
@@ -333,6 +332,7 @@ def build_kriging_model(self, surrogate):
         Build MixedIntegerKrigingModel from given SMT surrogate model.
         """
         surrogate.options["design_space"] = self._design_space
+        surrogate.options["hyper_opt"] = "Cobyla"
         return MixedIntegerKrigingModel(
             surrogate=surrogate,
             input_in_folded_space=self._work_in_folded_space,

smt/applications/tests/test_vfm.py

@@ -68,7 +68,12 @@ def test_vfm(self):
         Bridge_candidate = "KRG"
         type_bridge = "Multiplicative"
         optionsLF = {}
-        optionsB = {"theta0": [1e-2] * ndim, "print_prediction": False, "deriv": False}
+        optionsB = {
+            "theta0": [1e-2] * ndim,
+            "print_prediction": False,
+            "deriv": False,
+            "hyper_opt": "Cobyla",
+        }
 
         # Construct low/high fidelity data and validation points
         sampling = LHS(xlimits=funLF.xlimits, criterion="m", random_state=42)
@@ -138,7 +143,12 @@ def run_vfm_example(self):
         Bridge_candidate = "KRG"
         type_bridge = "Multiplicative"
         optionsLF = {}
-        optionsB = {"theta0": [1e-2] * ndim, "print_prediction": False, "deriv": False}
+        optionsB = {
+            "theta0": [1e-2] * ndim,
+            "print_prediction": False,
+            "deriv": False,
+            "hyper_opt": "Cobyla",
+        }
 
         # Construct low/high fidelity data and validation points
         sampling = LHS(xlimits=funLF.xlimits, criterion="m")
@@ -200,7 +210,9 @@ def test_KRG_KRG_additive(self):
             yp = M.predict_values(np.atleast_2d(xt[0]))
             dyp = M.predict_derivatives(np.atleast_2d(xt[0]), kx=0)
         self.assert_error(yp, np.array([[0.015368, 0.367424]]), atol=2e-2, rtol=3e-2)
-        self.assert_error(dyp, np.array([[0.07007729, 3.619421]]), atol=3e-1, rtol=1e-2)
+        self.assert_error(
+            dyp, np.array([[-3.11718627e-03, 3.19506239e00]]), atol=3e-1, rtol=1e-2
+        )
 
     def test_QP_KRG_additive(self):
         with Silence():
@@ -214,7 +226,7 @@ def test_QP_KRG_additive(self):
 
         self.assert_error(yp, np.array([[0.015368, 0.367424]]), atol=1e-2, rtol=1e-2)
         self.assert_error(
-            dyp, np.array([[1.16130832e-03, 4.36712162e00]]), atol=3e-1, rtol=1e-2
+            dyp, np.array([[0.02596425, 4.70243162]]), atol=3e-1, rtol=1e-2
         )
 
     def test_KRG_KRG_mult(self):
@@ -228,7 +240,9 @@ def test_KRG_KRG_mult(self):
             dyp = M.predict_derivatives(np.atleast_2d(xt[0]), kx=0)
 
         self.assert_error(yp, np.array([[0.015368, 0.367424]]), atol=2e-2, rtol=3e-2)
-        self.assert_error(dyp, np.array([[0.07007729, 3.619421]]), atol=3e-1, rtol=1e-2)
+        self.assert_error(
+            dyp, np.array([[-3.11718627e-03, 3.19506239e00]]), atol=3e-1, rtol=1e-2
+        )
 
     def test_QP_KRG_mult(self):
         with Silence():

smt/surrogate_models/gekpls.py

@@ -32,6 +32,13 @@ def _initialize(self):
             types=int,
             desc="Number of extra points per training point",
         )
+        declare(
+            "hyper_opt",
+            "Cobyla",
+            values=("Cobyla"),
+            desc="Optimiser for hyperparameters optimisation",
+            types=str,
+        )
         self.supports["training_derivatives"] = True
 
     def _check_param(self):

smt/surrogate_models/krg_based.py

@@ -138,7 +138,7 @@ def _initialize(self):
         )
         declare(
             "hyper_opt",
-            "Cobyla",
+            "TNC",
             values=("Cobyla", "TNC"),
             desc="Optimiser for hyperparameters optimisation",
             types=str,
@@ -1016,7 +1016,7 @@ def _reduced_likelihood_gradient(self, theta):
         gamma = par["gamma"]
         Q = par["Q"]
         G = par["G"]
-        sigma_2 = par["sigma2"]
+        sigma_2 = par["sigma2"] + self.options["nugget"]
 
         nb_theta = len(theta)
         grad_red = np.zeros(nb_theta)
@@ -1908,6 +1908,10 @@ def grad_minus_reduced_likelihood_function(log10t):
                                 optimal_theta_res = optimal_theta_res_loop
 
                     elif self.options["hyper_opt"] == "TNC":
+                        if self.options["use_het_noise"]:
+                            raise ValueError(
+                                "For heteroscedastic noise, please use Cobyla"
+                            )
                         theta_all_loops = 10**theta_all_loops
                         for theta0_loop in theta_all_loops:
                             optimal_theta_res_loop = optimize.minimize(
@@ -1916,7 +1920,7 @@ def grad_minus_reduced_likelihood_function(log10t):
                                 method="TNC",
                                 jac=grad_minus_reduced_likelihood_function,
                                 bounds=bounds_hyp,
-                                options={"maxiter": 100},
+                                options={"maxfun": 2 * limit},
                             )
                             if optimal_theta_res_loop["fun"] < optimal_theta_res["fun"]:
                                 optimal_theta_res = optimal_theta_res_loop

smt/surrogate_models/sgp.py

@@ -50,6 +50,13 @@ def _initialize(self):
             desc="Gaussian noise on observed training data",
             types=(list, np.ndarray),
         )
+        declare(
+            "hyper_opt",
+            "Cobyla",
+            values=("Cobyla"),
+            desc="Optimiser for hyperparameters optimisation",
+            types=str,
+        )
         declare(
             "eval_noise",
             True,  # for SGP evaluate noise by default

smt/surrogate_models/tests/test_krg_het_noise.py

@@ -23,7 +23,13 @@ def test_predict_output(self):
         xt_full = np.array(3 * xt.tolist())
         yt_full = np.concatenate((yt, yt + 0.2 * yt_std_rand, yt - 0.2 * yt_std_rand))
 
-        sm = KRG(theta0=[1.0], eval_noise=True, use_het_noise=True, n_start=1)
+        sm = KRG(
+            theta0=[1.0],
+            eval_noise=True,
+            use_het_noise=True,
+            n_start=1,
+            hyper_opt="Cobyla",
+        )
         sm.set_training_values(xt_full, yt_full)
         sm.train()
 

smt/tests/test_array_outputs.py

@@ -38,7 +38,7 @@ def test_KRG(self):
             d0 = interp.predict_derivatives(np.atleast_2d(xt[10, :]), 0)
 
         self.assert_error(
-            d0, np.array([[0.06874097, 4.366292277996716]]), atol=0.55, rtol=0.15
+            d0, np.array([[0.24897752, 3.72290526]]), atol=0.55, rtol=0.15
         )
 
     def test_RBF(self):

smt/tests/test_kpls_auto.py

@@ -46,12 +46,9 @@ def setUp(self):
         n_comp_opt["Branin"] = 2
         n_comp_opt["Rosenbrock"] = 1
         n_comp_opt["sphere"] = 1
-        if platform.startswith("linux"):  # result depends on platform
-            n_comp_opt["exp"] = 2
-        else:
-            n_comp_opt["exp"] = 3
+        n_comp_opt["exp"] = 3
         n_comp_opt["tanh"] = 1
-        n_comp_opt["cos"] = 1
+        n_comp_opt["cos"] = 2
 
         self.nt = nt
         self.ne = ne