From ae425f1019b57ec6ba30179f2a5411fb3544dde7 Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Tue, 27 Feb 2024 09:13:47 +0100
Subject: [PATCH 01/90] mean pinball loss

---
 lib/scholar/metrics/regression.ex        | 113 +++--------------------
 test/scholar/metrics/regression_test.exs |  87 +----------------
 2 files changed, 17 insertions(+), 183 deletions(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index b26e4605..1a958b98 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -519,55 +519,6 @@ defmodule Scholar.Metrics.Regression do
     Nx.reduce_max(Nx.abs(y_true - y_pred))
   end
 
-  mean_pinball_loss_opts = [
-    alpha: [
-      type: :float,
-      default: 0.5,
-      doc: """
-      The slope of the pinball loss, default=0.5,
-      This loss is equivalent to $$mean_absolute_error$$ when $$\alpha$$ is 0.5,
-      $$\alpha = 0.95$$ is minimized by estimators of the 95th percentile.
-      """
-    ],
-    sample_weights: [
-      type:
-        {:or,
-         [
-           {:custom, Scholar.Options, :weights, []},
-           {:custom, Scholar.Options, :multi_weights, []}
-         ]},
-      doc: """
-      The weights for each observation. If not provided,
-      all observations are assigned equal weight.
-      """
-    ],
-    multioutput: [
-      type:
-        {:or,
-         [
-           {:custom, Scholar.Options, :weights, []},
-           {:in, [:raw_values, :uniform_average]}
-         ]},
-      default: :uniform_average,
-      doc: """
-      Defines aggregating of multiple output values.
-      Array-like value defines weights used to average errors.
-      Defaults to `:uniform_average`.
-
-        `:raw_values` :
-            Returns a full set of errors in case of multioutput input.
-
-        `:uniform_average` :
-            Errors of all outputs are averaged with uniform weight.
-
-      The weights for each observation. If not provided,
-      all observations are assigned equal weight.
-      """
-    ]
-  ]
-
-  @mean_pinball_loss_schema NimbleOptions.new!(mean_pinball_loss_opts)
-
   @doc ~S"""
   Calculates the mean pinball loss to evaluate predictive performance of quantile regression models.
 
@@ -576,66 +527,30 @@ defmodule Scholar.Metrics.Regression do
 
   The residual error is defined as $$|y - \hat{y}|$$ where $y$ is a true value
   and $\hat{y}$ is a predicted value.
-
-  #{NimbleOptions.docs(@mean_pinball_loss_schema)}
+  Equivalent to half of the $$mean_absolute_error$$ when $$\alpha$$ is 0.5.
 
   ## Examples
 
       iex> y_true = Nx.tensor([1, 2, 3])
       iex> y_pred = Nx.tensor([2, 3, 4])
-      iex> Scholar.Metrics.Regression.mean_pinball_loss(y_true, y_pred)
+      iex> Scholar.Metrics.Regression.mean_pinball_loss(y_true, y_pred, 0.5)
       #Nx.Tensor<
         f32
         0.5
       >
-      iex> y_true = Nx.tensor([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
-      iex> y_pred = Nx.tensor([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])
-      iex> Scholar.Metrics.Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.5, multioutput: :raw_values)
-      #Nx.Tensor<
-        f32[4]
-        [0.5, 0.3333333432674408, 0.0, 0.0]
-      >
   """
-  deftransform mean_pinball_loss(y_true, y_pred, opts \\ []) do
-    mean_pinball_loss_n(y_true, y_pred, NimbleOptions.validate!(opts, @mean_pinball_loss_schema))
-  end
-
-  defnp mean_pinball_loss_n(y_true, y_pred, opts) do
-    assert_same_shape!(y_true, y_pred)
-    alpha = opts[:alpha]
-
-    # Formula adapted from sklearn:
-    # https://github.com/scikit-learn/scikit-learn/blob/128e40ed593c57e8b9e57a4109928d58fa8bf359/sklearn/metrics/_regression.py#L299
-    diff = y_true - y_pred
-    sign = diff >= 0
-    loss = alpha * sign * diff - (1 - alpha) * (1 - sign) * diff
-
-    output_errors = handle_sample_weights(loss, opts, axes: [0])
-    # mimics the sklearn behavior
-    case opts[:multioutput] do
-      # raw_values returns plain output errors. One value per channel.
-      :raw_values ->
-        output_errors
-
-      # uniform_average returns the mean of the above. Note how they are averaged.
-      :uniform_average ->
-        output_errors
-        |> Nx.mean()
-
-      # pass `:multioutput` as sample weights to average the error of each output
-      multi_output_weights ->
-        handle_sample_weights(output_errors, sample_weights: multi_output_weights)
-    end
-  end
-
-  defnp handle_sample_weights(loss, opts, mean_opts \\ []) do
-    case opts[:sample_weights] do
-      nil ->
-        Nx.mean(loss, mean_opts)
-
-      weights ->
-        Nx.weighted_mean(loss, weights, mean_opts)
-    end
+  defn mean_pinball_loss(y_true, y_pred, alpha \\ 0.5) do
+    check_shape(y_true, y_pred)
+    diff = Nx.subtract(y_true, y_pred)
+    sign = Nx.greater_equal(diff, 0)
+    subtracted_sign = Nx.subtract(1, sign)
+    
+    Nx.subtract(
+      Nx.multiply(alpha, sign)
+      |> Nx.multiply(diff),
+      Nx.multiply(1 - alpha, subtracted_sign)
+      |> Nx.multiply(diff))
+    |> Nx.mean()
   end
 
   defnp check_shape(y_true, y_pred) do
diff --git a/test/scholar/metrics/regression_test.exs b/test/scholar/metrics/regression_test.exs
index ef305e75..7a2e0fdb 100644
--- a/test/scholar/metrics/regression_test.exs
+++ b/test/scholar/metrics/regression_test.exs
@@ -66,98 +66,17 @@ defmodule Scholar.Metrics.RegressionTest do
       assert Nx.equal(d2, r2)
     end
   end
-
+  
   describe "mean_pinball_loss/3" do
     test "mean_pinball_loss cases from sklearn" do
-      # Test cases copied from sklearn:
-      # https://github.com/scikit-learn/scikit-learn/blob/128e40ed593c57e8b9e57a4109928d58fa8bf359/sklearn/metrics/tests/test_regression.py#L49      
-
       y_true = Nx.linspace(1, 50, n: 50)
       y_pred = Nx.add(y_true, 1)
       y_pred_2 = Nx.add(y_true, -1)
 
       assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(0.5)
       assert Regression.mean_pinball_loss(y_true, y_pred_2) == Nx.tensor(0.5)
-      assert Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.4) == Nx.tensor(0.6)
-      assert Regression.mean_pinball_loss(y_true, y_pred_2, alpha: 0.4) == Nx.tensor(0.4)
-    end
-
-    test "mean_pinball_loss with sample weight" do
-      y_true = Nx.tensor([1, 2, 3, 4, 5, 6])
-      y_pred = Nx.tensor([2, 3, 4, 6, 7, 8])
-      sample_weights = Nx.tensor([1.5, 1.5, 1.5, 0.5, 0.5, 0.5])
-      wrong_sample_weights = Nx.tensor([1.5, 1.5, 1.5, 0.5, 0.5, 0.5, 1, 1, 1])
-
-      assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(0.75)
-
-      assert Regression.mean_pinball_loss(
-               y_true,
-               y_pred,
-               alpha: 0.5,
-               sample_weights: sample_weights
-             ) == Nx.tensor(0.625)
-
-      assert_raise ArgumentError, fn ->
-        Regression.mean_pinball_loss(y_true, y_pred,
-          alpha: 0.5,
-          sample_weights: wrong_sample_weights
-        )
-      end
-    end
-
-    test "mean_pinball_loss with multioutput" do
-      y_true = Nx.tensor([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
-      y_pred = Nx.tensor([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])
-
-      sample_weight =
-        Nx.tensor([[0.5, 0.5, 0.5, 1.5], [1.5, 0.5, 1.5, 1.5], [1.5, 1.5, 1.5, 1.5]])
-
-      expected_error = Nx.tensor((1 + 2 / 3) / 8)
-      expected_raw_values_tensor = Nx.tensor([0.5, 0.33333333, 0.0, 0.0])
-      expected_raw_values_weighted_tensor = Nx.tensor([0.5, 0.4, 0.0, 0.0])
-
-      mpbl = Regression.mean_pinball_loss(y_true, y_pred)
-      assert_all_close(mpbl, expected_error)
-      ## this assertion yields false due to precision error
-      mpbl =
-        Regression.mean_pinball_loss(
-          y_true,
-          y_pred,
-          alpha: 0.5,
-          multioutput: :uniform_average
-        )
-
-      assert_all_close(mpbl, expected_error)
-      mpbl = Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.5, multioutput: :raw_values)
-      assert_all_close(mpbl, expected_raw_values_tensor)
-
-      mpbl =
-        Regression.mean_pinball_loss(y_true, y_pred,
-          alpha: 0.5,
-          sample_weights: sample_weight,
-          multioutput: :raw_values
-        )
-
-      assert_all_close(mpbl, expected_raw_values_weighted_tensor)
-
-      mpbl =
-        Regression.mean_pinball_loss(y_true, y_pred,
-          alpha: 0.5,
-          sample_weights: sample_weight,
-          multioutput: :uniform_average
-        )
-
-      assert_all_close(mpbl, Nx.tensor(0.225))
-
-      mpbl =
-        Regression.mean_pinball_loss(y_true, y_pred,
-          alpha: 0.5,
-          multioutput: Nx.tensor([1, 2, 3, 4])
-        )
-
-      assert_all_close(mpbl, Nx.tensor(0.1166666))
-      mpbl = Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.5, multioutput: nil)
-      assert_all_close(mpbl, expected_error)
+      assert Regression.mean_pinball_loss(y_true, y_pred, 0.4) == Nx.tensor(0.6)            
+      assert Regression.mean_pinball_loss(y_true, y_pred_2, 0.4) == Nx.tensor(0.4)
     end
   end
 end

From ad9297889ee92e898e233a3348f3b9b09eedee65 Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Tue, 27 Feb 2024 09:54:37 +0100
Subject: [PATCH 02/90] add links to origin of cases and formulas

---
 lib/scholar/metrics/regression.ex        | 3 +++
 test/scholar/metrics/regression_test.exs | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index 1a958b98..8146d76d 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -541,6 +541,9 @@ defmodule Scholar.Metrics.Regression do
   """
   defn mean_pinball_loss(y_true, y_pred, alpha \\ 0.5) do
     check_shape(y_true, y_pred)
+
+    # Formula adapted from sklearn:
+    # https://github.com/scikit-learn/scikit-learn/blob/128e40ed593c57e8b9e57a4109928d58fa8bf359/sklearn/metrics/_regression.py#L299
     diff = Nx.subtract(y_true, y_pred)
     sign = Nx.greater_equal(diff, 0)
     subtracted_sign = Nx.subtract(1, sign)
diff --git a/test/scholar/metrics/regression_test.exs b/test/scholar/metrics/regression_test.exs
index 7a2e0fdb..6486a8d8 100644
--- a/test/scholar/metrics/regression_test.exs
+++ b/test/scholar/metrics/regression_test.exs
@@ -69,6 +69,9 @@ defmodule Scholar.Metrics.RegressionTest do
   
   describe "mean_pinball_loss/3" do
     test "mean_pinball_loss cases from sklearn" do
+      # Test cases copied from sklearn:
+      # https://github.com/scikit-learn/scikit-learn/blob/128e40ed593c57e8b9e57a4109928d58fa8bf359/sklearn/metrics/tests/test_regression.py#L49      
+
       y_true = Nx.linspace(1, 50, n: 50)
       y_pred = Nx.add(y_true, 1)
       y_pred_2 = Nx.add(y_true, -1)

From 8cbcabb723a5863279bbc3b3ce665f4dce4dbe9b Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Tue, 27 Feb 2024 12:09:53 +0100
Subject: [PATCH 03/90] Update lib/scholar/metrics/regression.ex

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 lib/scholar/metrics/regression.ex | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index 8146d76d..8e2bf5fd 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -544,16 +544,10 @@ defmodule Scholar.Metrics.Regression do
 
     # Formula adapted from sklearn:
     # https://github.com/scikit-learn/scikit-learn/blob/128e40ed593c57e8b9e57a4109928d58fa8bf359/sklearn/metrics/_regression.py#L299
-    diff = Nx.subtract(y_true, y_pred)
-    sign = Nx.greater_equal(diff, 0)
-    subtracted_sign = Nx.subtract(1, sign)
-    
-    Nx.subtract(
-      Nx.multiply(alpha, sign)
-      |> Nx.multiply(diff),
-      Nx.multiply(1 - alpha, subtracted_sign)
-      |> Nx.multiply(diff))
-    |> Nx.mean()
+    diff = y_true - y_pred
+    sign = diff >= 0
+    loss = alpha * sign * diff - (1 - alpha) * (1 - sign) * diff
+    Nx.mean(loss)
   end
 
   defnp check_shape(y_true, y_pred) do

From 67b716db59f0ea4ef4dd2a21d1583a7acf5dbc89 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Tue, 27 Feb 2024 12:10:13 +0100
Subject: [PATCH 04/90] Update lib/scholar/metrics/regression.ex

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 lib/scholar/metrics/regression.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index 8e2bf5fd..62b04cdd 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -539,8 +539,9 @@ defmodule Scholar.Metrics.Regression do
         0.5
       >
   """
-  defn mean_pinball_loss(y_true, y_pred, alpha \\ 0.5) do
+  defn mean_pinball_loss(y_true, y_pred, opts \\ []) do
     check_shape(y_true, y_pred)
+    alpha = opts[:alpha]
 
     # Formula adapted from sklearn:
     # https://github.com/scikit-learn/scikit-learn/blob/128e40ed593c57e8b9e57a4109928d58fa8bf359/sklearn/metrics/_regression.py#L299

From 864a0dd15247e68f487a955ab2b56f03984203fb Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Wed, 28 Feb 2024 08:59:22 +0100
Subject: [PATCH 05/90] fix tests to use opts in optional arguments

---
 test/scholar/metrics/regression_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/scholar/metrics/regression_test.exs b/test/scholar/metrics/regression_test.exs
index 6486a8d8..9219a8b2 100644
--- a/test/scholar/metrics/regression_test.exs
+++ b/test/scholar/metrics/regression_test.exs
@@ -78,8 +78,8 @@ defmodule Scholar.Metrics.RegressionTest do
 
       assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(0.5)
       assert Regression.mean_pinball_loss(y_true, y_pred_2) == Nx.tensor(0.5)
-      assert Regression.mean_pinball_loss(y_true, y_pred, 0.4) == Nx.tensor(0.6)            
-      assert Regression.mean_pinball_loss(y_true, y_pred_2, 0.4) == Nx.tensor(0.4)
+      assert Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.4) == Nx.tensor(0.6)            
+      assert Regression.mean_pinball_loss(y_true, y_pred_2, alpha: 0.4) == Nx.tensor(0.4)
     end
   end
 end

From 3be98840b18254aaac7d0a72f0e4b32b041a9ee8 Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Wed, 28 Feb 2024 09:00:35 +0100
Subject: [PATCH 06/90] add default alpha 0.5, same as sklearn

---
 lib/scholar/metrics/regression.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index 62b04cdd..055a90e3 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -533,13 +533,13 @@ defmodule Scholar.Metrics.Regression do
 
       iex> y_true = Nx.tensor([1, 2, 3])
       iex> y_pred = Nx.tensor([2, 3, 4])
-      iex> Scholar.Metrics.Regression.mean_pinball_loss(y_true, y_pred, 0.5)
+      iex> Scholar.Metrics.Regression.mean_pinball_loss(y_true, y_pred)
       #Nx.Tensor<
         f32
         0.5
       >
   """
-  defn mean_pinball_loss(y_true, y_pred, opts \\ []) do
+  defn mean_pinball_loss(y_true, y_pred, opts \\ [alpha: 0.5]) do
     check_shape(y_true, y_pred)
     alpha = opts[:alpha]
 

From 85725341ee79eae99d07d5287426633d509438bc Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Wed, 28 Feb 2024 10:16:53 +0100
Subject: [PATCH 07/90] added sample_weights option to mean_pinball_loss

---
 lib/scholar/metrics/regression.ex        | 10 +++++++++-
 test/scholar/metrics/regression_test.exs | 15 +++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index 055a90e3..f0366edd 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -548,7 +548,15 @@ defmodule Scholar.Metrics.Regression do
     diff = y_true - y_pred
     sign = diff >= 0
     loss = alpha * sign * diff - (1 - alpha) * (1 - sign) * diff
-    Nx.mean(loss)
+
+    handle_sample_weights(loss, opts)
+  end
+
+  defnp handle_sample_weights(loss, opts) do
+    case opts[:sample_weights] do
+      nil -> Nx.mean(loss)
+      weights -> Nx.weighted_mean(loss, weights)
+    end
   end
 
   defnp check_shape(y_true, y_pred) do
diff --git a/test/scholar/metrics/regression_test.exs b/test/scholar/metrics/regression_test.exs
index 9219a8b2..45915e2a 100644
--- a/test/scholar/metrics/regression_test.exs
+++ b/test/scholar/metrics/regression_test.exs
@@ -81,5 +81,20 @@ defmodule Scholar.Metrics.RegressionTest do
       assert Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.4) == Nx.tensor(0.6)            
       assert Regression.mean_pinball_loss(y_true, y_pred_2, alpha: 0.4) == Nx.tensor(0.4)
     end
+
+    test "mean_pinball_loss with sample weight" do
+      y_true = Nx.tensor([1, 2, 3, 4, 5, 6])
+      y_pred = Nx.tensor([2, 3, 4, 6, 7, 8])
+      sample_weights = Nx.tensor([1.5, 1.5, 1.5, 0.5, 0.5, 0.5])
+      wrong_sample_weights = Nx.tensor([1.5, 1.5, 1.5, 0.5, 0.5, 0.5, 1, 1, 1])      
+
+      assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(0.75)
+      assert Regression.mean_pinball_loss(
+        y_true, y_pred, alpha: 0.5, sample_weights: sample_weights) == Nx.tensor(0.625)
+      assert_raise ArgumentError, fn ->
+        Regression.mean_pinball_loss(y_true, y_pred,
+          alpha: 0.5, sample_weights: wrong_sample_weights)
+      end
+    end
   end
 end

From 30952da2f517f43ff8cc203b045567cc4ac510bb Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Wed, 28 Feb 2024 18:35:47 +0100
Subject: [PATCH 08/90] added multioutput support

---
 lib/scholar/metrics/regression.ex        | 15 +++++++++++++--
 test/scholar/metrics/regression_test.exs | 13 +++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index f0366edd..e3de7f18 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -538,9 +538,16 @@ defmodule Scholar.Metrics.Regression do
         f32
         0.5
       >
+      iex> y_true = Nx.tensor([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
+      iex> y_pred = Nx.tensor([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])
+      iex> Scholar.Metrics.Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.5, multioutput: :raw_values)
+      #Nx.Tensor<
+        f32[4]
+        [0.5, 0.3333333432674408, 0.0, 0.0]
+      >
   """
   defn mean_pinball_loss(y_true, y_pred, opts \\ [alpha: 0.5]) do
-    check_shape(y_true, y_pred)
+    assert_same_shape!(y_true, y_pred)
     alpha = opts[:alpha]
 
     # Formula adapted from sklearn:
@@ -549,7 +556,11 @@ defmodule Scholar.Metrics.Regression do
     sign = diff >= 0
     loss = alpha * sign * diff - (1 - alpha) * (1 - sign) * diff
 
-    handle_sample_weights(loss, opts)
+    case opts[:multioutput] do
+      :raw_values -> Nx.mean(loss, axes: [0])
+      :uniform_average -> Nx.mean(loss)
+      _ -> handle_sample_weights(loss, opts)
+    end
   end
 
   defnp handle_sample_weights(loss, opts) do
diff --git a/test/scholar/metrics/regression_test.exs b/test/scholar/metrics/regression_test.exs
index 45915e2a..67a0f356 100644
--- a/test/scholar/metrics/regression_test.exs
+++ b/test/scholar/metrics/regression_test.exs
@@ -96,5 +96,18 @@ defmodule Scholar.Metrics.RegressionTest do
           alpha: 0.5, sample_weights: wrong_sample_weights)
       end
     end
+
+    test "mean_pinball_loss with multioutput" do
+      y_true = Nx.tensor([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
+      y_pred = Nx.tensor([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])
+      expected_error = (1 + 2 / 3) / 8
+      expected_raw_values_tensor = Nx.tensor([0.5, 0.33333333, 0.0, 0.0])
+
+      assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(expected_error)
+      assert Regression.mean_pinball_loss(y_true, y_pred,
+        alpha: 0.5, multioutput: :uniform_average) == Nx.tensor(expected_error)      
+      assert Regression.mean_pinball_loss(y_true, y_pred,
+        alpha: 0.5, multioutput: :raw_values) == expected_raw_values_tensor
+    end    
   end
 end

From 0ca67258a420a3f3c056122233860bc4d90be9d9 Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Wed, 28 Feb 2024 19:16:09 +0100
Subject: [PATCH 09/90] fixed multioutput behavior to be on par with sklearn

---
 lib/scholar/metrics/regression.ex        | 17 +++++++++++------
 test/scholar/metrics/regression_test.exs | 19 ++++++++++++++-----
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index e3de7f18..9f27f3e3 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -556,17 +556,22 @@ defmodule Scholar.Metrics.Regression do
     sign = diff >= 0
     loss = alpha * sign * diff - (1 - alpha) * (1 - sign) * diff
 
+    output_errors = handle_sample_weights(loss, opts, axes: [0])
+    # mimics the sklearn behavior
     case opts[:multioutput] do
-      :raw_values -> Nx.mean(loss, axes: [0])
-      :uniform_average -> Nx.mean(loss)
+      :raw_values -> output_errors
+      :uniform_average ->
+        output_errors
+        |> Nx.mean()
       _ -> handle_sample_weights(loss, opts)
     end
   end
 
-  defnp handle_sample_weights(loss, opts) do
-    case opts[:sample_weights] do
-      nil -> Nx.mean(loss)
-      weights -> Nx.weighted_mean(loss, weights)
+  defnp handle_sample_weights(loss, opts, mean_opts \\ []) do
+    case opts[:sample_weight] do
+      nil -> Nx.mean(loss, mean_opts)
+      weights ->
+        Nx.weighted_mean(loss, weights, mean_opts)
     end
   end
 
diff --git a/test/scholar/metrics/regression_test.exs b/test/scholar/metrics/regression_test.exs
index 67a0f356..fe8cf003 100644
--- a/test/scholar/metrics/regression_test.exs
+++ b/test/scholar/metrics/regression_test.exs
@@ -90,24 +90,33 @@ defmodule Scholar.Metrics.RegressionTest do
 
       assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(0.75)
       assert Regression.mean_pinball_loss(
-        y_true, y_pred, alpha: 0.5, sample_weights: sample_weights) == Nx.tensor(0.625)
+        y_true, y_pred, alpha: 0.5, sample_weight: sample_weights) == Nx.tensor(0.625)
       assert_raise ArgumentError, fn ->
         Regression.mean_pinball_loss(y_true, y_pred,
-          alpha: 0.5, sample_weights: wrong_sample_weights)
+          alpha: 0.5, sample_weight: wrong_sample_weights)
       end
     end
 
     test "mean_pinball_loss with multioutput" do
       y_true = Nx.tensor([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
       y_pred = Nx.tensor([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])
+      sample_weight = Nx.tensor([[0.5, 0.5, 0.5, 1.5], [1.5, 0.5, 1.5, 1.5], [1.5, 1.5, 1.5, 1.5]])
       expected_error = (1 + 2 / 3) / 8
       expected_raw_values_tensor = Nx.tensor([0.5, 0.33333333, 0.0, 0.0])
+      expected_raw_values_weighted_tensor = Nx.tensor([0.5, 0.4, 0.0 , 0.0])
 
       assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(expected_error)
-      assert Regression.mean_pinball_loss(y_true, y_pred,
-        alpha: 0.5, multioutput: :uniform_average) == Nx.tensor(expected_error)      
+      ## this assertion yields false due to precission error
+      mpbl = Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.5, multioutput: :uniform_average)
+      assert  Nx.abs(Nx.subtract(mpbl, Nx.tensor(expected_error))) <= Nx.tensor(0.0001)
       assert Regression.mean_pinball_loss(y_true, y_pred,
         alpha: 0.5, multioutput: :raw_values) == expected_raw_values_tensor
-    end    
+      assert Regression.mean_pinball_loss(
+        y_true, y_pred, alpha: 0.5,
+        sample_weight: sample_weight,  multioutput: :raw_values) == expected_raw_values_weighted_tensor
+      assert Regression.mean_pinball_loss(
+        y_true, y_pred, alpha: 0.5,
+        sample_weight: sample_weight, multioutput: :uniform_average) == Nx.tensor(0.225)
+    end
   end
 end

From cb2a2fac70a310ec7d3ace21917a8eefc6b9f15e Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Wed, 28 Feb 2024 19:18:20 +0100
Subject: [PATCH 10/90] added comments for better multioutput understanding

---
 lib/scholar/metrics/regression.ex | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index 9f27f3e3..478d3f89 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -559,10 +559,13 @@ defmodule Scholar.Metrics.Regression do
     output_errors = handle_sample_weights(loss, opts, axes: [0])
     # mimics the sklearn behavior
     case opts[:multioutput] do
+      # raw_values returns plain output errors. One value per channel.
       :raw_values -> output_errors
+      # uniform_average returns the mean of the above. Note how they are averaged.
       :uniform_average ->
         output_errors
         |> Nx.mean()
+      # every other case, just take the mean without any axes nor weight.
       _ -> handle_sample_weights(loss, opts)
     end
   end

From 0ab4741c783fe745d73a8332de0eb07de10465c6 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 1 Mar 2024 17:06:57 +0100
Subject: [PATCH 11/90] add option to allow for 2 dimensional sample weights

---
 lib/scholar/options.ex | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/scholar/options.ex b/lib/scholar/options.ex
index e1ac99c9..cfc59176 100644
--- a/lib/scholar/options.ex
+++ b/lib/scholar/options.ex
@@ -82,6 +82,16 @@ defmodule Scholar.Options do
       {:error, "expected weights to be a flat tensor or a flat list, got: #{inspect(weights)}"}
     end
   end
+  
+  def multi_weights(weights) do
+    if is_nil(weights) or
+         (Nx.is_tensor(weights) and Nx.rank(weights) > 0) or
+         (is_list(weights) and Enum.all?(weights, &is_number/1)) do
+      {:ok, weights}
+    else
+      {:error, "expected weights to be a flat tensor or a flat list, got: #{inspect(weights)}"}
+    end
+  end
 
   def multi_weights(weights) do
     if is_nil(weights) or

From b7f4a9dd41b33add4297daf2d3236763d87003d5 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 1 Mar 2024 17:07:34 +0100
Subject: [PATCH 12/90] add nimble options and rename sample_weight to
 sample_weights. consistent with regression

---
 lib/scholar/metrics/regression.ex | 60 +++++++++++++++++++++++++++++--
 1 file changed, 57 insertions(+), 3 deletions(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index 478d3f89..83c53cf4 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -519,6 +519,54 @@ defmodule Scholar.Metrics.Regression do
     Nx.reduce_max(Nx.abs(y_true - y_pred))
   end
 
+  mean_pinball_loss_opts = [
+    alpha: [
+      type: :float,
+      default: 0.5,
+      doc: """
+      The slope of the pinball loss, default=0.5,
+      This loss is equivalent to $$mean_absolute_error$$ when $$\alpha$$ is 0.5,
+      $$\alpha = 0.95$$ is minimized by estimators of the 95th percentile.
+      """
+    ],
+    sample_weights: [
+      type:
+        {:or,
+         [
+           {:custom, Scholar.Options, :weights, []},
+           {:custom, Scholar.Options, :multi_weights, []}           
+         ]},
+      doc: """
+      The weights for each observation. If not provided,
+      all observations are assigned equal weight.
+      """
+    ],
+    multioutput: [
+      type: {:or,
+             [
+               {:custom, Scholar.Options, :weights, []},
+               {:in, [:raw_values, :uniform_average]}
+             ]
+      },
+      default: :uniform_average,
+      doc: """
+      Defines aggregating of multiple output values.
+      Array-like value defines weights used to average errors.
+      Defaults to `:uniform_average`.
+
+        `:raw_values` :
+            Returns a full set of errors in case of multioutput input.
+
+        `:uniform_average` :
+            Errors of all outputs are averaged with uniform weight.
+      
+      The weights for each observation. If not provided,
+      all observations are assigned equal weight.
+      """      
+    ]
+  ]
+  @mean_pinball_loss_schema NimbleOptions.new!(mean_pinball_loss_opts)
+  
   @doc ~S"""
   Calculates the mean pinball loss to evaluate predictive performance of quantile regression models.
 
@@ -527,7 +575,8 @@ defmodule Scholar.Metrics.Regression do
 
   The residual error is defined as $$|y - \hat{y}|$$ where $y$ is a true value
   and $\hat{y}$ is a predicted value.
-  Equivalent to half of the $$mean_absolute_error$$ when $$\alpha$$ is 0.5.
+
+  #{NimbleOptions.docs(@mean_pinball_loss_schema)}
 
   ## Examples
 
@@ -546,7 +595,8 @@ defmodule Scholar.Metrics.Regression do
         [0.5, 0.3333333432674408, 0.0, 0.0]
       >
   """
-  defn mean_pinball_loss(y_true, y_pred, opts \\ [alpha: 0.5]) do
+  defn mean_pinball_loss(y_true, y_pred, opts \\ []) do
+    opts = validate_opts(opts)
     assert_same_shape!(y_true, y_pred)
     alpha = opts[:alpha]
 
@@ -570,8 +620,12 @@ defmodule Scholar.Metrics.Regression do
     end
   end
 
+  deftransform validate_opts(opts) do
+    NimbleOptions.validate!(opts, @mean_pinball_loss_schema)
+  end
+  
   defnp handle_sample_weights(loss, opts, mean_opts \\ []) do
-    case opts[:sample_weight] do
+    case opts[:sample_weights] do
       nil -> Nx.mean(loss, mean_opts)
       weights ->
         Nx.weighted_mean(loss, weights, mean_opts)

From ae8b3b0d12863bbce378bc419c75b7fc075641d0 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 1 Mar 2024 17:08:14 +0100
Subject: [PATCH 13/90] fix tests

---
 test/scholar/metrics/regression_test.exs | 38 +++++++++++++++---------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/test/scholar/metrics/regression_test.exs b/test/scholar/metrics/regression_test.exs
index fe8cf003..1686b3ad 100644
--- a/test/scholar/metrics/regression_test.exs
+++ b/test/scholar/metrics/regression_test.exs
@@ -90,10 +90,10 @@ defmodule Scholar.Metrics.RegressionTest do
 
       assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(0.75)
       assert Regression.mean_pinball_loss(
-        y_true, y_pred, alpha: 0.5, sample_weight: sample_weights) == Nx.tensor(0.625)
+        y_true, y_pred, alpha: 0.5, sample_weights: sample_weights) == Nx.tensor(0.625)
       assert_raise ArgumentError, fn ->
         Regression.mean_pinball_loss(y_true, y_pred,
-          alpha: 0.5, sample_weight: wrong_sample_weights)
+          alpha: 0.5, sample_weights: wrong_sample_weights)
       end
     end
 
@@ -105,18 +105,28 @@ defmodule Scholar.Metrics.RegressionTest do
       expected_raw_values_tensor = Nx.tensor([0.5, 0.33333333, 0.0, 0.0])
       expected_raw_values_weighted_tensor = Nx.tensor([0.5, 0.4, 0.0 , 0.0])
 
-      assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(expected_error)
-      ## this assertion yields false due to precission error
-      mpbl = Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.5, multioutput: :uniform_average)
-      assert  Nx.abs(Nx.subtract(mpbl, Nx.tensor(expected_error))) <= Nx.tensor(0.0001)
-      assert Regression.mean_pinball_loss(y_true, y_pred,
-        alpha: 0.5, multioutput: :raw_values) == expected_raw_values_tensor
-      assert Regression.mean_pinball_loss(
-        y_true, y_pred, alpha: 0.5,
-        sample_weight: sample_weight,  multioutput: :raw_values) == expected_raw_values_weighted_tensor
-      assert Regression.mean_pinball_loss(
-        y_true, y_pred, alpha: 0.5,
-        sample_weight: sample_weight, multioutput: :uniform_average) == Nx.tensor(0.225)
+      mpbl = Regression.mean_pinball_loss(y_true, y_pred)
+      assert almost_equal(mpbl, Nx.tensor(expected_error))
+      ## this assertion yields false due to precision error
+      mpbl = Regression.mean_pinball_loss(
+               y_true, y_pred, alpha: 0.5, multioutput: :uniform_average)
+      assert almost_equal(mpbl, Nx.tensor(expected_error))
+      mpbl = Regression.mean_pinball_loss(y_true, y_pred,
+        alpha: 0.5, multioutput: :raw_values)
+      assert almost_equal(mpbl, expected_raw_values_tensor)
+      mpbl = Regression.mean_pinball_loss(y_true, y_pred,
+        alpha: 0.5, sample_weights: sample_weight,  multioutput: :raw_values)
+      assert almost_equal(mpbl, expected_raw_values_weighted_tensor)
+      mpbl = Regression.mean_pinball_loss(y_true, y_pred,
+        alpha: 0.5, sample_weights: sample_weight, multioutput: :uniform_average)
+      assert almost_equal(mpbl, Nx.tensor(0.225))
+      mpbl = Regression.mean_pinball_loss(y_true, y_pred,
+        alpha: 0.5, multioutput: Nx.tensor([1, 2, 3]))
+      assert almost_equal(mpbl, expected_raw_values_weighted_tensor)
     end
   end
+
+  defp almost_equal(tensor1, tensor2, tolerance \\ 0.0001) do
+    Nx.abs(Nx.subtract(tensor1, tensor2)) <= Nx.tensor(tolerance)
+  end
 end

From afe88eecbd1daf4c3c81431a39b2182141bc140e Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sat, 2 Mar 2024 10:39:38 +0100
Subject: [PATCH 14/90] fixed sample_weights: as tensor behaviour

---
 lib/scholar/metrics/regression.ex        |  9 +++------
 test/scholar/metrics/regression_test.exs | 13 ++++++++-----
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index 83c53cf4..f6b3036b 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -615,15 +615,12 @@ defmodule Scholar.Metrics.Regression do
       :uniform_average ->
         output_errors
         |> Nx.mean()
-      # every other case, just take the mean without any axes nor weight.
-      _ -> handle_sample_weights(loss, opts)
+      # pass `:multioutput` as sample weights to average the error of each output
+      multi_output_weights ->
+        handle_sample_weights(output_errors, [sample_weights: multi_output_weights])
     end
   end
 
-  deftransform validate_opts(opts) do
-    NimbleOptions.validate!(opts, @mean_pinball_loss_schema)
-  end
-  
   defnp handle_sample_weights(loss, opts, mean_opts \\ []) do
     case opts[:sample_weights] do
       nil -> Nx.mean(loss, mean_opts)
diff --git a/test/scholar/metrics/regression_test.exs b/test/scholar/metrics/regression_test.exs
index 1686b3ad..c783f8a7 100644
--- a/test/scholar/metrics/regression_test.exs
+++ b/test/scholar/metrics/regression_test.exs
@@ -101,16 +101,16 @@ defmodule Scholar.Metrics.RegressionTest do
       y_true = Nx.tensor([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
       y_pred = Nx.tensor([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])
       sample_weight = Nx.tensor([[0.5, 0.5, 0.5, 1.5], [1.5, 0.5, 1.5, 1.5], [1.5, 1.5, 1.5, 1.5]])
-      expected_error = (1 + 2 / 3) / 8
+      expected_error = Nx.tensor((1 + 2 / 3) / 8)
       expected_raw_values_tensor = Nx.tensor([0.5, 0.33333333, 0.0, 0.0])
       expected_raw_values_weighted_tensor = Nx.tensor([0.5, 0.4, 0.0 , 0.0])
 
       mpbl = Regression.mean_pinball_loss(y_true, y_pred)
-      assert almost_equal(mpbl, Nx.tensor(expected_error))
+      assert almost_equal(mpbl, expected_error)
       ## this assertion yields false due to precision error
       mpbl = Regression.mean_pinball_loss(
                y_true, y_pred, alpha: 0.5, multioutput: :uniform_average)
-      assert almost_equal(mpbl, Nx.tensor(expected_error))
+      assert almost_equal(mpbl, expected_error)
       mpbl = Regression.mean_pinball_loss(y_true, y_pred,
         alpha: 0.5, multioutput: :raw_values)
       assert almost_equal(mpbl, expected_raw_values_tensor)
@@ -121,8 +121,11 @@ defmodule Scholar.Metrics.RegressionTest do
         alpha: 0.5, sample_weights: sample_weight, multioutput: :uniform_average)
       assert almost_equal(mpbl, Nx.tensor(0.225))
       mpbl = Regression.mean_pinball_loss(y_true, y_pred,
-        alpha: 0.5, multioutput: Nx.tensor([1, 2, 3]))
-      assert almost_equal(mpbl, expected_raw_values_weighted_tensor)
+        alpha: 0.5, multioutput: Nx.tensor([1, 2, 3, 4]))
+      assert almost_equal(mpbl, Nx.tensor(0.1166666))
+      mpbl = Regression.mean_pinball_loss(y_true, y_pred,
+        alpha: 0.5, multioutput: nil)
+      assert almost_equal(mpbl, expected_error)
     end
   end
 

From 421a8ef809e683ef73b3bc40f2085b773b355a3e Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sat, 2 Mar 2024 10:40:03 +0100
Subject: [PATCH 15/90] fixed call to NimbleOptions to be consistent

---
 lib/scholar/metrics/regression.ex | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index f6b3036b..b70a5095 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -595,8 +595,10 @@ defmodule Scholar.Metrics.Regression do
         [0.5, 0.3333333432674408, 0.0, 0.0]
       >
   """
-  defn mean_pinball_loss(y_true, y_pred, opts \\ []) do
-    opts = validate_opts(opts)
+  deftransform mean_pinball_loss(y_true, y_pred, opts \\ []) do
+    mean_pinball_loss_n(y_true, y_pred, NimbleOptions.validate!(opts, @mean_pinball_loss_schema))
+  end
+  defnp mean_pinball_loss_n(y_true, y_pred, opts \\ []) do
     assert_same_shape!(y_true, y_pred)
     alpha = opts[:alpha]
 

From 9be10e4cbc0b05e2fdbe68d03b024c181ec3dcc4 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sat, 2 Mar 2024 10:54:03 +0100
Subject: [PATCH 16/90] fixed multi_weights option and docs

---
 lib/scholar/options.ex | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/scholar/options.ex b/lib/scholar/options.ex
index cfc59176..6b0066ad 100644
--- a/lib/scholar/options.ex
+++ b/lib/scholar/options.ex
@@ -85,11 +85,10 @@ defmodule Scholar.Options do
   
   def multi_weights(weights) do
     if is_nil(weights) or
-         (Nx.is_tensor(weights) and Nx.rank(weights) > 0) or
-         (is_list(weights) and Enum.all?(weights, &is_number/1)) do
+         (Nx.is_tensor(weights) and Nx.rank(weights) > 1) do
       {:ok, weights}
     else
-      {:error, "expected weights to be a flat tensor or a flat list, got: #{inspect(weights)}"}
+      {:error, "expected weights to be a tensor with rank greater than 1, got: #{inspect(weights)}"}
     end
   end
 

From 02b285de5dc99fb6e6d355d116810ea413694b3c Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Sun, 3 Mar 2024 19:25:54 +0100
Subject: [PATCH 17/90] Update lib/scholar/metrics/regression.ex

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 lib/scholar/metrics/regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index b70a5095..6da7926e 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -598,7 +598,7 @@ defmodule Scholar.Metrics.Regression do
   deftransform mean_pinball_loss(y_true, y_pred, opts \\ []) do
     mean_pinball_loss_n(y_true, y_pred, NimbleOptions.validate!(opts, @mean_pinball_loss_schema))
   end
-  defnp mean_pinball_loss_n(y_true, y_pred, opts \\ []) do
+  defnp mean_pinball_loss_n(y_true, y_pred, opts) do
     assert_same_shape!(y_true, y_pred)
     alpha = opts[:alpha]
 

From 373eda00435c00ce260f85c233a5f7efe6d78c24 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sun, 3 Mar 2024 19:28:37 +0100
Subject: [PATCH 18/90] use assert_all_close on multi output pinball loss tests

---
 test/scholar/metrics/regression_test.exs | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/test/scholar/metrics/regression_test.exs b/test/scholar/metrics/regression_test.exs
index c783f8a7..112faa86 100644
--- a/test/scholar/metrics/regression_test.exs
+++ b/test/scholar/metrics/regression_test.exs
@@ -106,30 +106,26 @@ defmodule Scholar.Metrics.RegressionTest do
       expected_raw_values_weighted_tensor = Nx.tensor([0.5, 0.4, 0.0 , 0.0])
 
       mpbl = Regression.mean_pinball_loss(y_true, y_pred)
-      assert almost_equal(mpbl, expected_error)
+      assert_all_close(mpbl, expected_error)
       ## this assertion yields false due to precision error
       mpbl = Regression.mean_pinball_loss(
                y_true, y_pred, alpha: 0.5, multioutput: :uniform_average)
-      assert almost_equal(mpbl, expected_error)
+      assert_all_close(mpbl, expected_error)
       mpbl = Regression.mean_pinball_loss(y_true, y_pred,
         alpha: 0.5, multioutput: :raw_values)
-      assert almost_equal(mpbl, expected_raw_values_tensor)
+      assert_all_close(mpbl, expected_raw_values_tensor)
       mpbl = Regression.mean_pinball_loss(y_true, y_pred,
         alpha: 0.5, sample_weights: sample_weight,  multioutput: :raw_values)
-      assert almost_equal(mpbl, expected_raw_values_weighted_tensor)
+      assert_all_close(mpbl, expected_raw_values_weighted_tensor)
       mpbl = Regression.mean_pinball_loss(y_true, y_pred,
         alpha: 0.5, sample_weights: sample_weight, multioutput: :uniform_average)
-      assert almost_equal(mpbl, Nx.tensor(0.225))
+      assert_all_close(mpbl, Nx.tensor(0.225))
       mpbl = Regression.mean_pinball_loss(y_true, y_pred,
         alpha: 0.5, multioutput: Nx.tensor([1, 2, 3, 4]))
-      assert almost_equal(mpbl, Nx.tensor(0.1166666))
+      assert_all_close(mpbl, Nx.tensor(0.1166666))
       mpbl = Regression.mean_pinball_loss(y_true, y_pred,
         alpha: 0.5, multioutput: nil)
-      assert almost_equal(mpbl, expected_error)
+      assert_all_close(mpbl, expected_error)
     end
   end
-
-  defp almost_equal(tensor1, tensor2, tolerance \\ 0.0001) do
-    Nx.abs(Nx.subtract(tensor1, tensor2)) <= Nx.tensor(tolerance)
-  end
 end

From 3c508a49453d6980bc582e934a5dff0b397f1a8a Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sun, 3 Mar 2024 21:02:37 +0100
Subject: [PATCH 19/90] run formatter

---
 lib/scholar/metrics/regression.ex        | 33 ++++++-----
 lib/scholar/options.ex                   |  5 +-
 test/scholar/metrics/regression_test.exs | 70 +++++++++++++++++-------
 3 files changed, 74 insertions(+), 34 deletions(-)

diff --git a/lib/scholar/metrics/regression.ex b/lib/scholar/metrics/regression.ex
index 6da7926e..b26e4605 100644
--- a/lib/scholar/metrics/regression.ex
+++ b/lib/scholar/metrics/regression.ex
@@ -534,7 +534,7 @@ defmodule Scholar.Metrics.Regression do
         {:or,
          [
            {:custom, Scholar.Options, :weights, []},
-           {:custom, Scholar.Options, :multi_weights, []}           
+           {:custom, Scholar.Options, :multi_weights, []}
          ]},
       doc: """
       The weights for each observation. If not provided,
@@ -542,12 +542,12 @@ defmodule Scholar.Metrics.Regression do
       """
     ],
     multioutput: [
-      type: {:or,
-             [
-               {:custom, Scholar.Options, :weights, []},
-               {:in, [:raw_values, :uniform_average]}
-             ]
-      },
+      type:
+        {:or,
+         [
+           {:custom, Scholar.Options, :weights, []},
+           {:in, [:raw_values, :uniform_average]}
+         ]},
       default: :uniform_average,
       doc: """
       Defines aggregating of multiple output values.
@@ -559,14 +559,15 @@ defmodule Scholar.Metrics.Regression do
 
         `:uniform_average` :
             Errors of all outputs are averaged with uniform weight.
-      
+
       The weights for each observation. If not provided,
       all observations are assigned equal weight.
-      """      
+      """
     ]
   ]
+
   @mean_pinball_loss_schema NimbleOptions.new!(mean_pinball_loss_opts)
-  
+
   @doc ~S"""
   Calculates the mean pinball loss to evaluate predictive performance of quantile regression models.
 
@@ -598,6 +599,7 @@ defmodule Scholar.Metrics.Regression do
   deftransform mean_pinball_loss(y_true, y_pred, opts \\ []) do
     mean_pinball_loss_n(y_true, y_pred, NimbleOptions.validate!(opts, @mean_pinball_loss_schema))
   end
+
   defnp mean_pinball_loss_n(y_true, y_pred, opts) do
     assert_same_shape!(y_true, y_pred)
     alpha = opts[:alpha]
@@ -612,20 +614,25 @@ defmodule Scholar.Metrics.Regression do
     # mimics the sklearn behavior
     case opts[:multioutput] do
       # raw_values returns plain output errors. One value per channel.
-      :raw_values -> output_errors
+      :raw_values ->
+        output_errors
+
       # uniform_average returns the mean of the above. Note how they are averaged.
       :uniform_average ->
         output_errors
         |> Nx.mean()
+
       # pass `:multioutput` as sample weights to average the error of each output
       multi_output_weights ->
-        handle_sample_weights(output_errors, [sample_weights: multi_output_weights])
+        handle_sample_weights(output_errors, sample_weights: multi_output_weights)
     end
   end
 
   defnp handle_sample_weights(loss, opts, mean_opts \\ []) do
     case opts[:sample_weights] do
-      nil -> Nx.mean(loss, mean_opts)
+      nil ->
+        Nx.mean(loss, mean_opts)
+
       weights ->
         Nx.weighted_mean(loss, weights, mean_opts)
     end
diff --git a/lib/scholar/options.ex b/lib/scholar/options.ex
index 6b0066ad..d47489af 100644
--- a/lib/scholar/options.ex
+++ b/lib/scholar/options.ex
@@ -82,13 +82,14 @@ defmodule Scholar.Options do
       {:error, "expected weights to be a flat tensor or a flat list, got: #{inspect(weights)}"}
     end
   end
-  
+
   def multi_weights(weights) do
     if is_nil(weights) or
          (Nx.is_tensor(weights) and Nx.rank(weights) > 1) do
       {:ok, weights}
     else
-      {:error, "expected weights to be a tensor with rank greater than 1, got: #{inspect(weights)}"}
+      {:error,
+       "expected weights to be a tensor with rank greater than 1, got: #{inspect(weights)}"}
     end
   end
 
diff --git a/test/scholar/metrics/regression_test.exs b/test/scholar/metrics/regression_test.exs
index 112faa86..ef305e75 100644
--- a/test/scholar/metrics/regression_test.exs
+++ b/test/scholar/metrics/regression_test.exs
@@ -66,7 +66,7 @@ defmodule Scholar.Metrics.RegressionTest do
       assert Nx.equal(d2, r2)
     end
   end
-  
+
   describe "mean_pinball_loss/3" do
     test "mean_pinball_loss cases from sklearn" do
       # Test cases copied from sklearn:
@@ -78,7 +78,7 @@ defmodule Scholar.Metrics.RegressionTest do
 
       assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(0.5)
       assert Regression.mean_pinball_loss(y_true, y_pred_2) == Nx.tensor(0.5)
-      assert Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.4) == Nx.tensor(0.6)            
+      assert Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.4) == Nx.tensor(0.6)
       assert Regression.mean_pinball_loss(y_true, y_pred_2, alpha: 0.4) == Nx.tensor(0.4)
     end
 
@@ -86,45 +86,77 @@ defmodule Scholar.Metrics.RegressionTest do
       y_true = Nx.tensor([1, 2, 3, 4, 5, 6])
       y_pred = Nx.tensor([2, 3, 4, 6, 7, 8])
       sample_weights = Nx.tensor([1.5, 1.5, 1.5, 0.5, 0.5, 0.5])
-      wrong_sample_weights = Nx.tensor([1.5, 1.5, 1.5, 0.5, 0.5, 0.5, 1, 1, 1])      
+      wrong_sample_weights = Nx.tensor([1.5, 1.5, 1.5, 0.5, 0.5, 0.5, 1, 1, 1])
 
       assert Regression.mean_pinball_loss(y_true, y_pred) == Nx.tensor(0.75)
+
       assert Regression.mean_pinball_loss(
-        y_true, y_pred, alpha: 0.5, sample_weights: sample_weights) == Nx.tensor(0.625)
+               y_true,
+               y_pred,
+               alpha: 0.5,
+               sample_weights: sample_weights
+             ) == Nx.tensor(0.625)
+
       assert_raise ArgumentError, fn ->
         Regression.mean_pinball_loss(y_true, y_pred,
-          alpha: 0.5, sample_weights: wrong_sample_weights)
+          alpha: 0.5,
+          sample_weights: wrong_sample_weights
+        )
       end
     end
 
     test "mean_pinball_loss with multioutput" do
       y_true = Nx.tensor([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
       y_pred = Nx.tensor([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])
-      sample_weight = Nx.tensor([[0.5, 0.5, 0.5, 1.5], [1.5, 0.5, 1.5, 1.5], [1.5, 1.5, 1.5, 1.5]])
+
+      sample_weight =
+        Nx.tensor([[0.5, 0.5, 0.5, 1.5], [1.5, 0.5, 1.5, 1.5], [1.5, 1.5, 1.5, 1.5]])
+
       expected_error = Nx.tensor((1 + 2 / 3) / 8)
       expected_raw_values_tensor = Nx.tensor([0.5, 0.33333333, 0.0, 0.0])
-      expected_raw_values_weighted_tensor = Nx.tensor([0.5, 0.4, 0.0 , 0.0])
+      expected_raw_values_weighted_tensor = Nx.tensor([0.5, 0.4, 0.0, 0.0])
 
       mpbl = Regression.mean_pinball_loss(y_true, y_pred)
       assert_all_close(mpbl, expected_error)
       ## this assertion yields false due to precision error
-      mpbl = Regression.mean_pinball_loss(
-               y_true, y_pred, alpha: 0.5, multioutput: :uniform_average)
+      mpbl =
+        Regression.mean_pinball_loss(
+          y_true,
+          y_pred,
+          alpha: 0.5,
+          multioutput: :uniform_average
+        )
+
       assert_all_close(mpbl, expected_error)
-      mpbl = Regression.mean_pinball_loss(y_true, y_pred,
-        alpha: 0.5, multioutput: :raw_values)
+      mpbl = Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.5, multioutput: :raw_values)
       assert_all_close(mpbl, expected_raw_values_tensor)
-      mpbl = Regression.mean_pinball_loss(y_true, y_pred,
-        alpha: 0.5, sample_weights: sample_weight,  multioutput: :raw_values)
+
+      mpbl =
+        Regression.mean_pinball_loss(y_true, y_pred,
+          alpha: 0.5,
+          sample_weights: sample_weight,
+          multioutput: :raw_values
+        )
+
       assert_all_close(mpbl, expected_raw_values_weighted_tensor)
-      mpbl = Regression.mean_pinball_loss(y_true, y_pred,
-        alpha: 0.5, sample_weights: sample_weight, multioutput: :uniform_average)
+
+      mpbl =
+        Regression.mean_pinball_loss(y_true, y_pred,
+          alpha: 0.5,
+          sample_weights: sample_weight,
+          multioutput: :uniform_average
+        )
+
       assert_all_close(mpbl, Nx.tensor(0.225))
-      mpbl = Regression.mean_pinball_loss(y_true, y_pred,
-        alpha: 0.5, multioutput: Nx.tensor([1, 2, 3, 4]))
+
+      mpbl =
+        Regression.mean_pinball_loss(y_true, y_pred,
+          alpha: 0.5,
+          multioutput: Nx.tensor([1, 2, 3, 4])
+        )
+
       assert_all_close(mpbl, Nx.tensor(0.1166666))
-      mpbl = Regression.mean_pinball_loss(y_true, y_pred,
-        alpha: 0.5, multioutput: nil)
+      mpbl = Regression.mean_pinball_loss(y_true, y_pred, alpha: 0.5, multioutput: nil)
       assert_all_close(mpbl, expected_error)
     end
   end

From 2cf1c98721ded31e757e542aa43a7d215e158712 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 21 Mar 2024 23:45:46 +0100
Subject: [PATCH 20/90] working on bayesian ridge

---
 .../linear/bayesian_ridge_regression.ex       | 105 ++++++++++++++++++
 .../linear/bayesian_ridge_regression_test.exs |  30 +++++
 2 files changed, 135 insertions(+)
 create mode 100644 lib/scholar/linear/bayesian_ridge_regression.ex
 create mode 100644 test/scholar/linear/bayesian_ridge_regression_test.exs

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
new file mode 100644
index 00000000..98810be2
--- /dev/null
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -0,0 +1,105 @@
+defmodule Scholar.Linear.BayesianRidgeRegression do
+  require Nx
+  import Nx.Defn
+  import Scholar.Shared
+
+  @derive {Nx.Container, containers: [:coefficients, :intercept]}
+  defstruct [:coefficients, :intercept]
+  opts = [
+    sample_weights: [
+      type:
+        {:or,
+         [
+           {:custom, Scholar.Options, :non_negative_number, []},
+           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
+           {:custom, Scholar.Options, :weights, []}
+         ]},
+      doc: """
+      The weights for each observation. If not provided,
+      all observations are assigned equal weight.
+      """
+    ],
+    fit_intercept?: [
+      type: :boolean,
+      default: true,
+      doc: """
+      If set to `true`, a model will fit the intercept. Otherwise,
+      the intercept is set to `0.0`. The intercept is an independent term
+      in a linear model. Specifically, it is the expected mean value
+      of targets for a zero-vector on input.
+      """
+    ],
+    solver: [
+      type: {:in, [:svd, :cholesky]},
+      default: :svd,
+      doc: """
+      Solver to use in the computational routines:
+
+      * `:svd` - Uses a Singular Value Decomposition of A to compute the Ridge coefficients.
+      In particular, it is more stable for singular matrices than `:cholesky` at the cost of being slower.
+
+      * `:cholesky` - Uses the standard `Nx.LinAlg.solve` function to obtain a closed-form solution.
+      """
+    ],
+    alpha: [
+      type:
+        {:or,
+         [
+           {:custom, Scholar.Options, :non_negative_number, []},
+           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
+           {:custom, Scholar.Options, :weights, []}
+         ]},
+      default: 1.0,
+      doc: ~S"""
+      Constant that multiplies the $L_2$ term, controlling regularization strength.
+      `:alpha` must be a non-negative float i.e. in [0, inf).
+
+      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
+      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      """
+    ]
+  ]
+  @opts_schema NimbleOptions.new!(opts)
+  deftransform fit(x, y, opts \\ []) do
+    opts = NimbleOptions.validate!(opts, @opts_schema)    
+        {sample_weights, opts} = Keyword.pop(opts, :sample_weights, 1.0)
+    x_type = to_float_type(x)
+
+    sample_weights =
+      if Nx.is_tensor(sample_weights),
+        do: Nx.as_type(sample_weights, x_type),
+        else: Nx.tensor(sample_weights, type: x_type)
+
+    {alpha, opts} = Keyword.pop!(opts, :alpha)
+    alpha = Nx.tensor(alpha, type: x_type) |> Nx.flatten()
+    num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
+
+    if Nx.size(alpha) not in [0, 1, num_targets] do
+      raise ArgumentError,
+      "expected number of targets be the same as number of penalties, got: #{inspect(num_targets)} != #{inspect(Nx.size(alpha))}"
+    end
+
+    fit_n(x, y, sample_weights, alpha, opts)
+  end
+
+  defnp fit_n(a, b, sample_weights, alpha, opts) do
+    {u, s, vh} = Nx.LinAlg.svd(a, full_matrices?: false)
+    eigen_vals = Nx.pow(s, 2)
+    %__MODULE__{coefficients: Nx.tensor([1, 2, 3]), intercept: Nx.tensor([1])}
+  end
+
+  defnp update_coef(
+          x, y, n_samples, n_features, xt_y,
+          u, vh, eigen_vals,
+          alpha, lambda) do
+    regularization = vh / (eigen_vals + lambda / alpha)
+    reg_transpose = Nx.dot(regularization, xt_y)
+    coef = Nx.dot(Nx.transpose(vh), reg_transpose)
+
+    error = y - Nx.dot(x, coef)
+    squared_error = error ** 2
+    rmse = Nx.sum(squared_error)
+
+    {coef, rmse}
+  end
+end
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
new file mode 100644
index 00000000..e565f55a
--- /dev/null
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -0,0 +1,30 @@
+defmodule Scholar.Linear.BayesianRidgeRegressionTest do
+  use Scholar.Case, async: true
+  alias Scholar.Linear.BayesianRidgeRegression
+  doctest BayesianRidgeRegression
+
+  test "toy bayesian ridge" do
+    x = Nx.tensor([[1], [2], [6], [8], [10]])
+    {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
+    eigen_vals = Nx.pow(s, 2)
+    y = Nx.tensor([1, 2, 6, 8, 10])
+    alpha = 1
+    lambda = 1
+    xt_y = Nx.dot(Nx.transpose(x), y)
+    IO.inspect(xt_y)
+    regularization = # vh / (eigen_vals + lambda / alpha)
+      Nx.divide(vh, Nx.divide(Nx.add(eigen_vals, lambda), alpha))
+    IO.inspect(regularization)
+    reg_transpose = Nx.dot(regularization, xt_y)
+    coef = Nx.dot(Nx.transpose(vh), reg_transpose)
+    IO.inspect(coef)
+    
+    
+    clf = BayesianRidgeRegression.fit(x, y)
+    test = Nx.tensor([[1], [3], [4]])
+    expected_predict = Nx.tensor([1, 3, 4])
+
+    assert false
+  end
+  
+end

From 895dd9751000dae83e5caf07becfa816992f40f2 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sat, 23 Mar 2024 12:08:59 +0100
Subject: [PATCH 21/90] bayesian ridge algorithm works for simplest case

---
 .../linear/bayesian_ridge_regression.ex       | 178 ++++++++++++++++--
 .../linear/bayesian_ridge_regression_test.exs |  21 +--
 2 files changed, 170 insertions(+), 29 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 98810be2..e287a0c2 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -3,8 +3,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   import Nx.Defn
   import Scholar.Shared
 
-  @derive {Nx.Container, containers: [:coefficients, :intercept]}
-  defstruct [:coefficients, :intercept]
+  @derive {Nx.Container, containers: [:coefficients]}
+  defstruct [:coefficients]
   opts = [
     sample_weights: [
       type:
@@ -57,7 +57,92 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
       In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
       """
-    ]
+    ],
+    alpha_1: [
+      type:
+        {:or,
+         [
+           {:custom, Scholar.Options, :non_negative_number, []},
+           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
+           {:custom, Scholar.Options, :weights, []}
+         ]},
+      default: 1.0e-6,
+      doc: ~S"""
+      Constant that multiplies the $L_2$ term, controlling regularization strength.
+      `:alpha` must be a non-negative float i.e. in [0, inf).
+
+      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
+      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      """
+    ],
+    alpha_2: [
+      type:
+        {:or,
+         [
+           {:custom, Scholar.Options, :non_negative_number, []},
+           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
+           {:custom, Scholar.Options, :weights, []}
+         ]},
+      default: 1.0e-6,
+      doc: ~S"""
+      Constant that multiplies the $L_2$ term, controlling regularization strength.
+      `:alpha` must be a non-negative float i.e. in [0, inf).
+
+      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
+      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      """
+    ],        
+    lambda: [
+      type:
+        {:or,
+         [
+           {:custom, Scholar.Options, :non_negative_number, []},
+           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
+           {:custom, Scholar.Options, :weights, []}
+         ]},
+      default: 1.0,
+      doc: ~S"""
+      Constant that multiplies the $L_2$ term, controlling regularization strength.
+      `:alpha` must be a non-negative float i.e. in [0, inf).
+
+      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
+      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      """
+    ],
+    lambda_1: [
+      type:
+        {:or,
+         [
+           {:custom, Scholar.Options, :non_negative_number, []},
+           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
+           {:custom, Scholar.Options, :weights, []}
+         ]},
+      default: 1.0e-6,
+      doc: ~S"""
+      Constant that multiplies the $L_2$ term, controlling regularization strength.
+      `:alpha` must be a non-negative float i.e. in [0, inf).
+
+      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
+      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      """
+    ],
+    lambda_2: [
+      type:
+        {:or,
+         [
+           {:custom, Scholar.Options, :non_negative_number, []},
+           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
+           {:custom, Scholar.Options, :weights, []}
+         ]},
+      default: 1.0e-6,
+      doc: ~S"""
+      Constant that multiplies the $L_2$ term, controlling regularization strength.
+      `:alpha` must be a non-negative float i.e. in [0, inf).
+
+      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
+      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      """
+    ],        
   ]
   @opts_schema NimbleOptions.new!(opts)
   deftransform fit(x, y, opts \\ []) do
@@ -70,29 +155,91 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         do: Nx.as_type(sample_weights, x_type),
         else: Nx.tensor(sample_weights, type: x_type)
 
+    ## TODO: refactor this
     {alpha, opts} = Keyword.pop!(opts, :alpha)
     alpha = Nx.tensor(alpha, type: x_type) |> Nx.flatten()
+    {alpha_1, opts} = Keyword.pop!(opts, :alpha_1)
+    alpha_1 = Nx.tensor(alpha_1, type: x_type) |> Nx.flatten()
+    {alpha_2, opts} = Keyword.pop!(opts, :alpha_2)
+    alpha_2 = Nx.tensor(alpha_2, type: x_type) |> Nx.flatten()    
+    {lambda, opts} = Keyword.pop!(opts, :lambda)
+    lambda = Nx.tensor(lambda, type: x_type) |> Nx.flatten()
+    {lambda_1, opts} = Keyword.pop!(opts, :lambda_1)
+    lambda_1 = Nx.tensor(lambda_1, type: x_type) |> Nx.flatten()    
+    {lambda_2, opts} = Keyword.pop!(opts, :lambda_2)
+    lambda_2 = Nx.tensor(lambda_2, type: x_type) |> Nx.flatten()    
+    
     num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
-
     if Nx.size(alpha) not in [0, 1, num_targets] do
       raise ArgumentError,
       "expected number of targets be the same as number of penalties, got: #{inspect(num_targets)} != #{inspect(Nx.size(alpha))}"
     end
+    {coefficients, _rmse,
+     convergence, iterations} = fit_n(x, y, sample_weights,
+                                      alpha, lambda,
+                                      alpha_1, alpha_2, lambda_1, lambda_2)
+    %__MODULE__{coefficients: coefficients}
+  end
+
+  deftransformp fit_n(x, y, sample_weights,
+                      alpha, lambda,
+                      alpha_1, alpha_2, lambda_1, lambda_2) do
+    xt_y = Nx.dot(Nx.transpose(x), y)
+    {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
+    eigenvals = Nx.pow(s, 2)
+    {n_samples, n_features} = Nx.shape(x)
+    initial_coefficients = Nx.tensor(for _n <- 0..(n_features - 1), do: 0)
+    update_coef_iterations(
+      x, y, n_samples, n_features,
+      xt_y, u, vh, eigenvals,
+      alpha, lambda,
+      alpha_1, alpha_2, lambda_1, lambda_2,      
+      300, initial_coefficients)
+  end
 
-    fit_n(x, y, sample_weights, alpha, opts)
+  deftransformp update_coef_iterations(
+          x, y, n_samples, n_features,
+          xt_y, u, vh, eigenvals,
+          alpha, lambda,
+          alpha_1, alpha_2, lambda_1, lambda_2,
+          iterations, coef_old) do
+    {coefficients, rmse} = update_coef(
+                              x, y, n_samples, n_features,
+                              xt_y, u, vh, eigenvals,
+                              alpha, lambda)
+    {gamma, lambda, alpha} = update_hyperparameters(
+                               n_samples, coefficients, rmse,
+                               eigenvals, alpha, lambda,
+                               alpha_1, alpha_2, lambda_1, lambda_2)
+    convergence = check_convergence(
+                    coef_old, coefficients, Nx.tensor(1.0e-3))
+    |> Nx.to_number()
+    
+    if ((convergence == 1) or (iterations <= 0)) do
+      {coefficients, rmse, convergence == 1, iterations}
+    else
+      update_coef_iterations(
+        x, y, n_samples, n_features,
+        xt_y, u, vh, eigenvals,
+        alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2,
+        iterations - 1, coefficients)
+    end
   end
 
-  defnp fit_n(a, b, sample_weights, alpha, opts) do
-    {u, s, vh} = Nx.LinAlg.svd(a, full_matrices?: false)
-    eigen_vals = Nx.pow(s, 2)
-    %__MODULE__{coefficients: Nx.tensor([1, 2, 3]), intercept: Nx.tensor([1])}
+  defnp update_hyperparameters(n_samples, coefficients, rmse,
+                               eigenvals, alpha, lambda,
+                               alpha_1, alpha_2, lambda_1, lambda_2) do
+    gamma = Nx.sum((alpha * eigenvals) / (lambda + alpha * eigenvals))
+    lambda = (gamma + 2 * lambda_1) / (Nx.sum(coefficients ** 2) + 2 * lambda_2)
+    alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
+    {gamma, lambda, alpha}
   end
 
   defnp update_coef(
           x, y, n_samples, n_features, xt_y,
-          u, vh, eigen_vals,
+          u, vh, eigenvals,
           alpha, lambda) do
-    regularization = vh / (eigen_vals + lambda / alpha)
+    regularization = vh / (eigenvals + lambda / alpha)
     reg_transpose = Nx.dot(regularization, xt_y)
     coef = Nx.dot(Nx.transpose(vh), reg_transpose)
 
@@ -102,4 +249,13 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
 
     {coef, rmse}
   end
+
+  defnp check_convergence(coef_old, coef_new, tol) do
+    Nx.less(Nx.sum(Nx.abs(coef_old - coef_new)), tol)
+  end
+  
+  defn predict(%__MODULE__{coefficients: coeff} = _model, x) do
+    Nx.dot(x, coeff)
+  end
+  
 end
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index e565f55a..8e6ffbee 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -5,26 +5,11 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
 
   test "toy bayesian ridge" do
     x = Nx.tensor([[1], [2], [6], [8], [10]])
-    {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
-    eigen_vals = Nx.pow(s, 2)
     y = Nx.tensor([1, 2, 6, 8, 10])
-    alpha = 1
-    lambda = 1
-    xt_y = Nx.dot(Nx.transpose(x), y)
-    IO.inspect(xt_y)
-    regularization = # vh / (eigen_vals + lambda / alpha)
-      Nx.divide(vh, Nx.divide(Nx.add(eigen_vals, lambda), alpha))
-    IO.inspect(regularization)
-    reg_transpose = Nx.dot(regularization, xt_y)
-    coef = Nx.dot(Nx.transpose(vh), reg_transpose)
-    IO.inspect(coef)
-    
-    
     clf = BayesianRidgeRegression.fit(x, y)
     test = Nx.tensor([[1], [3], [4]])
-    expected_predict = Nx.tensor([1, 3, 4])
-
-    assert false
+    expected = Nx.tensor([1, 3, 4])
+    predicted = BayesianRidgeRegression.predict(clf, test)
+    assert_all_close(expected, predicted)
   end
-  
 end

From 1989c3f46a7f82d4d3ffce9cb21ba06f4375ca24 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sat, 23 Mar 2024 12:13:11 +0100
Subject: [PATCH 22/90] run formatter

---
 .../linear/bayesian_ridge_regression.ex       | 183 +++++++++++++-----
 1 file changed, 131 insertions(+), 52 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index e287a0c2..95f458e9 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -5,6 +5,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
 
   @derive {Nx.Container, containers: [:coefficients]}
   defstruct [:coefficients]
+
   opts = [
     sample_weights: [
       type:
@@ -91,7 +92,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
       In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
       """
-    ],        
+    ],
     lambda: [
       type:
         {:or,
@@ -142,12 +143,13 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
       In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
       """
-    ],        
+    ]
   ]
+
   @opts_schema NimbleOptions.new!(opts)
   deftransform fit(x, y, opts \\ []) do
-    opts = NimbleOptions.validate!(opts, @opts_schema)    
-        {sample_weights, opts} = Keyword.pop(opts, :sample_weights, 1.0)
+    opts = NimbleOptions.validate!(opts, @opts_schema)
+    {sample_weights, opts} = Keyword.pop(opts, :sample_weights, 1.0)
     x_type = to_float_type(x)
 
     sample_weights =
@@ -161,84 +163,162 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {alpha_1, opts} = Keyword.pop!(opts, :alpha_1)
     alpha_1 = Nx.tensor(alpha_1, type: x_type) |> Nx.flatten()
     {alpha_2, opts} = Keyword.pop!(opts, :alpha_2)
-    alpha_2 = Nx.tensor(alpha_2, type: x_type) |> Nx.flatten()    
+    alpha_2 = Nx.tensor(alpha_2, type: x_type) |> Nx.flatten()
     {lambda, opts} = Keyword.pop!(opts, :lambda)
     lambda = Nx.tensor(lambda, type: x_type) |> Nx.flatten()
     {lambda_1, opts} = Keyword.pop!(opts, :lambda_1)
-    lambda_1 = Nx.tensor(lambda_1, type: x_type) |> Nx.flatten()    
+    lambda_1 = Nx.tensor(lambda_1, type: x_type) |> Nx.flatten()
     {lambda_2, opts} = Keyword.pop!(opts, :lambda_2)
-    lambda_2 = Nx.tensor(lambda_2, type: x_type) |> Nx.flatten()    
-    
+    lambda_2 = Nx.tensor(lambda_2, type: x_type) |> Nx.flatten()
+
     num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
+
     if Nx.size(alpha) not in [0, 1, num_targets] do
       raise ArgumentError,
-      "expected number of targets be the same as number of penalties, got: #{inspect(num_targets)} != #{inspect(Nx.size(alpha))}"
+            "expected number of targets be the same as number of penalties, got: #{inspect(num_targets)} != #{inspect(Nx.size(alpha))}"
     end
-    {coefficients, _rmse,
-     convergence, iterations} = fit_n(x, y, sample_weights,
-                                      alpha, lambda,
-                                      alpha_1, alpha_2, lambda_1, lambda_2)
+
+    {coefficients, _rmse, convergence, iterations} =
+      fit_n(x, y, sample_weights, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2)
+
     %__MODULE__{coefficients: coefficients}
   end
 
-  deftransformp fit_n(x, y, sample_weights,
-                      alpha, lambda,
-                      alpha_1, alpha_2, lambda_1, lambda_2) do
+  deftransformp fit_n(x, y, sample_weights, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2) do
     xt_y = Nx.dot(Nx.transpose(x), y)
     {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
     eigenvals = Nx.pow(s, 2)
     {n_samples, n_features} = Nx.shape(x)
     initial_coefficients = Nx.tensor(for _n <- 0..(n_features - 1), do: 0)
+
     update_coef_iterations(
-      x, y, n_samples, n_features,
-      xt_y, u, vh, eigenvals,
-      alpha, lambda,
-      alpha_1, alpha_2, lambda_1, lambda_2,      
-      300, initial_coefficients)
+      x,
+      y,
+      n_samples,
+      n_features,
+      xt_y,
+      u,
+      vh,
+      eigenvals,
+      alpha,
+      lambda,
+      alpha_1,
+      alpha_2,
+      lambda_1,
+      lambda_2,
+      300,
+      initial_coefficients
+    )
   end
 
   deftransformp update_coef_iterations(
-          x, y, n_samples, n_features,
-          xt_y, u, vh, eigenvals,
-          alpha, lambda,
-          alpha_1, alpha_2, lambda_1, lambda_2,
-          iterations, coef_old) do
-    {coefficients, rmse} = update_coef(
-                              x, y, n_samples, n_features,
-                              xt_y, u, vh, eigenvals,
-                              alpha, lambda)
-    {gamma, lambda, alpha} = update_hyperparameters(
-                               n_samples, coefficients, rmse,
-                               eigenvals, alpha, lambda,
-                               alpha_1, alpha_2, lambda_1, lambda_2)
-    convergence = check_convergence(
-                    coef_old, coefficients, Nx.tensor(1.0e-3))
-    |> Nx.to_number()
-    
-    if ((convergence == 1) or (iterations <= 0)) do
+                  x,
+                  y,
+                  n_samples,
+                  n_features,
+                  xt_y,
+                  u,
+                  vh,
+                  eigenvals,
+                  alpha,
+                  lambda,
+                  alpha_1,
+                  alpha_2,
+                  lambda_1,
+                  lambda_2,
+                  iterations,
+                  coef_old
+                ) do
+    {coefficients, rmse} =
+      update_coef(
+        x,
+        y,
+        n_samples,
+        n_features,
+        xt_y,
+        u,
+        vh,
+        eigenvals,
+        alpha,
+        lambda
+      )
+
+    {gamma, lambda, alpha} =
+      update_hyperparameters(
+        n_samples,
+        coefficients,
+        rmse,
+        eigenvals,
+        alpha,
+        lambda,
+        alpha_1,
+        alpha_2,
+        lambda_1,
+        lambda_2
+      )
+
+    convergence =
+      check_convergence(
+        coef_old,
+        coefficients,
+        Nx.tensor(1.0e-3)
+      )
+      |> Nx.to_number()
+
+    if convergence == 1 or iterations <= 0 do
       {coefficients, rmse, convergence == 1, iterations}
     else
       update_coef_iterations(
-        x, y, n_samples, n_features,
-        xt_y, u, vh, eigenvals,
-        alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2,
-        iterations - 1, coefficients)
+        x,
+        y,
+        n_samples,
+        n_features,
+        xt_y,
+        u,
+        vh,
+        eigenvals,
+        alpha,
+        lambda,
+        alpha_1,
+        alpha_2,
+        lambda_1,
+        lambda_2,
+        iterations - 1,
+        coefficients
+      )
     end
   end
 
-  defnp update_hyperparameters(n_samples, coefficients, rmse,
-                               eigenvals, alpha, lambda,
-                               alpha_1, alpha_2, lambda_1, lambda_2) do
-    gamma = Nx.sum((alpha * eigenvals) / (lambda + alpha * eigenvals))
+  defnp update_hyperparameters(
+          n_samples,
+          coefficients,
+          rmse,
+          eigenvals,
+          alpha,
+          lambda,
+          alpha_1,
+          alpha_2,
+          lambda_1,
+          lambda_2
+        ) do
+    gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
     lambda = (gamma + 2 * lambda_1) / (Nx.sum(coefficients ** 2) + 2 * lambda_2)
     alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
     {gamma, lambda, alpha}
   end
 
   defnp update_coef(
-          x, y, n_samples, n_features, xt_y,
-          u, vh, eigenvals,
-          alpha, lambda) do
+          x,
+          y,
+          n_samples,
+          n_features,
+          xt_y,
+          u,
+          vh,
+          eigenvals,
+          alpha,
+          lambda
+        ) do
     regularization = vh / (eigenvals + lambda / alpha)
     reg_transpose = Nx.dot(regularization, xt_y)
     coef = Nx.dot(Nx.transpose(vh), reg_transpose)
@@ -253,9 +333,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   defnp check_convergence(coef_old, coef_new, tol) do
     Nx.less(Nx.sum(Nx.abs(coef_old - coef_new)), tol)
   end
-  
+
   defn predict(%__MODULE__{coefficients: coeff} = _model, x) do
     Nx.dot(x, coeff)
   end
-  
 end

From f5ba5b6b020dcdb2f8284fe61ae984781e8685c9 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Mon, 25 Mar 2024 23:33:24 +0100
Subject: [PATCH 23/90] refactor recursion to use while loop

---
 .../linear/bayesian_ridge_regression.ex       | 160 +++++-------------
 .../linear/bayesian_ridge_regression_test.exs |   2 +-
 2 files changed, 40 insertions(+), 122 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 95f458e9..6fd47301 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -178,140 +178,58 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
             "expected number of targets be the same as number of penalties, got: #{inspect(num_targets)} != #{inspect(Nx.size(alpha))}"
     end
 
-    {coefficients, _rmse, convergence, iterations} =
-      fit_n(x, y, sample_weights, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2)
-
+    IO.puts("begin fitting")
+    {{coefficients, _rmse, iterations, has_converged}, _} =
+      fit_n(x, y, sample_weights, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2, 300)
+    IO.puts("fitting ended!!")
+    IO.inspect(coefficients)
+    IO.inspect(iterations)
+    IO.inspect(has_converged)
     %__MODULE__{coefficients: coefficients}
   end
 
-  deftransformp fit_n(x, y, sample_weights, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2) do
+  defnp fit_n(x, y, sample_weights, alpha, lambda,
+              alpha_1, alpha_2, lambda_1, lambda_2, iterations) do
+    print_value(Nx.u8(0))
     xt_y = Nx.dot(Nx.transpose(x), y)
     {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
     eigenvals = Nx.pow(s, 2)
     {n_samples, n_features} = Nx.shape(x)
-    initial_coefficients = Nx.tensor(for _n <- 0..(n_features - 1), do: 0)
-
-    update_coef_iterations(
-      x,
-      y,
-      n_samples,
-      n_features,
-      xt_y,
-      u,
-      vh,
-      eigenvals,
-      alpha,
-      lambda,
-      alpha_1,
-      alpha_2,
-      lambda_1,
-      lambda_2,
-      300,
-      initial_coefficients
-    )
-  end
-
-  deftransformp update_coef_iterations(
-                  x,
-                  y,
-                  n_samples,
-                  n_features,
-                  xt_y,
-                  u,
-                  vh,
-                  eigenvals,
-                  alpha,
-                  lambda,
-                  alpha_1,
-                  alpha_2,
-                  lambda_1,
-                  lambda_2,
-                  iterations,
-                  coef_old
-                ) do
-    {coefficients, rmse} =
-      update_coef(
-        x,
-        y,
-        n_samples,
-        n_features,
-        xt_y,
-        u,
-        vh,
-        eigenvals,
-        alpha,
-        lambda
-      )
-
-    {gamma, lambda, alpha} =
-      update_hyperparameters(
-        n_samples,
-        coefficients,
-        rmse,
-        eigenvals,
-        alpha,
-        lambda,
-        alpha_1,
-        alpha_2,
-        lambda_1,
-        lambda_2
-      )
-
-    convergence =
-      check_convergence(
-        coef_old,
-        coefficients,
-        Nx.tensor(1.0e-3)
-      )
-      |> Nx.to_number()
+    {coef, rmse} = update_coef(x, y, n_samples, n_features,
+                               xt_y, u, vh, eigenvals,
+                               alpha, lambda)
 
-    if convergence == 1 or iterations <= 0 do
-      {coefficients, rmse, convergence == 1, iterations}
-    else
-      update_coef_iterations(
-        x,
-        y,
-        n_samples,
-        n_features,
-        xt_y,
-        u,
-        vh,
-        eigenvals,
-        alpha,
-        lambda,
-        alpha_1,
-        alpha_2,
-        lambda_1,
-        lambda_2,
-        iterations - 1,
-        coefficients
-      )
-    end
-  end
-
-  defnp update_hyperparameters(
-          n_samples,
-          coefficients,
-          rmse,
-          eigenvals,
-          alpha,
-          lambda,
-          alpha_1,
-          alpha_2,
-          lambda_1,
-          lambda_2
-        ) do
-    gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
-    lambda = (gamma + 2 * lambda_1) / (Nx.sum(coefficients ** 2) + 2 * lambda_2)
-    alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
-    {gamma, lambda, alpha}
+    {{final_coef, rmse, iter, has_converged}, _} =
+      while {{coef, rmse, iter = 1, has_converged = Nx.u8(0)},
+             {x, y,
+              xt_y, u, s, vh, eigenvals,
+              alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2,
+              iterations}},
+            iter < iterations and not has_converged do
+        {coef_new, rmse} = update_coef(
+                             x, y, n_samples, n_features,
+                             xt_y, u, vh, eigenvals,
+                             alpha, lambda)
+        
+        gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
+        lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef_new ** 2) + 2 * lambda_2)
+        alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
+        
+        has_converged = check_convergence(coef_new, coef, 1.0e-10)
+        
+        {{coef_new, rmse, iter + 1, has_converged},
+         {x, y,
+          xt_y, u, s, vh, eigenvals,
+          alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2,
+          iterations}}
+      end
   end
 
   defnp update_coef(
           x,
           y,
-          n_samples,
-          n_features,
+          _n_samples,
+          _n_features,
           xt_y,
           u,
           vh,
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 8e6ffbee..df5f1ddf 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -10,6 +10,6 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     test = Nx.tensor([[1], [3], [4]])
     expected = Nx.tensor([1, 3, 4])
     predicted = BayesianRidgeRegression.predict(clf, test)
-    assert_all_close(expected, predicted)
+    assert_all_close(expected, predicted, atol: 1.0e-1)
   end
 end

From 115c447a5f81bc10150abafde6ea3df019d3b63c Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Mon, 25 Mar 2024 23:49:45 +0100
Subject: [PATCH 24/90] cleanup code and add convergence message

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 6fd47301..9c000a32 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -178,13 +178,11 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
             "expected number of targets be the same as number of penalties, got: #{inspect(num_targets)} != #{inspect(Nx.size(alpha))}"
     end
 
-    IO.puts("begin fitting")
     {{coefficients, _rmse, iterations, has_converged}, _} =
       fit_n(x, y, sample_weights, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2, 300)
-    IO.puts("fitting ended!!")
-    IO.inspect(coefficients)
-    IO.inspect(iterations)
-    IO.inspect(has_converged)
+    if Nx.to_number(has_converged) == 1 do
+      IO.puts("Convergence after #{Nx.to_number(iterations)} iterations")
+    end
     %__MODULE__{coefficients: coefficients}
   end
 

From 1a3070ce9be2860b69c86e598c2e0784fe7e714f Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Tue, 26 Mar 2024 18:16:06 +0100
Subject: [PATCH 25/90] add test case

---
 .../linear/bayesian_ridge_regression_test.exs        | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index df5f1ddf..aeaa5adc 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -12,4 +12,16 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     predicted = BayesianRidgeRegression.predict(clf, test)
     assert_all_close(expected, predicted, atol: 1.0e-1)
   end
+  
+  test "toy bayesian ride expanded" do
+    x = Nx.tensor([[1, 1], [2, 2], [6, 6], [8, 8], [10, 10]])
+    y = Nx.tensor([1, 2, 6, 8, 10])
+    clf = BayesianRidgeRegression.fit(x, y)
+    test = Nx.tensor([[1, 1], [3, 3], [4, 4]])
+    expected = Nx.tensor([1, 3, 4])
+    predicted = BayesianRidgeRegression.predict(clf, test)
+    IO.inspect(clf)
+    IO.inspect(predicted)
+    assert false            
+  end  
 end

From c0ebe68523477d9d36f5842eaa8056412014acaa Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Wed, 27 Mar 2024 11:58:25 +0100
Subject: [PATCH 26/90] simple case works

---
 .../linear/bayesian_ridge_regression.ex       | 89 ++++++++++---------
 .../linear/bayesian_ridge_regression_test.exs |  2 +-
 2 files changed, 48 insertions(+), 43 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 9c000a32..4b2b7313 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -177,9 +177,19 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       raise ArgumentError,
             "expected number of targets be the same as number of penalties, got: #{inspect(num_targets)} != #{inspect(Nx.size(alpha))}"
     end
+    # 
+    xt_y = Nx.dot(Nx.transpose(x), y)
+    {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
+    eigenvals = Nx.pow(s, 2)
+    {n_samples, n_features} = Nx.shape(x)
+    {coef, _rmse} = update_coef(x, y, n_samples, n_features,
+                               xt_y, u, vh, eigenvals,
+                               alpha, lambda)
+    IO.inspect(coef)
 
     {{coefficients, _rmse, iterations, has_converged}, _} =
       fit_n(x, y, sample_weights, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2, 300)
+    IO.inspect(has_converged)
     if Nx.to_number(has_converged) == 1 do
       IO.puts("Convergence after #{Nx.to_number(iterations)} iterations")
     end
@@ -188,7 +198,6 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
 
   defnp fit_n(x, y, sample_weights, alpha, lambda,
               alpha_1, alpha_2, lambda_1, lambda_2, iterations) do
-    print_value(Nx.u8(0))
     xt_y = Nx.dot(Nx.transpose(x), y)
     {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
     eigenvals = Nx.pow(s, 2)
@@ -197,45 +206,45 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
                                xt_y, u, vh, eigenvals,
                                alpha, lambda)
 
-    {{final_coef, rmse, iter, has_converged}, _} =
-      while {{coef, rmse, iter = 1, has_converged = Nx.u8(0)},
-             {x, y,
-              xt_y, u, s, vh, eigenvals,
-              alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2,
-              iterations}},
-            iter < iterations and not has_converged do
-        {coef_new, rmse} = update_coef(
-                             x, y, n_samples, n_features,
-                             xt_y, u, vh, eigenvals,
-                             alpha, lambda)
-        
-        gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
-        lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef_new ** 2) + 2 * lambda_2)
-        alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
-        
-        has_converged = check_convergence(coef_new, coef, 1.0e-10)
-        
-        {{coef_new, rmse, iter + 1, has_converged},
-         {x, y,
-          xt_y, u, s, vh, eigenvals,
-          alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2,
-          iterations}}
-      end
+    while {{coef, rmse, iter = 0, has_converged = Nx.u8(0)},
+           {x, y,
+            xt_y, u, s, vh, eigenvals,
+            alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2,
+            iterations}},
+      iter < iterations and not has_converged do
+
+      gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
+      lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
+      alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
+      
+      {coef_new, rmse} = update_coef(
+        x, y, n_samples, n_features,
+        xt_y, u, vh, eigenvals,
+        alpha, lambda)
+      
+      has_converged = Nx.sum(Nx.abs(coef - coef_new)) < 1.0e-8
+      {{coef_new, rmse, iter + 1, has_converged},
+       {x, y,
+        xt_y, u, s, vh, eigenvals,
+        alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2,
+        iterations}}
+    end
   end
 
-  defnp update_coef(
-          x,
-          y,
-          _n_samples,
-          _n_features,
-          xt_y,
-          u,
-          vh,
-          eigenvals,
-          alpha,
-          lambda
-        ) do
-    regularization = vh / (eigenvals + lambda / alpha)
+  defn update_coef(
+    x,
+    y,
+    n_samples,
+    n_features,
+    xt_y,
+    u,
+    vh,
+    eigenvals,
+    alpha,
+    lambda
+  ) do
+    scaled_eigens = eigenvals + lambda / alpha
+    regularization = vh / scaled_eigens
     reg_transpose = Nx.dot(regularization, xt_y)
     coef = Nx.dot(Nx.transpose(vh), reg_transpose)
 
@@ -246,10 +255,6 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {coef, rmse}
   end
 
-  defnp check_convergence(coef_old, coef_new, tol) do
-    Nx.less(Nx.sum(Nx.abs(coef_old - coef_new)), tol)
-  end
-
   defn predict(%__MODULE__{coefficients: coeff} = _model, x) do
     Nx.dot(x, coeff)
   end
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index aeaa5adc..4dfd2027 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -10,7 +10,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     test = Nx.tensor([[1], [3], [4]])
     expected = Nx.tensor([1, 3, 4])
     predicted = BayesianRidgeRegression.predict(clf, test)
-    assert_all_close(expected, predicted, atol: 1.0e-1)
+    assert_all_close(expected, predicted, atol: 1.0e-6)
   end
   
   test "toy bayesian ride expanded" do

From 5533ab5db55475e5a74459d8722396476ed6cb17 Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Wed, 27 Mar 2024 15:08:12 +0100
Subject: [PATCH 27/90] reshape for multi feat

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 4b2b7313..e0e3135c 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -244,7 +244,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     lambda
   ) do
     scaled_eigens = eigenvals + lambda / alpha
-    regularization = vh / scaled_eigens
+    {n_eigens} = Nx.shape(scaled_eigens)
+    regularization = vh / Nx.reshape(scaled_eigens, {n_eigens, 1})
     reg_transpose = Nx.dot(regularization, xt_y)
     coef = Nx.dot(Nx.transpose(vh), reg_transpose)
 

From df1be29d1a717bc952f09155129e8d7219d89f60 Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Wed, 27 Mar 2024 17:04:42 +0100
Subject: [PATCH 28/90] expanded test passing

---
 .../linear/bayesian_ridge_regression.ex        | 18 ++++++------------
 .../linear/bayesian_ridge_regression_test.exs  | 12 +++++++-----
 2 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index e0e3135c..5c995314 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -159,17 +159,11 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
 
     ## TODO: refactor this
     {alpha, opts} = Keyword.pop!(opts, :alpha)
-    alpha = Nx.tensor(alpha, type: x_type) |> Nx.flatten()
     {alpha_1, opts} = Keyword.pop!(opts, :alpha_1)
-    alpha_1 = Nx.tensor(alpha_1, type: x_type) |> Nx.flatten()
     {alpha_2, opts} = Keyword.pop!(opts, :alpha_2)
-    alpha_2 = Nx.tensor(alpha_2, type: x_type) |> Nx.flatten()
     {lambda, opts} = Keyword.pop!(opts, :lambda)
-    lambda = Nx.tensor(lambda, type: x_type) |> Nx.flatten()
     {lambda_1, opts} = Keyword.pop!(opts, :lambda_1)
-    lambda_1 = Nx.tensor(lambda_1, type: x_type) |> Nx.flatten()
     {lambda_2, opts} = Keyword.pop!(opts, :lambda_2)
-    lambda_2 = Nx.tensor(lambda_2, type: x_type) |> Nx.flatten()
 
     num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
 
@@ -182,14 +176,10 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
     eigenvals = Nx.pow(s, 2)
     {n_samples, n_features} = Nx.shape(x)
-    {coef, _rmse} = update_coef(x, y, n_samples, n_features,
-                               xt_y, u, vh, eigenvals,
-                               alpha, lambda)
-    IO.inspect(coef)
 
     {{coefficients, _rmse, iterations, has_converged}, _} =
       fit_n(x, y, sample_weights, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2, 300)
-    IO.inspect(has_converged)
+
     if Nx.to_number(has_converged) == 1 do
       IO.puts("Convergence after #{Nx.to_number(iterations)} iterations")
     end
@@ -213,7 +203,11 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
             iterations}},
       iter < iterations and not has_converged do
 
-      gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
+      # gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
+      gamma =
+        Nx.multiply(alpha, eigenvals)
+        |> Nx.divide(Nx.multiply(lambda + alpha, eigenvals))
+        |> Nx.sum()      
       lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
       alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
       
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 4dfd2027..df2c90c9 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -14,14 +14,16 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
   end
   
   test "toy bayesian ride expanded" do
-    x = Nx.tensor([[1, 1], [2, 2], [6, 6], [8, 8], [10, 10]])
-    y = Nx.tensor([1, 2, 6, 8, 10])
+    x = Nx.tensor([
+      [1, 5], [2, 6], [6, 6], [8, 4], [10, 0],
+      [5, 5], [6, 2], [6, 4], [4, 2], [0, 10],      
+    ])
+    true_coef = Nx.tensor([0.5, 0.5])
+    y = Nx.dot(x, true_coef)
     clf = BayesianRidgeRegression.fit(x, y)
     test = Nx.tensor([[1, 1], [3, 3], [4, 4]])
     expected = Nx.tensor([1, 3, 4])
     predicted = BayesianRidgeRegression.predict(clf, test)
-    IO.inspect(clf)
-    IO.inspect(predicted)
-    assert false            
+    assert_all_close(expected, predicted, atol: 1.0e-3)
   end  
 end

From afa417d55a0d2d63a06bb1ca37b505b7b2abbc5d Mon Sep 17 00:00:00 2001
From: iglesiastj <iglesiastj@gmail.com>
Date: Wed, 27 Mar 2024 17:10:23 +0100
Subject: [PATCH 29/90] better test name

---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index df2c90c9..fdadce21 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -13,7 +13,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(expected, predicted, atol: 1.0e-6)
   end
   
-  test "toy bayesian ride expanded" do
+  test "multi column toy bayesian ridge" do
     x = Nx.tensor([
       [1, 5], [2, 6], [6, 6], [8, 4], [10, 0],
       [5, 5], [6, 2], [6, 4], [4, 2], [0, 10],      

From a761eab8409c70ba18ead3480430edb65e6bb5a9 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 00:31:23 +0100
Subject: [PATCH 30/90] use new_axis where I should

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 5c995314..812b7d80 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -239,7 +239,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   ) do
     scaled_eigens = eigenvals + lambda / alpha
     {n_eigens} = Nx.shape(scaled_eigens)
-    regularization = vh / Nx.reshape(scaled_eigens, {n_eigens, 1})
+    regularization = vh / Nx.new_axis(scaled_eigens, -1) 
     reg_transpose = Nx.dot(regularization, xt_y)
     coef = Nx.dot(Nx.transpose(vh), reg_transpose)
 

From 9cdeecc0a45f2a70ffb97d0cb2ca7758d134f795 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 00:40:01 +0100
Subject: [PATCH 31/90] iterations and eps as options. remove solve

---
 .../linear/bayesian_ridge_regression.ex       | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 812b7d80..46fb590a 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -7,6 +7,14 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   defstruct [:coefficients]
 
   opts = [
+    iterations: [
+      type: :pos_integer,
+      default: 300,
+      doc: """
+      Maximum number of iterations before stopping the fitting algorithm.
+      The number of iterations may be lower is parameters converge.
+      """
+    ],    
     sample_weights: [
       type:
         {:or,
@@ -30,18 +38,6 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       of targets for a zero-vector on input.
       """
     ],
-    solver: [
-      type: {:in, [:svd, :cholesky]},
-      default: :svd,
-      doc: """
-      Solver to use in the computational routines:
-
-      * `:svd` - Uses a Singular Value Decomposition of A to compute the Ridge coefficients.
-      In particular, it is more stable for singular matrices than `:cholesky` at the cost of being slower.
-
-      * `:cholesky` - Uses the standard `Nx.LinAlg.solve` function to obtain a closed-form solution.
-      """
-    ],
     alpha: [
       type:
         {:or,
@@ -143,7 +139,14 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
       In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
       """
-    ]
+    ],
+    eps: [
+      type: :float,
+      default: 1.0e-8,
+      doc:
+        "The convergence tolerance. When `Nx.sum(Nx.abs(coef - coef_new)) < :eps`, the algorithm is considered to have converged."
+    ]    
+    
   ]
 
   @opts_schema NimbleOptions.new!(opts)

From d5f25109f58e5ee5197be44d6c2096929945aef5 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 00:54:01 +0100
Subject: [PATCH 32/90] Options documented and renamed

---
 .../linear/bayesian_ridge_regression.ex       | 102 +++++-------------
 1 file changed, 27 insertions(+), 75 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 46fb590a..5e0d1ba2 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -38,108 +38,60 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       of targets for a zero-vector on input.
       """
     ],
-    alpha: [
+    alpha_init: [
       type:
-        {:or,
-         [
-           {:custom, Scholar.Options, :non_negative_number, []},
-           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
-           {:custom, Scholar.Options, :weights, []}
-         ]},
+        {:custom, Scholar.Options, :non_negative_number, []},
       default: 1.0,
       doc: ~S"""
-      Constant that multiplies the $L_2$ term, controlling regularization strength.
+      The initial value for alpha. This parameter influences the precision of the noise.
       `:alpha` must be a non-negative float i.e. in [0, inf).
-
-      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
-      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
       """
     ],
-    alpha_1: [
+    lambda_init: [
       type:
-        {:or,
-         [
-           {:custom, Scholar.Options, :non_negative_number, []},
-           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
-           {:custom, Scholar.Options, :weights, []}
-         ]},
-      default: 1.0e-6,
+        {:custom, Scholar.Options, :non_negative_number, []},
+      default: 1.0,
       doc: ~S"""
-      Constant that multiplies the $L_2$ term, controlling regularization strength.
-      `:alpha` must be a non-negative float i.e. in [0, inf).
-
-      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
-      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      The initial value for lambda. This parameter influences the precision of the weights.
+      `:lambda` must be a non-negative float i.e. in [0, inf).      
       """
     ],
-    alpha_2: [
+    alpha_1: [
       type:
-        {:or,
-         [
-           {:custom, Scholar.Options, :non_negative_number, []},
-           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
-           {:custom, Scholar.Options, :weights, []}
-         ]},
+        {:custom, Scholar.Options, :non_negative_number, []},
       default: 1.0e-6,
       doc: ~S"""
-      Constant that multiplies the $L_2$ term, controlling regularization strength.
-      `:alpha` must be a non-negative float i.e. in [0, inf).
-
-      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
-      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      Hyper-parameter : shape parameter for the Gamma distribution prior      
+      over the alpha parameter.
       """
-    ],
-    lambda: [
+    ],    
+    alpha_2: [
       type:
-        {:or,
-         [
-           {:custom, Scholar.Options, :non_negative_number, []},
-           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
-           {:custom, Scholar.Options, :weights, []}
-         ]},
-      default: 1.0,
+        {:custom, Scholar.Options, :non_negative_number, []},
+      default: 1.0e-6,
       doc: ~S"""
-      Constant that multiplies the $L_2$ term, controlling regularization strength.
-      `:alpha` must be a non-negative float i.e. in [0, inf).
-
-      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
-      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      Hyper-parameter : inverse scale (rate) parameter for the Gamma distribution prior
+      over the alpha parameter.
       """
     ],
     lambda_1: [
       type:
-        {:or,
-         [
-           {:custom, Scholar.Options, :non_negative_number, []},
-           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
-           {:custom, Scholar.Options, :weights, []}
-         ]},
+        {:custom, Scholar.Options, :non_negative_number, []},
       default: 1.0e-6,
       doc: ~S"""
-      Constant that multiplies the $L_2$ term, controlling regularization strength.
-      `:alpha` must be a non-negative float i.e. in [0, inf).
-
-      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
-      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      Hyper-parameter : shape parameter for the Gamma distribution prior
+      over the lambda parameter.            
       """
     ],
     lambda_2: [
       type:
-        {:or,
-         [
-           {:custom, Scholar.Options, :non_negative_number, []},
-           {:list, {:custom, Scholar.Options, :non_negative_number, []}},
-           {:custom, Scholar.Options, :weights, []}
-         ]},
+        {:custom, Scholar.Options, :non_negative_number, []},
       default: 1.0e-6,
       doc: ~S"""
-      Constant that multiplies the $L_2$ term, controlling regularization strength.
-      `:alpha` must be a non-negative float i.e. in [0, inf).
-
-      If `:alpha` is set to 0.0 the objective is the ordinary least squares regression.
-      In this case, for numerical reasons, you should use `Scholar.Linear.LinearRegression` instead.
+      Hyper-parameter : inverse scale (rate) parameter for the Gamma distribution prior      
+      over the lambda parameter.            
       """
-    ],
+    ],    
     eps: [
       type: :float,
       default: 1.0e-8,
@@ -161,10 +113,10 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         else: Nx.tensor(sample_weights, type: x_type)
 
     ## TODO: refactor this
-    {alpha, opts} = Keyword.pop!(opts, :alpha)
+    {alpha, opts} = Keyword.pop!(opts, :alpha_init)
     {alpha_1, opts} = Keyword.pop!(opts, :alpha_1)
     {alpha_2, opts} = Keyword.pop!(opts, :alpha_2)
-    {lambda, opts} = Keyword.pop!(opts, :lambda)
+    {lambda, opts} = Keyword.pop!(opts, :lambda_init)
     {lambda_1, opts} = Keyword.pop!(opts, :lambda_1)
     {lambda_2, opts} = Keyword.pop!(opts, :lambda_2)
 

From 0f054c67ff6d70103d083548a10cf7b31f63cbbd Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 01:19:25 +0100
Subject: [PATCH 33/90] refactor options

---
 .../linear/bayesian_ridge_regression.ex       | 36 +++++++++----------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 5e0d1ba2..b2cb4f03 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -112,28 +112,15 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         do: Nx.as_type(sample_weights, x_type),
         else: Nx.tensor(sample_weights, type: x_type)
 
-    ## TODO: refactor this
-    {alpha, opts} = Keyword.pop!(opts, :alpha_init)
-    {alpha_1, opts} = Keyword.pop!(opts, :alpha_1)
-    {alpha_2, opts} = Keyword.pop!(opts, :alpha_2)
-    {lambda, opts} = Keyword.pop!(opts, :lambda_init)
-    {lambda_1, opts} = Keyword.pop!(opts, :lambda_1)
-    {lambda_2, opts} = Keyword.pop!(opts, :lambda_2)
+    IO.inspect(opts)
+    lambda = Keyword.get(opts, :lambda_init, 1 / Nx.variance(y))
+    opts = Keyword.put(opts, :lambda_init, lambda)
+    IO.inspect(opts)
 
     num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
 
-    if Nx.size(alpha) not in [0, 1, num_targets] do
-      raise ArgumentError,
-            "expected number of targets be the same as number of penalties, got: #{inspect(num_targets)} != #{inspect(Nx.size(alpha))}"
-    end
-    # 
-    xt_y = Nx.dot(Nx.transpose(x), y)
-    {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
-    eigenvals = Nx.pow(s, 2)
-    {n_samples, n_features} = Nx.shape(x)
-
     {{coefficients, _rmse, iterations, has_converged}, _} =
-      fit_n(x, y, sample_weights, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2, 300)
+      fit_n(x, y, sample_weights, opts)
 
     if Nx.to_number(has_converged) == 1 do
       IO.puts("Convergence after #{Nx.to_number(iterations)} iterations")
@@ -141,8 +128,17 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     %__MODULE__{coefficients: coefficients}
   end
 
-  defnp fit_n(x, y, sample_weights, alpha, lambda,
-              alpha_1, alpha_2, lambda_1, lambda_2, iterations) do
+  defnp fit_n(x, y, sample_weights, opts) do
+    alpha = opts[:alpha_init]
+    alpha_1 = opts[:alpha_1]
+    alpha_2 = opts[:alpha_2]
+    lambda = opts[:lambda_init]
+    lambda = opts[:lambda_init]
+               
+    lambda_1 = opts[:lambda_1]
+    lambda_2 = opts[:lambda_2]
+    iterations = opts[:iterations]    
+    
     xt_y = Nx.dot(Nx.transpose(x), y)
     {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
     eigenvals = Nx.pow(s, 2)

From 7cc4683f1da076a153f3a4b85cba8a57e9647971 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 01:25:20 +0100
Subject: [PATCH 34/90] fix alpha default option

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index b2cb4f03..7b913e6e 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -41,10 +41,10 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     alpha_init: [
       type:
         {:custom, Scholar.Options, :non_negative_number, []},
-      default: 1.0,
       doc: ~S"""
       The initial value for alpha. This parameter influences the precision of the noise.
       `:alpha` must be a non-negative float i.e. in [0, inf).
+      Defaults to 1/Var(y).      
       """
     ],
     lambda_init: [
@@ -53,7 +53,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       default: 1.0,
       doc: ~S"""
       The initial value for lambda. This parameter influences the precision of the weights.
-      `:lambda` must be a non-negative float i.e. in [0, inf).      
+      `:lambda` must be a non-negative float i.e. in [0, inf).
+      Defaults to 1.
       """
     ],
     alpha_1: [
@@ -112,10 +113,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         do: Nx.as_type(sample_weights, x_type),
         else: Nx.tensor(sample_weights, type: x_type)
 
-    IO.inspect(opts)
-    lambda = Keyword.get(opts, :lambda_init, 1 / Nx.variance(y))
-    opts = Keyword.put(opts, :lambda_init, lambda)
-    IO.inspect(opts)
+    alpha = Keyword.get(opts, :alpha_init, Nx.divide(1, Nx.variance(y)))
+    opts = Keyword.put(opts, :alpha_init, alpha)
 
     num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
 
@@ -132,11 +131,11 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     alpha = opts[:alpha_init]
     alpha_1 = opts[:alpha_1]
     alpha_2 = opts[:alpha_2]
+
     lambda = opts[:lambda_init]
-    lambda = opts[:lambda_init]
-               
     lambda_1 = opts[:lambda_1]
     lambda_2 = opts[:lambda_2]
+
     iterations = opts[:iterations]    
     
     xt_y = Nx.dot(Nx.transpose(x), y)

From 7d8d470e05ece055674d451899cd2b6f0579b0d0 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 01:31:41 +0100
Subject: [PATCH 35/90] model contains all fitted parameters

---
 .../linear/bayesian_ridge_regression.ex       | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 7b913e6e..38db6566 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -3,8 +3,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   import Nx.Defn
   import Scholar.Shared
 
-  @derive {Nx.Container, containers: [:coefficients]}
-  defstruct [:coefficients]
+  @derive {Nx.Container, containers: [:coefficients, :alpha, :lambda, :rmse, :iterations]}
+  defstruct [:coefficients, :alpha, :lambda, :rmse, :iterations]
 
   opts = [
     iterations: [
@@ -118,13 +118,16 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
 
     num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
 
-    {{coefficients, _rmse, iterations, has_converged}, _} =
+    {{coefficients, rmse, iterations, has_converged, alpha, lambda}, _} =
       fit_n(x, y, sample_weights, opts)
 
     if Nx.to_number(has_converged) == 1 do
       IO.puts("Convergence after #{Nx.to_number(iterations)} iterations")
     end
-    %__MODULE__{coefficients: coefficients}
+    %__MODULE__{
+      coefficients: coefficients,
+      alpha: alpha, lambda: lambda,
+      rmse: rmse, iterations: iterations}
   end
 
   defnp fit_n(x, y, sample_weights, opts) do
@@ -146,10 +149,10 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
                                xt_y, u, vh, eigenvals,
                                alpha, lambda)
 
-    while {{coef, rmse, iter = 0, has_converged = Nx.u8(0)},
+    while {{coef, rmse, iter = 0, has_converged = Nx.u8(0), alpha, lambda,},
            {x, y,
             xt_y, u, s, vh, eigenvals,
-            alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2,
+            alpha_1, alpha_2, lambda_1, lambda_2,
             iterations}},
       iter < iterations and not has_converged do
 
@@ -167,10 +170,10 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         alpha, lambda)
       
       has_converged = Nx.sum(Nx.abs(coef - coef_new)) < 1.0e-8
-      {{coef_new, rmse, iter + 1, has_converged},
+      {{coef_new, rmse, iter + 1, has_converged, alpha, lambda,},
        {x, y,
         xt_y, u, s, vh, eigenvals,
-        alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2,
+        alpha_1, alpha_2, lambda_1, lambda_2,
         iterations}}
     end
   end

From e1e62d6d0c0f4e9615b8e4a11578480533efd42e Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 01:34:12 +0100
Subject: [PATCH 36/90] add next test

---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index fdadce21..0b7a7674 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -25,5 +25,11 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     expected = Nx.tensor([1, 3, 4])
     predicted = BayesianRidgeRegression.predict(clf, test)
     assert_all_close(expected, predicted, atol: 1.0e-3)
-  end  
+  end
+
+  test "compare ridge vs bayesian ridge" do
+    # go to sklearn tests and copy
+    assert false
+  end
+
 end

From a575c3b141e4159fcb9d253e50489f79d149c270 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 11:25:36 +0100
Subject: [PATCH 37/90] add intrecept option

---
 .../linear/bayesian_ridge_regression.ex       | 140 +++++++++++++-----
 .../linear/bayesian_ridge_regression_test.exs |  25 +---
 2 files changed, 114 insertions(+), 51 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 38db6566..ac97d556 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -3,8 +3,13 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   import Nx.Defn
   import Scholar.Shared
 
-  @derive {Nx.Container, containers: [:coefficients, :alpha, :lambda, :rmse, :iterations]}
-  defstruct [:coefficients, :alpha, :lambda, :rmse, :iterations]
+  @derive {Nx.Container,
+           containers: [:coefficients, :intercept,
+                        :alpha, :lambda,
+                        :rmse, :iterations]}
+  defstruct [:coefficients, :intercept,
+             :alpha, :lambda,
+             :rmse, :iterations]
 
   opts = [
     iterations: [
@@ -105,6 +110,13 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   @opts_schema NimbleOptions.new!(opts)
   deftransform fit(x, y, opts \\ []) do
     opts = NimbleOptions.validate!(opts, @opts_schema)
+
+    opts =
+      [
+        sample_weights_flag: opts[:sample_weights] != nil
+      ] ++
+        opts
+
     {sample_weights, opts} = Keyword.pop(opts, :sample_weights, 1.0)
     x_type = to_float_type(x)
 
@@ -113,24 +125,49 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         do: Nx.as_type(sample_weights, x_type),
         else: Nx.tensor(sample_weights, type: x_type)
 
+    # handle default alpha value
     alpha = Keyword.get(opts, :alpha_init, Nx.divide(1, Nx.variance(y)))
     opts = Keyword.put(opts, :alpha_init, alpha)
 
     num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
 
-    {{coefficients, rmse, iterations, has_converged, alpha, lambda}, _} =
+    {coefficients, intercept,
+      alpha, lambda, rmse,
+      iterations, has_converged} =
       fit_n(x, y, sample_weights, opts)
-
     if Nx.to_number(has_converged) == 1 do
       IO.puts("Convergence after #{Nx.to_number(iterations)} iterations")
     end
     %__MODULE__{
-      coefficients: coefficients,
-      alpha: alpha, lambda: lambda,
-      rmse: rmse, iterations: iterations}
+      coefficients: coefficients, intercept: intercept,
+      alpha: Nx.to_number(alpha), lambda: Nx.to_number(lambda),
+      rmse: Nx.to_number(rmse), iterations: Nx.to_number(iterations)}
   end
 
   defnp fit_n(x, y, sample_weights, opts) do
+    x = to_float(x)
+    y = to_float(y)
+
+    {x_offset, y_offset} =
+      if opts[:fit_intercept?] do
+        preprocess_data(x, y, sample_weights, opts)
+      else
+        x_offset_shape = Nx.axis_size(x, 1)
+        y_reshaped = if Nx.rank(y) > 1, do: y, else: Nx.reshape(y, {:auto, 1})
+        y_offset_shape = Nx.axis_size(y_reshaped, 1)
+
+        {Nx.broadcast(Nx.tensor(0.0, type: Nx.type(x)), {x_offset_shape}),
+         Nx.broadcast(Nx.tensor(0.0, type: Nx.type(y)), {y_offset_shape})}
+      end
+    {x, y} = {x - x_offset, y - y_offset}
+
+    {x, y} =
+      if opts[:sample_weights_flag] do
+        rescale(x, y, sample_weights)
+      else
+        {x, y}
+      end
+    
     alpha = opts[:alpha_init]
     alpha_1 = opts[:alpha_1]
     alpha_2 = opts[:alpha_2]
@@ -148,34 +185,38 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {coef, rmse} = update_coef(x, y, n_samples, n_features,
                                xt_y, u, vh, eigenvals,
                                alpha, lambda)
+    intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])    
 
-    while {{coef, rmse, iter = 0, has_converged = Nx.u8(0), alpha, lambda,},
-           {x, y,
-            xt_y, u, s, vh, eigenvals,
-            alpha_1, alpha_2, lambda_1, lambda_2,
-            iterations}},
-      iter < iterations and not has_converged do
-
-      # gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
-      gamma =
-        Nx.multiply(alpha, eigenvals)
-        |> Nx.divide(Nx.multiply(lambda + alpha, eigenvals))
-        |> Nx.sum()      
-      lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
-      alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
+    {{coef, alpha, lambda, rmse, iter, has_converged,}, _} =
+      while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0),},
+             {x, y,
+              xt_y, u, s, vh, eigenvals,
+              alpha_1, alpha_2, lambda_1, lambda_2,
+              iterations}},
+            iter < iterations and not has_converged do
+
+        # gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
+        gamma =
+          Nx.multiply(alpha, eigenvals)
+          |> Nx.divide(Nx.multiply(lambda + alpha, eigenvals))
+          |> Nx.sum()      
+        lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
+        alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
       
-      {coef_new, rmse} = update_coef(
-        x, y, n_samples, n_features,
-        xt_y, u, vh, eigenvals,
-        alpha, lambda)
+        {coef_new, rmse} = update_coef(
+                             x, y, n_samples, n_features,
+                             xt_y, u, vh, eigenvals,
+                             alpha, lambda)
       
-      has_converged = Nx.sum(Nx.abs(coef - coef_new)) < 1.0e-8
-      {{coef_new, rmse, iter + 1, has_converged, alpha, lambda,},
-       {x, y,
-        xt_y, u, s, vh, eigenvals,
-        alpha_1, alpha_2, lambda_1, lambda_2,
-        iterations}}
-    end
+        has_converged = Nx.sum(Nx.abs(coef - coef_new)) < 1.0e-8
+        {{coef_new, alpha, lambda, rmse, iter + 1, has_converged,},
+         {x, y,
+          xt_y, u, s, vh, eigenvals,
+          alpha_1, alpha_2, lambda_1, lambda_2,
+          iterations}}
+      end
+    intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
+    {coef, intercept, alpha, lambda, rmse, iter, has_converged,}
   end
 
   defn update_coef(
@@ -203,7 +244,38 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {coef, rmse}
   end
 
-  defn predict(%__MODULE__{coefficients: coeff} = _model, x) do
-    Nx.dot(x, coeff)
+  defn predict(%__MODULE__{coefficients: coeff, intercept: intercept} = _model, x) do
+    Nx.dot(x, [-1], coeff, [-1]) + intercept    
   end
+
+  # Implements sample weighting by rescaling inputs and
+  # targets by sqrt(sample_weight).
+  defnp rescale(x, y, sample_weights) do
+    case Nx.shape(sample_weights) do
+      {} = scalar ->
+        scalar = Nx.sqrt(scalar)
+        {scalar * x, scalar * y}
+
+      _ ->
+        scale = sample_weights |> Nx.sqrt() |> Nx.make_diagonal()
+        {Nx.dot(scale, x), Nx.dot(scale, y)}
+    end
+  end
+
+
+  defnp set_intercept(coeff, x_offset, y_offset, fit_intercept?) do
+    if fit_intercept? do
+      y_offset - Nx.dot(coeff, x_offset)
+    else
+      Nx.tensor(0.0, type: Nx.type(coeff))
+    end
+  end
+
+  defnp preprocess_data(x, y, sample_weights, opts) do
+    if opts[:sample_weights_flag],
+      do:
+        {Nx.weighted_mean(x, sample_weights, axes: [0]),
+         Nx.weighted_mean(y, sample_weights, axes: [0])},
+      else: {Nx.mean(x, axes: [0]), Nx.mean(y, axes: [0])}
+  end  
 end
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 0b7a7674..a06938c2 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -1,6 +1,7 @@
 defmodule Scholar.Linear.BayesianRidgeRegressionTest do
   use Scholar.Case, async: true
   alias Scholar.Linear.BayesianRidgeRegression
+  alias Scholar.Linear.RidgeRegression  
   doctest BayesianRidgeRegression
 
   test "toy bayesian ridge" do
@@ -10,26 +11,16 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     test = Nx.tensor([[1], [3], [4]])
     expected = Nx.tensor([1, 3, 4])
     predicted = BayesianRidgeRegression.predict(clf, test)
-    assert_all_close(expected, predicted, atol: 1.0e-6)
-  end
-  
-  test "multi column toy bayesian ridge" do
-    x = Nx.tensor([
-      [1, 5], [2, 6], [6, 6], [8, 4], [10, 0],
-      [5, 5], [6, 2], [6, 4], [4, 2], [0, 10],      
-    ])
-    true_coef = Nx.tensor([0.5, 0.5])
-    y = Nx.dot(x, true_coef)
-    clf = BayesianRidgeRegression.fit(x, y)
-    test = Nx.tensor([[1, 1], [3, 3], [4, 4]])
-    expected = Nx.tensor([1, 3, 4])
-    predicted = BayesianRidgeRegression.predict(clf, test)
-    assert_all_close(expected, predicted, atol: 1.0e-3)
+    assert_all_close(expected, predicted, atol: 1.0e-1)
   end
 
   test "compare ridge vs bayesian ridge" do
-    # go to sklearn tests and copy
-    assert false
+    x = Nx.tensor([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
+    y = Nx.tensor([1, 2, 3, 2, 0, 4, 5])
+    brr = BayesianRidgeRegression.fit(x, y)
+    rr = RidgeRegression.fit(x, y, alpha: brr.lambda / brr.alpha)
+    assert_all_close(brr.coefficients, rr.coefficients, atol: 1.0e-2)
+    assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
 
 end

From 6d5d2d4589259aa1d2307c3f6e56b5f4ec84ca46 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 11:31:10 +0100
Subject: [PATCH 38/90] add test weights options

---
 .../linear/bayesian_ridge_regression_test.exs      | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index a06938c2..b26f7509 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -14,7 +14,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(expected, predicted, atol: 1.0e-1)
   end
 
-  test "compare ridge vs bayesian ridge" do
+  test "compare ridge vs bayesian ridge: parameters" do
     x = Nx.tensor([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
     y = Nx.tensor([1, 2, 3, 2, 0, 4, 5])
     brr = BayesianRidgeRegression.fit(x, y)
@@ -22,5 +22,17 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(brr.coefficients, rr.coefficients, atol: 1.0e-2)
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
+  
+  test "compare ridge vs bayesian ridge: weights" do
+    x = Nx.tensor([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
+    y = Nx.tensor([1, 2, 3, 2, 0, 4, 5])
+    w = Nx.tensor([4, 3, 3, 1, 1, 2, 3])
+    brr = BayesianRidgeRegression.fit(x, y, sample_weights: w)
+    IO.inspect(brr)
+    rr = RidgeRegression.fit(x, y, alpha: brr.lambda / brr.alpha, sample_weights: w)
+    IO.inspect(rr)
+    assert_all_close(brr.coefficients, rr.coefficients, atol: 1.0e-2)
+    assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
+  end  
 
 end

From fc4e12443222ce89b800d0b5e760ff97a6110a83 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 11:42:08 +0100
Subject: [PATCH 39/90] better naming + next steps

---
 .../linear/bayesian_ridge_regression_test.exs     | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index b26f7509..1dc77d42 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -14,7 +14,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(expected, predicted, atol: 1.0e-1)
   end
 
-  test "compare ridge vs bayesian ridge: parameters" do
+  test "ridge vs bayesian ridge: parameters" do
     x = Nx.tensor([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
     y = Nx.tensor([1, 2, 3, 2, 0, 4, 5])
     brr = BayesianRidgeRegression.fit(x, y)
@@ -23,7 +23,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
   
-  test "compare ridge vs bayesian ridge: weights" do
+  test "ridge vs bayesian ridge: weights" do
     x = Nx.tensor([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
     y = Nx.tensor([1, 2, 3, 2, 0, 4, 5])
     w = Nx.tensor([4, 3, 3, 1, 1, 2, 3])
@@ -33,6 +33,15 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     IO.inspect(rr)
     assert_all_close(brr.coefficients, rr.coefficients, atol: 1.0e-2)
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
-  end  
+  end
 
+  test "compute scores" do
+    assert false
+  end
+  test "constant inputs: prediction" do
+    assert false
+  end
+  test "constant inputs: variance" do
+    assert false
+  end  
 end

From c96a6f0a992e7aec3fb9fc0dbb47d321da6b8782 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 12:31:32 +0100
Subject: [PATCH 40/90] add eps to initial alpha to avoid division by 0

---
 lib/scholar/linear/bayesian_ridge_regression.ex        | 4 +++-
 test/scholar/linear/bayesian_ridge_regression_test.exs | 2 --
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index ac97d556..b180b7c9 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -126,7 +126,9 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         else: Nx.tensor(sample_weights, type: x_type)
 
     # handle default alpha value
-    alpha = Keyword.get(opts, :alpha_init, Nx.divide(1, Nx.variance(y)))
+    eps = Nx.Constants.smallest_positive_normal({:f, 64})
+    default_alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
+    alpha = Keyword.get(opts, :alpha_init, Nx.to_number(default_alpha))
     opts = Keyword.put(opts, :alpha_init, alpha)
 
     num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 1dc77d42..9030858a 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -28,9 +28,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     y = Nx.tensor([1, 2, 3, 2, 0, 4, 5])
     w = Nx.tensor([4, 3, 3, 1, 1, 2, 3])
     brr = BayesianRidgeRegression.fit(x, y, sample_weights: w)
-    IO.inspect(brr)
     rr = RidgeRegression.fit(x, y, alpha: brr.lambda / brr.alpha, sample_weights: w)
-    IO.inspect(rr)
     assert_all_close(brr.coefficients, rr.coefficients, atol: 1.0e-2)
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end

From bd7514303e1200956794868172affa4361ba61b8 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 12:32:07 +0100
Subject: [PATCH 41/90] comment on previous commit

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index b180b7c9..b5047dcf 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -125,7 +125,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         do: Nx.as_type(sample_weights, x_type),
         else: Nx.tensor(sample_weights, type: x_type)
 
-    # handle default alpha value
+    # handle default alpha value, add eps to avoid division by 0
     eps = Nx.Constants.smallest_positive_normal({:f, 64})
     default_alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
     alpha = Keyword.get(opts, :alpha_init, Nx.to_number(default_alpha))

From 37de967f7dfc2accacdb4888e3d402b4fe8b4215 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 12:33:38 +0100
Subject: [PATCH 42/90] clean gamma calculation

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index b5047dcf..fed0260d 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -197,11 +197,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
               iterations}},
             iter < iterations and not has_converged do
 
-        # gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
-        gamma =
-          Nx.multiply(alpha, eigenvals)
-          |> Nx.divide(Nx.multiply(lambda + alpha, eigenvals))
-          |> Nx.sum()      
+        gamma = (alpha * eigenvals / (lambda + alpha * eigenvals))
+        |> Nx.sum()
         lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
         alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
       

From 469a9b9bf288dff626271a70c59deaca7d726dbf Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 12:44:10 +0100
Subject: [PATCH 43/90] add another todo

---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 9030858a..e2f9e6df 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -41,5 +41,8 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
   end
   test "constant inputs: variance" do
     assert false
-  end  
+  end
+  test "n_features > n_samples" do
+    assert false
+  end    
 end

From 4d2d782ca032a4ee1ff0411e7dbd0b0358d0f2ee Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Thu, 28 Mar 2024 12:46:05 +0100
Subject: [PATCH 44/90] run formatter

---
 .../linear/bayesian_ridge_regression.ex       | 141 +++++++++---------
 .../linear/bayesian_ridge_regression_test.exs |   9 +-
 2 files changed, 75 insertions(+), 75 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index fed0260d..9765f793 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -4,12 +4,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   import Scholar.Shared
 
   @derive {Nx.Container,
-           containers: [:coefficients, :intercept,
-                        :alpha, :lambda,
-                        :rmse, :iterations]}
-  defstruct [:coefficients, :intercept,
-             :alpha, :lambda,
-             :rmse, :iterations]
+           containers: [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations]}
+  defstruct [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations]
 
   opts = [
     iterations: [
@@ -19,7 +15,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       Maximum number of iterations before stopping the fitting algorithm.
       The number of iterations may be lower is parameters converge.
       """
-    ],    
+    ],
     sample_weights: [
       type:
         {:or,
@@ -44,8 +40,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       """
     ],
     alpha_init: [
-      type:
-        {:custom, Scholar.Options, :non_negative_number, []},
+      type: {:custom, Scholar.Options, :non_negative_number, []},
       doc: ~S"""
       The initial value for alpha. This parameter influences the precision of the noise.
       `:alpha` must be a non-negative float i.e. in [0, inf).
@@ -53,8 +48,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       """
     ],
     lambda_init: [
-      type:
-        {:custom, Scholar.Options, :non_negative_number, []},
+      type: {:custom, Scholar.Options, :non_negative_number, []},
       default: 1.0,
       doc: ~S"""
       The initial value for lambda. This parameter influences the precision of the weights.
@@ -63,17 +57,15 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       """
     ],
     alpha_1: [
-      type:
-        {:custom, Scholar.Options, :non_negative_number, []},
+      type: {:custom, Scholar.Options, :non_negative_number, []},
       default: 1.0e-6,
       doc: ~S"""
       Hyper-parameter : shape parameter for the Gamma distribution prior      
       over the alpha parameter.
       """
-    ],    
+    ],
     alpha_2: [
-      type:
-        {:custom, Scholar.Options, :non_negative_number, []},
+      type: {:custom, Scholar.Options, :non_negative_number, []},
       default: 1.0e-6,
       doc: ~S"""
       Hyper-parameter : inverse scale (rate) parameter for the Gamma distribution prior
@@ -81,8 +73,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       """
     ],
     lambda_1: [
-      type:
-        {:custom, Scholar.Options, :non_negative_number, []},
+      type: {:custom, Scholar.Options, :non_negative_number, []},
       default: 1.0e-6,
       doc: ~S"""
       Hyper-parameter : shape parameter for the Gamma distribution prior
@@ -90,21 +81,19 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       """
     ],
     lambda_2: [
-      type:
-        {:custom, Scholar.Options, :non_negative_number, []},
+      type: {:custom, Scholar.Options, :non_negative_number, []},
       default: 1.0e-6,
       doc: ~S"""
       Hyper-parameter : inverse scale (rate) parameter for the Gamma distribution prior      
       over the lambda parameter.            
       """
-    ],    
+    ],
     eps: [
       type: :float,
       default: 1.0e-8,
       doc:
         "The convergence tolerance. When `Nx.sum(Nx.abs(coef - coef_new)) < :eps`, the algorithm is considered to have converged."
-    ]    
-    
+    ]
   ]
 
   @opts_schema NimbleOptions.new!(opts)
@@ -133,17 +122,21 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
 
     num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
 
-    {coefficients, intercept,
-      alpha, lambda, rmse,
-      iterations, has_converged} =
+    {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged} =
       fit_n(x, y, sample_weights, opts)
+
     if Nx.to_number(has_converged) == 1 do
       IO.puts("Convergence after #{Nx.to_number(iterations)} iterations")
     end
+
     %__MODULE__{
-      coefficients: coefficients, intercept: intercept,
-      alpha: Nx.to_number(alpha), lambda: Nx.to_number(lambda),
-      rmse: Nx.to_number(rmse), iterations: Nx.to_number(iterations)}
+      coefficients: coefficients,
+      intercept: intercept,
+      alpha: Nx.to_number(alpha),
+      lambda: Nx.to_number(lambda),
+      rmse: Nx.to_number(rmse),
+      iterations: Nx.to_number(iterations)
+    }
   end
 
   defnp fit_n(x, y, sample_weights, opts) do
@@ -161,6 +154,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         {Nx.broadcast(Nx.tensor(0.0, type: Nx.type(x)), {x_offset_shape}),
          Nx.broadcast(Nx.tensor(0.0, type: Nx.type(y)), {y_offset_shape})}
       end
+
     {x, y} = {x - x_offset, y - y_offset}
 
     {x, y} =
@@ -169,7 +163,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       else
         {x, y}
       end
-    
+
     alpha = opts[:alpha_init]
     alpha_1 = opts[:alpha_1]
     alpha_2 = opts[:alpha_2]
@@ -178,61 +172,65 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     lambda_1 = opts[:lambda_1]
     lambda_2 = opts[:lambda_2]
 
-    iterations = opts[:iterations]    
-    
+    iterations = opts[:iterations]
+
     xt_y = Nx.dot(Nx.transpose(x), y)
     {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
     eigenvals = Nx.pow(s, 2)
     {n_samples, n_features} = Nx.shape(x)
-    {coef, rmse} = update_coef(x, y, n_samples, n_features,
-                               xt_y, u, vh, eigenvals,
-                               alpha, lambda)
-    intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])    
-
-    {{coef, alpha, lambda, rmse, iter, has_converged,}, _} =
-      while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0),},
-             {x, y,
-              xt_y, u, s, vh, eigenvals,
-              alpha_1, alpha_2, lambda_1, lambda_2,
-              iterations}},
+    {coef, rmse} = update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
+    intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
+
+    {{coef, alpha, lambda, rmse, iter, has_converged}, _} =
+      while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0)},
+             {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}},
             iter < iterations and not has_converged do
+        gamma =
+          (alpha * eigenvals / (lambda + alpha * eigenvals))
+          |> Nx.sum()
 
-        gamma = (alpha * eigenvals / (lambda + alpha * eigenvals))
-        |> Nx.sum()
         lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
         alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
-      
-        {coef_new, rmse} = update_coef(
-                             x, y, n_samples, n_features,
-                             xt_y, u, vh, eigenvals,
-                             alpha, lambda)
-      
+
+        {coef_new, rmse} =
+          update_coef(
+            x,
+            y,
+            n_samples,
+            n_features,
+            xt_y,
+            u,
+            vh,
+            eigenvals,
+            alpha,
+            lambda
+          )
+
         has_converged = Nx.sum(Nx.abs(coef - coef_new)) < 1.0e-8
-        {{coef_new, alpha, lambda, rmse, iter + 1, has_converged,},
-         {x, y,
-          xt_y, u, s, vh, eigenvals,
-          alpha_1, alpha_2, lambda_1, lambda_2,
-          iterations}}
+
+        {{coef_new, alpha, lambda, rmse, iter + 1, has_converged},
+         {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}}
       end
+
     intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
-    {coef, intercept, alpha, lambda, rmse, iter, has_converged,}
+    {coef, intercept, alpha, lambda, rmse, iter, has_converged}
   end
 
   defn update_coef(
-    x,
-    y,
-    n_samples,
-    n_features,
-    xt_y,
-    u,
-    vh,
-    eigenvals,
-    alpha,
-    lambda
-  ) do
+         x,
+         y,
+         n_samples,
+         n_features,
+         xt_y,
+         u,
+         vh,
+         eigenvals,
+         alpha,
+         lambda
+       ) do
     scaled_eigens = eigenvals + lambda / alpha
     {n_eigens} = Nx.shape(scaled_eigens)
-    regularization = vh / Nx.new_axis(scaled_eigens, -1) 
+    regularization = vh / Nx.new_axis(scaled_eigens, -1)
     reg_transpose = Nx.dot(regularization, xt_y)
     coef = Nx.dot(Nx.transpose(vh), reg_transpose)
 
@@ -244,7 +242,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   end
 
   defn predict(%__MODULE__{coefficients: coeff, intercept: intercept} = _model, x) do
-    Nx.dot(x, [-1], coeff, [-1]) + intercept    
+    Nx.dot(x, [-1], coeff, [-1]) + intercept
   end
 
   # Implements sample weighting by rescaling inputs and
@@ -261,7 +259,6 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     end
   end
 
-
   defnp set_intercept(coeff, x_offset, y_offset, fit_intercept?) do
     if fit_intercept? do
       y_offset - Nx.dot(coeff, x_offset)
@@ -276,5 +273,5 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         {Nx.weighted_mean(x, sample_weights, axes: [0]),
          Nx.weighted_mean(y, sample_weights, axes: [0])},
       else: {Nx.mean(x, axes: [0]), Nx.mean(y, axes: [0])}
-  end  
+  end
 end
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index e2f9e6df..402900da 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -1,7 +1,7 @@
 defmodule Scholar.Linear.BayesianRidgeRegressionTest do
   use Scholar.Case, async: true
   alias Scholar.Linear.BayesianRidgeRegression
-  alias Scholar.Linear.RidgeRegression  
+  alias Scholar.Linear.RidgeRegression
   doctest BayesianRidgeRegression
 
   test "toy bayesian ridge" do
@@ -22,7 +22,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(brr.coefficients, rr.coefficients, atol: 1.0e-2)
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
-  
+
   test "ridge vs bayesian ridge: weights" do
     x = Nx.tensor([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
     y = Nx.tensor([1, 2, 3, 2, 0, 4, 5])
@@ -36,13 +36,16 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
   test "compute scores" do
     assert false
   end
+
   test "constant inputs: prediction" do
     assert false
   end
+
   test "constant inputs: variance" do
     assert false
   end
+
   test "n_features > n_samples" do
     assert false
-  end    
+  end
 end

From 3cf69c235c66b2c84ef0cdeeea87185ee2c08a5c Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 29 Mar 2024 10:25:34 +0100
Subject: [PATCH 45/90] remove unused variable

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 9765f793..49d172d6 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -229,7 +229,6 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
          lambda
        ) do
     scaled_eigens = eigenvals + lambda / alpha
-    {n_eigens} = Nx.shape(scaled_eigens)
     regularization = vh / Nx.new_axis(scaled_eigens, -1)
     reg_transpose = Nx.dot(regularization, xt_y)
     coef = Nx.dot(Nx.transpose(vh), reg_transpose)

From 988243cc0d8dece5c25304e77e6a17f5d4610021 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 29 Mar 2024 10:48:23 +0100
Subject: [PATCH 46/90] fix regularization parameter types

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 49d172d6..918589e1 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -113,14 +113,17 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       if Nx.is_tensor(sample_weights),
         do: Nx.as_type(sample_weights, x_type),
         else: Nx.tensor(sample_weights, type: x_type)
-
+    # handle vector types
     # handle default alpha value, add eps to avoid division by 0
     eps = Nx.Constants.smallest_positive_normal({:f, 64})
     default_alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
-    alpha = Keyword.get(opts, :alpha_init, Nx.to_number(default_alpha))
+    alpha = Keyword.get(opts, :alpha_init, default_alpha)
+    alpha = Nx.tensor(alpha, type: x_type)
     opts = Keyword.put(opts, :alpha_init, alpha)
 
-    num_targets = if Nx.rank(y) == 1, do: 1, else: Nx.axis_size(y, 1)
+    {lambda, opts} = Keyword.pop!(opts, :lambda_init)
+    lambda = Nx.tensor(lambda, type: x_type)
+    opts = Keyword.put(opts, :lambda_init, lambda)
 
     {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged} =
       fit_n(x, y, sample_weights, opts)
@@ -163,12 +166,11 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       else
         {x, y}
       end
-
     alpha = opts[:alpha_init]
+    lambda = opts[:lambda_init]
+
     alpha_1 = opts[:alpha_1]
     alpha_2 = opts[:alpha_2]
-
-    lambda = opts[:lambda_init]
     lambda_1 = opts[:lambda_1]
     lambda_2 = opts[:lambda_2]
 
@@ -179,7 +181,6 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     eigenvals = Nx.pow(s, 2)
     {n_samples, n_features} = Nx.shape(x)
     {coef, rmse} = update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
-    intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
 
     {{coef, alpha, lambda, rmse, iter, has_converged}, _} =
       while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0)},

From 889de37c0249e8257ce8866858b8ba02c8279a1f Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 29 Mar 2024 13:05:50 +0100
Subject: [PATCH 47/90] working on test to compute score

---
 .../linear/bayesian_ridge_regression_test.exs | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 402900da..99c2ade6 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -1,4 +1,5 @@
 defmodule Scholar.Linear.BayesianRidgeRegressionTest do
+  import Nx.Defn  
   use Scholar.Case, async: true
   alias Scholar.Linear.BayesianRidgeRegression
   alias Scholar.Linear.RidgeRegression
@@ -33,10 +34,51 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
 
+  @tag :wip
   test "compute scores" do
+    {x, y, noise} = data()
+
+    eps = Nx.Constants.smallest_positive_normal({:f, 64})
+    alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
+    lambda = Nx.tensor(1.0)
+    alpha_1 = 0.1
+    alpha_2 = 0.1
+    lambda_1 = 0.1
+    lambda_2 = 0.1
+    # compute score
+    score = compute_score(x, y, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2)
+    IO.inspect(score)
+    brr = BayesianRidgeRegression.fit(x, y,
+      alpha_1: alpha_1, alpha_2: alpha_2, lambda_1: lambda_1, lambda_2: lambda_2,
+      fit_intercept?: false, iterations: 1)
     assert false
   end
 
+  defnp compute_score(x, y, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2) do
+    {n_samples, _} = Nx.shape(x)
+    score_lambda = lambda_1 + Nx.log(lambda) - lambda_2 * lambda
+    score_alpha =  alpha_1 + Nx.log(alpha_2) - alpha_2 * alpha
+    m =  (1.0 / alpha) * Nx.eye(n_samples) + (1.0 / lambda) * Nx.dot(x, Nx.transpose(x))
+    m_inv_dot_y = Nx.LinAlg.solve(m, y)
+    logdet = m |> Nx.LinAlg.determinant |> Nx.log()
+    y_score = -0.5 * (logdet + Nx.dot(y, m_inv_dot_y) +
+                        n_samples * Nx.log(2 * Nx.Constants.pi()))
+    
+    score_alpha + score_lambda + y_score
+  end
+
+  # copied from the linear regression notebook
+  # https://hexdocs.pm/scholar/linear_regression.html
+  defnp data do
+    key = Nx.Random.key(42)
+    size = 30
+    {x, new_key} = Nx.Random.normal(key, 0, 2, shape: {size, 3}, type: :f64)
+    {noise, _} = Nx.Random.normal(new_key, -0.5, 0.5, shape: {size}, type: :f64)
+    w = Nx.tensor([1, 2, 3])
+    y = Nx.dot(x, w) + 4
+    {x, y, noise}
+  end
+
   test "constant inputs: prediction" do
     assert false
   end

From 8c969c68eda9c9507f3b721fcc9198ffde5d4362 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 29 Mar 2024 13:08:27 +0100
Subject: [PATCH 48/90] better naming + data

---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 99c2ade6..26ffe204 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -56,24 +56,23 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
 
   defnp compute_score(x, y, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2) do
     {n_samples, _} = Nx.shape(x)
-    score_lambda = lambda_1 + Nx.log(lambda) - lambda_2 * lambda
-    score_alpha =  alpha_1 + Nx.log(alpha_2) - alpha_2 * alpha
+    lambda_score = lambda_1 + Nx.log(lambda) - lambda_2 * lambda
+    alpha_score =  alpha_1 + Nx.log(alpha_2) - alpha_2 * alpha
     m =  (1.0 / alpha) * Nx.eye(n_samples) + (1.0 / lambda) * Nx.dot(x, Nx.transpose(x))
     m_inv_dot_y = Nx.LinAlg.solve(m, y)
     logdet = m |> Nx.LinAlg.determinant |> Nx.log()
     y_score = -0.5 * (logdet + Nx.dot(y, m_inv_dot_y) +
                         n_samples * Nx.log(2 * Nx.Constants.pi()))
-    
     score_alpha + score_lambda + y_score
   end
 
-  # copied from the linear regression notebook
+  # adapted from the linear regression notebook
   # https://hexdocs.pm/scholar/linear_regression.html
   defnp data do
     key = Nx.Random.key(42)
     size = 30
     {x, new_key} = Nx.Random.normal(key, 0, 2, shape: {size, 3}, type: :f64)
-    {noise, _} = Nx.Random.normal(new_key, -0.5, 0.5, shape: {size}, type: :f64)
+    {noise, _} = Nx.Random.normal(new_key, 0, 1, shape: {size}, type: :f64)
     w = Nx.tensor([1, 2, 3])
     y = Nx.dot(x, w) + 4
     {x, y, noise}

From 514b16764875839cd024458ebe1fb19efceb8293 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sat, 30 Mar 2024 12:23:12 +0100
Subject: [PATCH 49/90] add diabetes data

---
 test/support/diabetes_data_raw.csv | 442 +++++++++++++++++++++++++++++
 test/support/diabetes_target.csv   | 442 +++++++++++++++++++++++++++++
 test/support/scholar_case.ex       |  29 ++
 3 files changed, 913 insertions(+)
 create mode 100644 test/support/diabetes_data_raw.csv
 create mode 100644 test/support/diabetes_target.csv

diff --git a/test/support/diabetes_data_raw.csv b/test/support/diabetes_data_raw.csv
new file mode 100644
index 00000000..95ed9eca
--- /dev/null
+++ b/test/support/diabetes_data_raw.csv
@@ -0,0 +1,442 @@
+59 2 32.1 101.0 157 93.2 38.0 4.0 4.8598 87
+48 1 21.6 87.0 183 103.2 70.0 3.0 3.8918 69
+72 2 30.5 93.0 156 93.6 41.0 4.0 4.6728 85
+24 1 25.3 84.0 198 131.4 40.0 5.0 4.8903 89
+50 1 23.0 101.0 192 125.4 52.0 4.0 4.2905 80
+23 1 22.6 89.0 139 64.8 61.0 2.0 4.1897 68
+36 2 22.0 90.0 160 99.6 50.0 3.0 3.9512 82
+66 2 26.2 114.0 255 185.0 56.0 4.55 4.2485 92
+60 2 32.1 83.0 179 119.4 42.0 4.0 4.4773 94
+29 1 30.0 85.0 180 93.4 43.0 4.0 5.3845 88
+22 1 18.6 97.0 114 57.6 46.0 2.0 3.9512 83
+56 2 28.0 85.0 184 144.8 32.0 6.0 3.5835 77
+53 1 23.7 92.0 186 109.2 62.0 3.0 4.3041 81
+50 2 26.2 97.0 186 105.4 49.0 4.0 5.0626 88
+61 1 24.0 91.0 202 115.4 72.0 3.0 4.2905 73
+34 2 24.7 118.0 254 184.2 39.0 7.0 5.037 81
+47 1 30.3 109.0 207 100.2 70.0 3.0 5.2149 98
+68 2 27.5 111.0 214 147.0 39.0 5.0 4.9416 91
+38 1 25.4 84.0 162 103.0 42.0 4.0 4.4427 87
+41 1 24.7 83.0 187 108.2 60.0 3.0 4.5433 78
+35 1 21.1 82.0 156 87.8 50.0 3.0 4.5109 95
+25 2 24.3 95.0 162 98.6 54.0 3.0 3.8501 87
+25 1 26.0 92.0 187 120.4 56.0 3.0 3.9703 88
+61 2 32.0 103.67 210 85.2 35.0 6.0 6.107 124
+31 1 29.7 88.0 167 103.4 48.0 4.0 4.3567 78
+30 2 25.2 83.0 178 118.4 34.0 5.0 4.852 83
+19 1 19.2 87.0 124 54.0 57.0 2.0 4.1744 90
+42 1 31.9 83.0 158 87.6 53.0 3.0 4.4659 101
+63 1 24.4 73.0 160 91.4 48.0 3.0 4.6347 78
+67 2 25.8 113.0 158 54.2 64.0 2.0 5.2933 104
+32 1 30.5 89.0 182 110.6 56.0 3.0 4.3438 89
+42 1 20.3 71.0 161 81.2 66.0 2.0 4.2341 81
+58 2 38.0 103.0 150 107.2 22.0 7.0 4.6444 98
+57 1 21.7 94.0 157 58.0 82.0 2.0 4.4427 92
+53 1 20.5 78.0 147 84.2 52.0 3.0 3.989 75
+62 2 23.5 80.33 225 112.8 86.0 2.62 4.8752 96
+52 1 28.5 110.0 195 97.2 60.0 3.0 5.2417 85
+46 1 27.4 78.0 171 88.0 58.0 3.0 4.8283 90
+48 2 33.0 123.0 253 163.6 44.0 6.0 5.425 97
+48 2 27.7 73.0 191 119.4 46.0 4.0 4.852 92
+50 2 25.6 101.0 229 162.2 43.0 5.0 4.7791 114
+21 1 20.1 63.0 135 69.0 54.0 3.0 4.0943 89
+32 2 25.4 90.33 153 100.4 34.0 4.5 4.5326 83
+54 1 24.2 74.0 204 109.0 82.0 2.0 4.1744 109
+61 2 32.7 97.0 177 118.4 29.0 6.0 4.9972 87
+56 2 23.1 104.0 181 116.4 47.0 4.0 4.4773 79
+33 1 25.3 85.0 155 85.0 51.0 3.0 4.5539 70
+27 1 19.6 78.0 128 68.0 43.0 3.0 4.4427 71
+67 2 22.5 98.0 191 119.2 61.0 3.0 3.989 86
+37 2 27.7 93.0 180 119.4 30.0 6.0 5.0304 88
+58 1 25.7 99.0 157 91.6 49.0 3.0 4.4067 93
+65 2 27.9 103.0 159 96.8 42.0 4.0 4.6151 86
+34 1 25.5 93.0 218 144.0 57.0 4.0 4.4427 88
+46 1 24.9 115.0 198 129.6 54.0 4.0 4.2767 103
+35 1 28.7 97.0 204 126.8 64.0 3.0 4.1897 93
+37 1 21.8 84.0 184 101.0 73.0 3.0 3.912 93
+37 1 30.2 87.0 166 96.0 40.0 4.15 5.0106 87
+41 1 20.5 80.0 124 48.8 64.0 2.0 4.0254 75
+60 1 20.4 105.0 198 78.4 99.0 2.0 4.6347 79
+66 2 24.0 98.0 236 146.4 58.0 4.0 5.0626 96
+29 1 26.0 83.0 141 65.2 64.0 2.0 4.0775 83
+37 2 26.8 79.0 157 98.0 28.0 6.0 5.0434 96
+41 2 25.7 83.0 181 106.6 66.0 3.0 3.7377 85
+39 1 22.9 77.0 204 143.2 46.0 4.0 4.3041 74
+67 2 24.0 83.0 143 77.2 49.0 3.0 4.4308 94
+36 2 24.1 112.0 193 125.0 35.0 6.0 5.1059 95
+46 2 24.7 85.0 174 123.2 30.0 6.0 4.6444 96
+60 2 25.0 89.67 185 120.8 46.0 4.02 4.5109 92
+59 2 23.6 83.0 165 100.0 47.0 4.0 4.4998 92
+53 1 22.1 93.0 134 76.2 46.0 3.0 4.0775 96
+48 1 19.9 91.0 189 109.6 69.0 3.0 3.9512 101
+48 1 29.5 131.0 207 132.2 47.0 4.0 4.9345 106
+66 2 26.0 91.0 264 146.6 65.0 4.0 5.5683 87
+52 2 24.5 94.0 217 149.4 48.0 5.0 4.585 89
+52 2 26.6 111.0 209 126.4 61.0 3.0 4.6821 109
+46 2 23.5 87.0 181 114.8 44.0 4.0 4.7095 98
+40 2 29.0 115.0 97 47.2 35.0 2.77 4.3041 95
+22 1 23.0 73.0 161 97.8 54.0 3.0 3.8286 91
+50 1 21.0 88.0 140 71.8 35.0 4.0 5.112 71
+20 1 22.9 87.0 191 128.2 53.0 4.0 3.8918 85
+68 1 27.5 107.0 241 149.6 64.0 4.0 4.92 90
+52 2 24.3 86.0 197 133.6 44.0 5.0 4.5747 91
+44 1 23.1 87.0 213 126.4 77.0 3.0 3.8712 72
+38 1 27.3 81.0 146 81.6 47.0 3.0 4.4659 81
+49 1 22.7 65.33 168 96.2 62.0 2.71 3.8918 60
+61 1 33.0 95.0 182 114.8 54.0 3.0 4.1897 74
+29 2 19.4 83.0 152 105.8 39.0 4.0 3.5835 83
+61 1 25.8 98.0 235 125.8 76.0 3.0 5.112 82
+34 2 22.6 75.0 166 91.8 60.0 3.0 4.2627 108
+36 1 21.9 89.0 189 105.2 68.0 3.0 4.3694 96
+52 1 24.0 83.0 167 86.6 71.0 2.0 3.8501 94
+61 1 31.2 79.0 235 156.8 47.0 5.0 5.0499 96
+43 1 26.8 123.0 193 102.2 67.0 3.0 4.7791 94
+35 1 20.4 65.0 187 105.6 67.0 2.79 4.2767 78
+27 1 24.8 91.0 189 106.8 69.0 3.0 4.1897 69
+29 1 21.0 71.0 156 97.0 38.0 4.0 4.654 90
+64 2 27.3 109.0 186 107.6 38.0 5.0 5.3083 99
+41 1 34.6 87.33 205 142.6 41.0 5.0 4.6728 110
+49 2 25.9 91.0 178 106.6 52.0 3.0 4.5747 75
+48 1 20.4 98.0 209 139.4 46.0 5.0 4.7707 78
+53 1 28.0 88.0 233 143.8 58.0 4.0 5.0499 91
+53 2 22.2 113.0 197 115.2 67.0 3.0 4.3041 100
+23 1 29.0 90.0 216 131.4 65.0 3.0 4.585 91
+65 2 30.2 98.0 219 160.6 40.0 5.0 4.5218 84
+41 1 32.4 94.0 171 104.4 56.0 3.0 3.9703 76
+55 2 23.4 83.0 166 101.6 46.0 4.0 4.5218 96
+22 1 19.3 82.0 156 93.2 52.0 3.0 3.989 71
+56 1 31.0 78.67 187 141.4 34.0 5.5 4.0604 90
+54 2 30.6 103.33 144 79.8 30.0 4.8 5.1417 101
+59 2 25.5 95.33 190 139.4 35.0 5.43 4.3567 117
+60 2 23.4 88.0 153 89.8 58.0 3.0 3.2581 95
+54 1 26.8 87.0 206 122.0 68.0 3.0 4.382 80
+25 1 28.3 87.0 193 128.0 49.0 4.0 4.382 92
+54 2 27.7 113.0 200 128.4 37.0 5.0 5.1533 113
+55 1 36.6 113.0 199 94.4 43.0 4.63 5.7301 97
+40 2 26.5 93.0 236 147.0 37.0 7.0 5.5607 92
+62 2 31.8 115.0 199 128.6 44.0 5.0 4.8828 98
+65 1 24.4 120.0 222 135.6 37.0 6.0 5.5094 124
+33 2 25.4 102.0 206 141.0 39.0 5.0 4.8675 105
+53 1 22.0 94.0 175 88.0 59.0 3.0 4.9416 98
+35 1 26.8 98.0 162 103.6 45.0 4.0 4.2047 86
+66 1 28.0 101.0 195 129.2 40.0 5.0 4.8598 94
+62 2 33.9 101.0 221 156.4 35.0 6.0 4.9972 103
+50 2 29.6 94.33 300 242.4 33.0 9.09 4.8122 109
+47 1 28.6 97.0 164 90.6 56.0 3.0 4.4659 88
+47 2 25.6 94.0 165 74.8 40.0 4.0 5.5255 93
+24 1 20.7 87.0 149 80.6 61.0 2.0 3.6109 78
+58 2 26.2 91.0 217 124.2 71.0 3.0 4.6913 68
+34 1 20.6 87.0 185 112.2 58.0 3.0 4.3041 74
+51 1 27.9 96.0 196 122.2 42.0 5.0 5.0689 120
+31 2 35.3 125.0 187 112.4 48.0 4.0 4.8903 109
+22 1 19.9 75.0 175 108.6 54.0 3.0 4.1271 72
+53 2 24.4 92.0 214 146.0 50.0 4.0 4.4998 97
+37 2 21.4 83.0 128 69.6 49.0 3.0 3.8501 84
+28 1 30.4 85.0 198 115.6 67.0 3.0 4.3438 80
+47 1 31.6 84.0 154 88.0 30.0 5.1 5.1985 105
+23 1 18.8 78.0 145 72.0 63.0 2.0 3.912 86
+50 1 31.0 123.0 178 105.0 48.0 4.0 4.8283 88
+58 2 36.7 117.0 166 93.8 44.0 4.0 4.9488 109
+55 1 32.1 110.0 164 84.2 42.0 4.0 5.2417 90
+60 2 27.7 107.0 167 114.6 38.0 4.0 4.2767 95
+41 1 30.8 81.0 214 152.0 28.0 7.6 5.1358 123
+60 2 27.5 106.0 229 143.8 51.0 4.0 5.1417 91
+40 1 26.9 92.0 203 119.8 70.0 3.0 4.1897 81
+57 2 30.7 90.0 204 147.8 34.0 6.0 4.7095 93
+37 1 38.3 113.0 165 94.6 53.0 3.0 4.4659 79
+40 2 31.9 95.0 198 135.6 38.0 5.0 4.803999999999999 93
+33 1 35.0 89.0 200 130.4 42.0 4.76 4.9273 101
+32 2 27.8 89.0 216 146.2 55.0 4.0 4.3041 91
+35 2 25.9 81.0 174 102.4 31.0 6.0 5.3132 82
+55 1 32.9 102.0 164 106.2 41.0 4.0 4.4308 89
+49 1 26.0 93.0 183 100.2 64.0 3.0 4.5433 88
+39 2 26.3 115.0 218 158.2 32.0 7.0 4.9345 109
+60 2 22.3 113.0 186 125.8 46.0 4.0 4.2627 94
+67 2 28.3 93.0 204 132.2 49.0 4.0 4.7362 92
+41 2 32.0 109.0 251 170.6 49.0 5.0 5.0562 103
+44 1 25.4 95.0 162 92.6 53.0 3.0 4.4067 83
+48 2 23.3 89.33 212 142.8 46.0 4.61 4.7536 98
+45 1 20.3 74.33 190 126.2 49.0 3.88 4.3041 79
+47 1 30.4 120.0 199 120.0 46.0 4.0 5.1059 87
+46 1 20.6 73.0 172 107.0 51.0 3.0 4.2485 80
+36 2 32.3 115.0 286 199.4 39.0 7.0 5.4723 112
+34 1 29.2 73.0 172 108.2 49.0 4.0 4.3041 91
+53 2 33.1 117.0 183 119.0 48.0 4.0 4.382 106
+61 1 24.6 101.0 209 106.8 77.0 3.0 4.8363 88
+37 1 20.2 81.0 162 87.8 63.0 3.0 4.0254 88
+33 2 20.8 84.0 125 70.2 46.0 3.0 3.7842 66
+68 1 32.8 105.67 205 116.4 40.0 5.13 5.4931 117
+49 2 31.9 94.0 234 155.8 34.0 7.0 5.3982 122
+48 1 23.9 109.0 232 105.2 37.0 6.0 6.107 96
+55 2 24.5 84.0 179 105.8 66.0 3.0 3.5835 87
+43 1 22.1 66.0 134 77.2 45.0 3.0 4.0775 80
+60 2 33.0 97.0 217 125.6 45.0 5.0 5.4467 112
+31 2 19.0 93.0 137 73.0 47.0 3.0 4.4427 78
+53 2 27.3 82.0 119 55.0 39.0 3.0 4.8283 93
+67 1 22.8 87.0 166 98.6 52.0 3.0 4.3438 92
+61 2 28.2 106.0 204 132.0 52.0 4.0 4.6052 96
+62 1 28.9 87.33 206 127.2 33.0 6.24 5.4337 99
+60 1 25.6 87.0 207 125.8 69.0 3.0 4.1109 84
+42 1 24.9 91.0 204 141.8 38.0 5.0 4.7958 89
+38 2 26.8 105.0 181 119.2 37.0 5.0 4.8203 91
+62 1 22.4 79.0 222 147.4 59.0 4.0 4.3567 76
+61 2 26.9 111.0 236 172.4 39.0 6.0 4.8122 89
+61 2 23.1 113.0 186 114.4 47.0 4.0 4.8122 105
+53 1 28.6 88.0 171 98.8 41.0 4.0 5.0499 99
+28 2 24.7 97.0 175 99.6 32.0 5.0 5.3799 87
+26 2 30.3 89.0 218 152.2 31.0 7.0 5.1591 82
+30 1 21.3 87.0 134 63.0 63.0 2.0 3.6889 66
+50 1 26.1 109.0 243 160.6 62.0 4.0 4.625 89
+48 1 20.2 95.0 187 117.4 53.0 4.0 4.4188 85
+51 1 25.2 103.0 176 112.2 37.0 5.0 4.8978 90
+47 2 22.5 82.0 131 66.8 41.0 3.0 4.7536 89
+64 2 23.5 97.0 203 129.0 59.0 3.0 4.3175 77
+51 2 25.9 76.0 240 169.0 39.0 6.0 5.0752 96
+30 1 20.9 104.0 152 83.8 47.0 3.0 4.6634 97
+56 2 28.7 99.0 208 146.4 39.0 5.0 4.7274 97
+42 1 22.1 85.0 213 138.6 60.0 4.0 4.2767 94
+62 2 26.7 115.0 183 124.0 35.0 5.0 4.7875 100
+34 1 31.4 87.0 149 93.8 46.0 3.0 3.8286 77
+60 1 22.2 104.67 221 105.4 60.0 3.68 5.6276 93
+64 1 21.0 92.33 227 146.8 65.0 3.49 4.3307 102
+39 2 21.2 90.0 182 110.4 60.0 3.0 4.0604 98
+71 2 26.5 105.0 281 173.6 55.0 5.0 5.5683 84
+48 2 29.2 110.0 218 151.6 39.0 6.0 4.92 98
+79 2 27.0 103.0 169 110.8 37.0 5.0 4.6634 110
+40 1 30.7 99.0 177 85.4 50.0 4.0 5.3375 85
+49 2 28.8 92.0 207 140.0 44.0 5.0 4.7449 92
+51 1 30.6 103.0 198 106.6 57.0 3.0 5.1475 100
+57 1 30.1 117.0 202 139.6 42.0 5.0 4.625 120
+59 2 24.7 114.0 152 104.8 29.0 5.0 4.5109 88
+51 1 27.7 99.0 229 145.6 69.0 3.0 4.2767 77
+74 1 29.8 101.0 171 104.8 50.0 3.0 4.3944 86
+67 1 26.7 105.0 225 135.4 69.0 3.0 4.6347 96
+49 1 19.8 88.0 188 114.8 57.0 3.0 4.3944 93
+57 1 23.3 88.0 155 63.6 78.0 2.0 4.2047 78
+56 2 35.1 123.0 164 95.0 38.0 4.0 5.0434 117
+52 2 29.7 109.0 228 162.8 31.0 8.0 5.1417 103
+69 1 29.3 124.0 223 139.0 54.0 4.0 5.0106 102
+37 1 20.3 83.0 185 124.6 38.0 5.0 4.7185 88
+24 1 22.5 89.0 141 68.0 52.0 3.0 4.654 84
+55 2 22.7 93.0 154 94.2 53.0 3.0 3.5264 75
+36 1 22.8 87.0 178 116.0 41.0 4.0 4.654 82
+42 2 24.0 107.0 150 85.0 44.0 3.0 4.654 96
+21 1 24.2 76.0 147 77.0 53.0 3.0 4.4427 79
+41 1 20.2 62.0 153 89.0 50.0 3.0 4.2485 89
+57 2 29.4 109.0 160 87.6 31.0 5.0 5.3327 92
+20 2 22.1 87.0 171 99.6 58.0 3.0 4.2047 78
+67 2 23.6 111.33 189 105.4 70.0 2.7 4.2195 93
+34 1 25.2 77.0 189 120.6 53.0 4.0 4.3438 79
+41 2 24.9 86.0 192 115.0 61.0 3.0 4.382 94
+38 2 33.0 78.0 301 215.0 50.0 6.02 5.193 108
+51 1 23.5 101.0 195 121.0 51.0 4.0 4.7449 94
+52 2 26.4 91.33 218 152.0 39.0 5.59 4.9053 99
+67 1 29.8 80.0 172 93.4 63.0 3.0 4.3567 82
+61 1 30.0 108.0 194 100.0 52.0 3.73 5.3471 105
+67 2 25.0 111.67 146 93.4 33.0 4.42 4.585 103
+56 1 27.0 105.0 247 160.6 54.0 5.0 5.0876 94
+64 1 20.0 74.67 189 114.8 62.0 3.05 4.1109 91
+58 2 25.5 112.0 163 110.6 29.0 6.0 4.7622 86
+55 1 28.2 91.0 250 140.2 67.0 4.0 5.3660000000000005 103
+62 2 33.3 114.0 182 114.0 38.0 5.0 5.0106 96
+57 2 25.6 96.0 200 133.0 52.0 3.85 4.3175 105
+20 2 24.2 88.0 126 72.2 45.0 3.0 3.7842 74
+53 2 22.1 98.0 165 105.2 47.0 4.0 4.1589 81
+32 2 31.4 89.0 153 84.2 56.0 3.0 4.1589 90
+41 1 23.1 86.0 148 78.0 58.0 3.0 4.0943 60
+60 1 23.4 76.67 247 148.0 65.0 3.8 5.1358 77
+26 1 18.8 83.0 191 103.6 69.0 3.0 4.5218 69
+37 1 30.8 112.0 282 197.2 43.0 7.0 5.3423 101
+45 1 32.0 110.0 224 134.2 45.0 5.0 5.4116 93
+67 1 31.6 116.0 179 90.4 41.0 4.0 5.4723 100
+34 2 35.5 120.0 233 146.6 34.0 7.0 5.5683 101
+50 1 31.9 78.33 207 149.2 38.0 5.45 4.5951 84
+71 1 29.5 97.0 227 151.6 45.0 5.0 5.0239 108
+57 2 31.6 117.0 225 107.6 40.0 6.0 5.9584 113
+49 1 20.3 93.0 184 103.0 61.0 3.0 4.6052 93
+35 1 41.3 81.0 168 102.8 37.0 5.0 4.9488 94
+41 2 21.2 102.0 184 100.4 64.0 3.0 4.585 79
+70 2 24.1 82.33 194 149.2 31.0 6.26 4.2341 105
+52 1 23.0 107.0 179 123.7 42.5 4.21 4.1589 93
+60 1 25.6 78.0 195 95.4 91.0 2.0 3.7612 87
+62 1 22.5 125.0 215 99.0 98.0 2.0 4.4998 95
+44 2 38.2 123.0 201 126.6 44.0 5.0 5.0239 92
+28 2 19.2 81.0 155 94.6 51.0 3.0 3.8501 87
+58 2 29.0 85.0 156 109.2 36.0 4.0 3.989 86
+39 2 24.0 89.67 190 113.6 52.0 3.65 4.803999999999999 101
+34 2 20.6 98.0 183 92.0 83.0 2.0 3.6889 92
+65 1 26.3 70.0 244 166.2 51.0 5.0 4.8978 98
+66 2 34.6 115.0 204 139.4 36.0 6.0 4.9628 109
+51 1 23.4 87.0 220 108.8 93.0 2.0 4.5109 82
+50 2 29.2 119.0 162 85.2 54.0 3.0 4.7362 95
+59 2 27.2 107.0 158 102.0 39.0 4.0 4.4427 93
+52 1 27.0 78.33 134 73.0 44.0 3.05 4.4427 69
+69 2 24.5 108.0 243 136.4 40.0 6.0 5.8081 100
+53 1 24.1 105.0 184 113.4 46.0 4.0 4.8122 95
+47 2 25.3 98.0 173 105.6 44.0 4.0 4.7622 108
+52 1 28.8 113.0 280 174.0 67.0 4.0 5.273 86
+39 1 20.9 95.0 150 65.6 68.0 2.0 4.4067 95
+67 2 23.0 70.0 184 128.0 35.0 5.0 4.654 99
+59 2 24.1 96.0 170 98.6 54.0 3.0 4.4659 85
+51 2 28.1 106.0 202 122.2 55.0 4.0 4.8203 87
+23 2 18.0 78.0 171 96.0 48.0 4.0 4.9053 92
+68 1 25.9 93.0 253 181.2 53.0 5.0 4.5433 98
+44 1 21.5 85.0 157 92.2 55.0 3.0 3.8918 84
+60 2 24.3 103.0 141 86.6 33.0 4.0 4.6728 78
+52 1 24.5 90.0 198 129.0 29.0 7.0 5.2983 86
+38 1 21.3 72.0 165 60.2 88.0 2.0 4.4308 90
+61 1 25.8 90.0 280 195.4 55.0 5.0 4.9972 90
+68 2 24.8 101.0 221 151.4 60.0 4.0 3.8712 87
+28 2 31.5 83.0 228 149.4 38.0 6.0 5.3132 83
+65 2 33.5 102.0 190 126.2 35.0 5.0 4.9698 102
+69 1 28.1 113.0 234 142.8 52.0 4.0 5.2781 77
+51 1 24.3 85.33 153 71.6 71.0 2.15 3.9512 82
+29 1 35.0 98.33 204 142.6 50.0 4.08 4.0431 91
+55 2 23.5 93.0 177 126.8 41.0 4.0 3.8286 83
+34 2 30.0 83.0 185 107.2 53.0 3.0 4.8203 92
+67 1 20.7 83.0 170 99.8 59.0 3.0 4.0254 77
+49 1 25.6 76.0 161 99.8 51.0 3.0 3.9318 78
+55 2 22.9 81.0 123 67.2 41.0 3.0 4.3041 88
+59 2 25.1 90.0 163 101.4 46.0 4.0 4.3567 91
+53 1 33.2 82.67 186 106.8 46.0 4.04 5.112 102
+48 2 24.1 110.0 209 134.6 58.0 4.0 4.4067 100
+52 1 29.5 104.33 211 132.8 49.0 4.31 4.9836 98
+69 1 29.6 122.0 231 128.4 56.0 4.0 5.4510000000000005 86
+60 2 22.8 110.0 245 189.8 39.0 6.0 4.3944 88
+46 2 22.7 83.0 183 125.8 32.0 6.0 4.8363 75
+51 2 26.2 101.0 161 99.6 48.0 3.0 4.2047 88
+67 2 23.5 96.0 207 138.2 42.0 5.0 4.8978 111
+49 1 22.1 85.0 136 63.4 62.0 2.19 3.9703 72
+46 2 26.5 94.0 247 160.2 59.0 4.0 4.9345 111
+47 1 32.4 105.0 188 125.0 46.0 4.09 4.4427 99
+75 1 30.1 78.0 222 154.2 44.0 5.05 4.7791 97
+28 1 24.2 93.0 174 106.4 54.0 3.0 4.2195 84
+65 2 31.3 110.0 213 128.0 47.0 5.0 5.247000000000001 91
+42 1 30.1 91.0 182 114.8 49.0 4.0 4.5109 82
+51 1 24.5 79.0 212 128.6 65.0 3.0 4.5218 91
+53 2 27.7 95.0 190 101.8 41.0 5.0 5.4638 101
+54 1 23.2 110.67 238 162.8 48.0 4.96 4.9127 108
+73 1 27.0 102.0 211 121.0 67.0 3.0 4.7449 99
+54 1 26.8 108.0 176 80.6 67.0 3.0 4.9558 106
+42 1 29.2 93.0 249 174.2 45.0 6.0 5.0039 92
+75 1 31.2 117.67 229 138.8 29.0 7.9 5.7236 106
+55 2 32.1 112.67 207 92.4 25.0 8.28 6.1048 111
+68 2 25.7 109.0 233 112.6 35.0 7.0 6.0568 105
+57 1 26.9 98.0 246 165.2 38.0 7.0 5.3660000000000005 96
+48 1 31.4 75.33 242 151.6 38.0 6.37 5.5683 103
+61 2 25.6 85.0 184 116.2 39.0 5.0 4.9698 98
+69 1 37.0 103.0 207 131.4 55.0 4.0 4.6347 90
+38 1 32.6 77.0 168 100.6 47.0 4.0 4.625 96
+45 2 21.2 94.0 169 96.8 55.0 3.0 4.4543 102
+51 2 29.2 107.0 187 139.0 32.0 6.0 4.382 95
+71 2 24.0 84.0 138 85.8 39.0 4.0 4.1897 90
+57 1 36.1 117.0 181 108.2 34.0 5.0 5.2679 100
+56 2 25.8 103.0 177 114.4 34.0 5.0 4.9628 99
+32 2 22.0 88.0 137 78.6 48.0 3.0 3.9512 78
+50 1 21.9 91.0 190 111.2 67.0 3.0 4.0775 77
+43 1 34.3 84.0 256 172.6 33.0 8.0 5.5294 104
+54 2 25.2 115.0 181 120.0 39.0 5.0 4.7005 92
+31 1 23.3 85.0 190 130.8 43.0 4.0 4.3944 77
+56 1 25.7 80.0 244 151.6 59.0 4.0 5.118 95
+44 1 25.1 133.0 182 113.0 55.0 3.0 4.2485 84
+57 2 31.9 111.0 173 116.2 41.0 4.0 4.3694 87
+64 2 28.4 111.0 184 127.0 41.0 4.0 4.382 97
+43 1 28.1 121.0 192 121.0 60.0 3.0 4.0073 93
+19 1 25.3 83.0 225 156.6 46.0 5.0 4.7185 84
+71 2 26.1 85.0 220 152.4 47.0 5.0 4.6347 91
+50 2 28.0 104.0 282 196.8 44.0 6.0 5.3279 95
+59 2 23.6 73.0 180 107.4 51.0 4.0 4.6821 84
+57 1 24.5 93.0 186 96.6 71.0 3.0 4.5218 91
+49 2 21.0 82.0 119 85.4 23.0 5.0 3.9703 74
+41 2 32.0 126.0 198 104.2 49.0 4.0 5.4116 124
+25 2 22.6 85.0 130 71.0 48.0 3.0 4.0073 81
+52 2 19.7 81.0 152 53.4 82.0 2.0 4.4188 82
+34 1 21.2 84.0 254 113.4 52.0 5.0 6.0936 92
+42 2 30.6 101.0 269 172.2 50.0 5.0 5.4553 106
+28 2 25.5 99.0 162 101.6 46.0 4.0 4.2767 94
+47 2 23.3 90.0 195 125.8 54.0 4.0 4.3307 73
+32 2 31.0 100.0 177 96.2 45.0 4.0 5.1874 77
+43 1 18.5 87.0 163 93.6 61.0 2.67 3.7377 80
+59 2 26.9 104.0 194 126.6 43.0 5.0 4.803999999999999 106
+53 1 28.3 101.0 179 107.0 48.0 4.0 4.7875 101
+60 1 25.7 103.0 158 84.6 64.0 2.0 3.8501 97
+54 2 36.1 115.0 163 98.4 43.0 4.0 4.6821 101
+35 2 24.1 94.67 155 97.4 32.0 4.84 4.852 94
+49 2 25.8 89.0 182 118.6 39.0 5.0 4.803999999999999 115
+58 1 22.8 91.0 196 118.8 48.0 4.0 4.9836 115
+36 2 39.1 90.0 219 135.8 38.0 6.0 5.4205 103
+46 2 42.2 99.0 211 137.0 44.0 5.0 5.0106 99
+44 2 26.6 99.0 205 109.0 43.0 5.0 5.5797 111
+46 1 29.9 83.0 171 113.0 38.0 4.5 4.585 98
+54 1 21.0 78.0 188 107.4 70.0 3.0 3.9703 73
+63 2 25.5 109.0 226 103.2 46.0 5.0 5.9506 87
+41 2 24.2 90.0 199 123.6 57.0 4.0 4.5218 86
+28 1 25.4 93.0 141 79.0 49.0 3.0 4.1744 91
+19 1 23.2 75.0 143 70.4 52.0 3.0 4.6347 72
+61 2 26.1 126.0 215 129.8 57.0 4.0 4.9488 96
+48 1 32.7 93.0 276 198.6 43.0 6.42 5.1475 91
+54 2 27.3 100.0 200 144.0 33.0 6.0 4.7449 76
+53 2 26.6 93.0 185 122.4 36.0 5.0 4.8903 82
+48 1 22.8 101.0 110 41.6 56.0 2.0 4.1271 97
+53 1 28.8 111.67 145 87.2 46.0 3.15 4.0775 85
+29 2 18.1 73.0 158 99.0 41.0 4.0 4.4998 78
+62 1 32.0 88.0 172 69.0 38.0 4.0 5.7838 100
+50 2 23.7 92.0 166 97.0 52.0 3.0 4.4427 93
+58 2 23.6 96.0 257 171.0 59.0 4.0 4.9053 82
+55 2 24.6 109.0 143 76.4 51.0 3.0 4.3567 88
+54 1 22.6 90.0 183 104.2 64.0 3.0 4.3041 92
+36 1 27.8 73.0 153 104.4 42.0 4.0 3.4965 73
+63 2 24.1 111.0 184 112.2 44.0 4.0 4.9345 82
+47 2 26.5 70.0 181 104.8 63.0 3.0 4.1897 70
+51 2 32.8 112.0 202 100.6 37.0 5.0 5.7746 109
+42 1 19.9 76.0 146 83.2 55.0 3.0 3.6636 79
+37 2 23.6 94.0 205 138.8 53.0 4.0 4.1897 107
+28 1 22.1 82.0 168 100.6 54.0 3.0 4.2047 86
+58 1 28.1 111.0 198 80.6 31.0 6.0 6.0684 93
+32 1 26.5 86.0 184 101.6 53.0 4.0 4.9904 78
+25 2 23.5 88.0 143 80.8 55.0 3.0 3.5835 83
+63 1 26.0 85.67 155 78.2 46.0 3.37 5.037 97
+52 1 27.8 85.0 219 136.0 49.0 4.0 5.1358 75
+65 2 28.5 109.0 201 123.0 46.0 4.0 5.0752 96
+42 1 30.6 121.0 176 92.8 69.0 3.0 4.2627 89
+53 1 22.2 78.0 164 81.0 70.0 2.0 4.1744 101
+79 2 23.3 88.0 186 128.4 33.0 6.0 4.8122 102
+43 1 35.4 93.0 185 100.2 44.0 4.0 5.3181 101
+44 1 31.4 115.0 165 97.6 52.0 3.0 4.3438 89
+62 2 37.8 119.0 113 51.0 31.0 4.0 5.0434 84
+33 1 18.9 70.0 162 91.8 59.0 3.0 4.0254 58
+56 1 35.0 79.33 195 140.8 42.0 4.64 4.1109 96
+66 1 21.7 126.0 212 127.8 45.0 4.71 5.2781 101
+34 2 25.3 111.0 230 162.0 39.0 6.0 4.9767 90
+46 2 23.8 97.0 224 139.2 42.0 5.0 5.3660000000000005 81
+50 1 31.8 82.0 136 69.2 55.0 2.0 4.0775 85
+69 1 34.3 113.0 200 123.8 54.0 4.0 4.7095 112
+34 1 26.3 87.0 197 120.0 63.0 3.0 4.2485 96
+71 2 27.0 93.33 269 190.2 41.0 6.56 5.2417 93
+47 1 27.2 80.0 208 145.6 38.0 6.0 4.803999999999999 92
+41 1 33.8 123.33 187 127.0 45.0 4.16 4.3175 100
+34 1 33.0 73.0 178 114.6 51.0 3.49 4.1271 92
+51 1 24.1 87.0 261 175.6 69.0 4.0 4.4067 93
+43 1 21.3 79.0 141 78.8 53.0 3.0 3.8286 90
+55 1 23.0 94.67 190 137.6 38.0 5.0 4.2767 106
+59 2 27.9 101.0 218 144.2 38.0 6.0 5.1874 95
+27 2 33.6 110.0 246 156.6 57.0 4.0 5.0876 89
+51 2 22.7 103.0 217 162.4 30.0 7.0 4.8122 80
+49 2 27.4 89.0 177 113.0 37.0 5.0 4.9053 97
+27 1 22.6 71.0 116 43.4 56.0 2.0 4.4188 79
+57 2 23.2 107.33 231 159.4 41.0 5.63 5.0304 112
+39 2 26.9 93.0 136 75.4 48.0 3.0 4.1431 99
+62 2 34.6 120.0 215 129.2 43.0 5.0 5.3660000000000005 123
+37 1 23.3 88.0 223 142.0 65.0 3.4 4.3567 82
+46 1 21.1 80.0 205 144.4 42.0 5.0 4.5326 87
+68 2 23.5 101.0 162 85.4 59.0 3.0 4.4773 91
+51 1 31.5 93.0 231 144.0 49.0 4.7 5.2523 117
+41 1 20.8 86.0 223 128.2 83.0 3.0 4.0775 89
+53 1 26.5 97.0 193 122.4 58.0 3.0 4.1431 99
+45 1 24.2 83.0 177 118.4 45.0 4.0 4.2195 82
+33 1 19.5 80.0 171 85.4 75.0 2.0 3.9703 80
+60 2 28.2 112.0 185 113.8 42.0 4.0 4.9836 93
+47 2 24.9 75.0 225 166.0 42.0 5.0 4.4427 102
+60 2 24.9 99.67 162 106.6 43.0 3.77 4.1271 95
+36 1 30.0 95.0 201 125.2 42.0 4.79 5.1299 85
+36 1 19.6 71.0 250 133.2 97.0 3.0 4.5951 92
\ No newline at end of file
diff --git a/test/support/diabetes_target.csv b/test/support/diabetes_target.csv
new file mode 100644
index 00000000..ce5e4a5e
--- /dev/null
+++ b/test/support/diabetes_target.csv
@@ -0,0 +1,442 @@
+1.510000000000000000e+02
+7.500000000000000000e+01
+1.410000000000000000e+02
+2.060000000000000000e+02
+1.350000000000000000e+02
+9.700000000000000000e+01
+1.380000000000000000e+02
+6.300000000000000000e+01
+1.100000000000000000e+02
+3.100000000000000000e+02
+1.010000000000000000e+02
+6.900000000000000000e+01
+1.790000000000000000e+02
+1.850000000000000000e+02
+1.180000000000000000e+02
+1.710000000000000000e+02
+1.660000000000000000e+02
+1.440000000000000000e+02
+9.700000000000000000e+01
+1.680000000000000000e+02
+6.800000000000000000e+01
+4.900000000000000000e+01
+6.800000000000000000e+01
+2.450000000000000000e+02
+1.840000000000000000e+02
+2.020000000000000000e+02
+1.370000000000000000e+02
+8.500000000000000000e+01
+1.310000000000000000e+02
+2.830000000000000000e+02
+1.290000000000000000e+02
+5.900000000000000000e+01
+3.410000000000000000e+02
+8.700000000000000000e+01
+6.500000000000000000e+01
+1.020000000000000000e+02
+2.650000000000000000e+02
+2.760000000000000000e+02
+2.520000000000000000e+02
+9.000000000000000000e+01
+1.000000000000000000e+02
+5.500000000000000000e+01
+6.100000000000000000e+01
+9.200000000000000000e+01
+2.590000000000000000e+02
+5.300000000000000000e+01
+1.900000000000000000e+02
+1.420000000000000000e+02
+7.500000000000000000e+01
+1.420000000000000000e+02
+1.550000000000000000e+02
+2.250000000000000000e+02
+5.900000000000000000e+01
+1.040000000000000000e+02
+1.820000000000000000e+02
+1.280000000000000000e+02
+5.200000000000000000e+01
+3.700000000000000000e+01
+1.700000000000000000e+02
+1.700000000000000000e+02
+6.100000000000000000e+01
+1.440000000000000000e+02
+5.200000000000000000e+01
+1.280000000000000000e+02
+7.100000000000000000e+01
+1.630000000000000000e+02
+1.500000000000000000e+02
+9.700000000000000000e+01
+1.600000000000000000e+02
+1.780000000000000000e+02
+4.800000000000000000e+01
+2.700000000000000000e+02
+2.020000000000000000e+02
+1.110000000000000000e+02
+8.500000000000000000e+01
+4.200000000000000000e+01
+1.700000000000000000e+02
+2.000000000000000000e+02
+2.520000000000000000e+02
+1.130000000000000000e+02
+1.430000000000000000e+02
+5.100000000000000000e+01
+5.200000000000000000e+01
+2.100000000000000000e+02
+6.500000000000000000e+01
+1.410000000000000000e+02
+5.500000000000000000e+01
+1.340000000000000000e+02
+4.200000000000000000e+01
+1.110000000000000000e+02
+9.800000000000000000e+01
+1.640000000000000000e+02
+4.800000000000000000e+01
+9.600000000000000000e+01
+9.000000000000000000e+01
+1.620000000000000000e+02
+1.500000000000000000e+02
+2.790000000000000000e+02
+9.200000000000000000e+01
+8.300000000000000000e+01
+1.280000000000000000e+02
+1.020000000000000000e+02
+3.020000000000000000e+02
+1.980000000000000000e+02
+9.500000000000000000e+01
+5.300000000000000000e+01
+1.340000000000000000e+02
+1.440000000000000000e+02
+2.320000000000000000e+02
+8.100000000000000000e+01
+1.040000000000000000e+02
+5.900000000000000000e+01
+2.460000000000000000e+02
+2.970000000000000000e+02
+2.580000000000000000e+02
+2.290000000000000000e+02
+2.750000000000000000e+02
+2.810000000000000000e+02
+1.790000000000000000e+02
+2.000000000000000000e+02
+2.000000000000000000e+02
+1.730000000000000000e+02
+1.800000000000000000e+02
+8.400000000000000000e+01
+1.210000000000000000e+02
+1.610000000000000000e+02
+9.900000000000000000e+01
+1.090000000000000000e+02
+1.150000000000000000e+02
+2.680000000000000000e+02
+2.740000000000000000e+02
+1.580000000000000000e+02
+1.070000000000000000e+02
+8.300000000000000000e+01
+1.030000000000000000e+02
+2.720000000000000000e+02
+8.500000000000000000e+01
+2.800000000000000000e+02
+3.360000000000000000e+02
+2.810000000000000000e+02
+1.180000000000000000e+02
+3.170000000000000000e+02
+2.350000000000000000e+02
+6.000000000000000000e+01
+1.740000000000000000e+02
+2.590000000000000000e+02
+1.780000000000000000e+02
+1.280000000000000000e+02
+9.600000000000000000e+01
+1.260000000000000000e+02
+2.880000000000000000e+02
+8.800000000000000000e+01
+2.920000000000000000e+02
+7.100000000000000000e+01
+1.970000000000000000e+02
+1.860000000000000000e+02
+2.500000000000000000e+01
+8.400000000000000000e+01
+9.600000000000000000e+01
+1.950000000000000000e+02
+5.300000000000000000e+01
+2.170000000000000000e+02
+1.720000000000000000e+02
+1.310000000000000000e+02
+2.140000000000000000e+02
+5.900000000000000000e+01
+7.000000000000000000e+01
+2.200000000000000000e+02
+2.680000000000000000e+02
+1.520000000000000000e+02
+4.700000000000000000e+01
+7.400000000000000000e+01
+2.950000000000000000e+02
+1.010000000000000000e+02
+1.510000000000000000e+02
+1.270000000000000000e+02
+2.370000000000000000e+02
+2.250000000000000000e+02
+8.100000000000000000e+01
+1.510000000000000000e+02
+1.070000000000000000e+02
+6.400000000000000000e+01
+1.380000000000000000e+02
+1.850000000000000000e+02
+2.650000000000000000e+02
+1.010000000000000000e+02
+1.370000000000000000e+02
+1.430000000000000000e+02
+1.410000000000000000e+02
+7.900000000000000000e+01
+2.920000000000000000e+02
+1.780000000000000000e+02
+9.100000000000000000e+01
+1.160000000000000000e+02
+8.600000000000000000e+01
+1.220000000000000000e+02
+7.200000000000000000e+01
+1.290000000000000000e+02
+1.420000000000000000e+02
+9.000000000000000000e+01
+1.580000000000000000e+02
+3.900000000000000000e+01
+1.960000000000000000e+02
+2.220000000000000000e+02
+2.770000000000000000e+02
+9.900000000000000000e+01
+1.960000000000000000e+02
+2.020000000000000000e+02
+1.550000000000000000e+02
+7.700000000000000000e+01
+1.910000000000000000e+02
+7.000000000000000000e+01
+7.300000000000000000e+01
+4.900000000000000000e+01
+6.500000000000000000e+01
+2.630000000000000000e+02
+2.480000000000000000e+02
+2.960000000000000000e+02
+2.140000000000000000e+02
+1.850000000000000000e+02
+7.800000000000000000e+01
+9.300000000000000000e+01
+2.520000000000000000e+02
+1.500000000000000000e+02
+7.700000000000000000e+01
+2.080000000000000000e+02
+7.700000000000000000e+01
+1.080000000000000000e+02
+1.600000000000000000e+02
+5.300000000000000000e+01
+2.200000000000000000e+02
+1.540000000000000000e+02
+2.590000000000000000e+02
+9.000000000000000000e+01
+2.460000000000000000e+02
+1.240000000000000000e+02
+6.700000000000000000e+01
+7.200000000000000000e+01
+2.570000000000000000e+02
+2.620000000000000000e+02
+2.750000000000000000e+02
+1.770000000000000000e+02
+7.100000000000000000e+01
+4.700000000000000000e+01
+1.870000000000000000e+02
+1.250000000000000000e+02
+7.800000000000000000e+01
+5.100000000000000000e+01
+2.580000000000000000e+02
+2.150000000000000000e+02
+3.030000000000000000e+02
+2.430000000000000000e+02
+9.100000000000000000e+01
+1.500000000000000000e+02
+3.100000000000000000e+02
+1.530000000000000000e+02
+3.460000000000000000e+02
+6.300000000000000000e+01
+8.900000000000000000e+01
+5.000000000000000000e+01
+3.900000000000000000e+01
+1.030000000000000000e+02
+3.080000000000000000e+02
+1.160000000000000000e+02
+1.450000000000000000e+02
+7.400000000000000000e+01
+4.500000000000000000e+01
+1.150000000000000000e+02
+2.640000000000000000e+02
+8.700000000000000000e+01
+2.020000000000000000e+02
+1.270000000000000000e+02
+1.820000000000000000e+02
+2.410000000000000000e+02
+6.600000000000000000e+01
+9.400000000000000000e+01
+2.830000000000000000e+02
+6.400000000000000000e+01
+1.020000000000000000e+02
+2.000000000000000000e+02
+2.650000000000000000e+02
+9.400000000000000000e+01
+2.300000000000000000e+02
+1.810000000000000000e+02
+1.560000000000000000e+02
+2.330000000000000000e+02
+6.000000000000000000e+01
+2.190000000000000000e+02
+8.000000000000000000e+01
+6.800000000000000000e+01
+3.320000000000000000e+02
+2.480000000000000000e+02
+8.400000000000000000e+01
+2.000000000000000000e+02
+5.500000000000000000e+01
+8.500000000000000000e+01
+8.900000000000000000e+01
+3.100000000000000000e+01
+1.290000000000000000e+02
+8.300000000000000000e+01
+2.750000000000000000e+02
+6.500000000000000000e+01
+1.980000000000000000e+02
+2.360000000000000000e+02
+2.530000000000000000e+02
+1.240000000000000000e+02
+4.400000000000000000e+01
+1.720000000000000000e+02
+1.140000000000000000e+02
+1.420000000000000000e+02
+1.090000000000000000e+02
+1.800000000000000000e+02
+1.440000000000000000e+02
+1.630000000000000000e+02
+1.470000000000000000e+02
+9.700000000000000000e+01
+2.200000000000000000e+02
+1.900000000000000000e+02
+1.090000000000000000e+02
+1.910000000000000000e+02
+1.220000000000000000e+02
+2.300000000000000000e+02
+2.420000000000000000e+02
+2.480000000000000000e+02
+2.490000000000000000e+02
+1.920000000000000000e+02
+1.310000000000000000e+02
+2.370000000000000000e+02
+7.800000000000000000e+01
+1.350000000000000000e+02
+2.440000000000000000e+02
+1.990000000000000000e+02
+2.700000000000000000e+02
+1.640000000000000000e+02
+7.200000000000000000e+01
+9.600000000000000000e+01
+3.060000000000000000e+02
+9.100000000000000000e+01
+2.140000000000000000e+02
+9.500000000000000000e+01
+2.160000000000000000e+02
+2.630000000000000000e+02
+1.780000000000000000e+02
+1.130000000000000000e+02
+2.000000000000000000e+02
+1.390000000000000000e+02
+1.390000000000000000e+02
+8.800000000000000000e+01
+1.480000000000000000e+02
+8.800000000000000000e+01
+2.430000000000000000e+02
+7.100000000000000000e+01
+7.700000000000000000e+01
+1.090000000000000000e+02
+2.720000000000000000e+02
+6.000000000000000000e+01
+5.400000000000000000e+01
+2.210000000000000000e+02
+9.000000000000000000e+01
+3.110000000000000000e+02
+2.810000000000000000e+02
+1.820000000000000000e+02
+3.210000000000000000e+02
+5.800000000000000000e+01
+2.620000000000000000e+02
+2.060000000000000000e+02
+2.330000000000000000e+02
+2.420000000000000000e+02
+1.230000000000000000e+02
+1.670000000000000000e+02
+6.300000000000000000e+01
+1.970000000000000000e+02
+7.100000000000000000e+01
+1.680000000000000000e+02
+1.400000000000000000e+02
+2.170000000000000000e+02
+1.210000000000000000e+02
+2.350000000000000000e+02
+2.450000000000000000e+02
+4.000000000000000000e+01
+5.200000000000000000e+01
+1.040000000000000000e+02
+1.320000000000000000e+02
+8.800000000000000000e+01
+6.900000000000000000e+01
+2.190000000000000000e+02
+7.200000000000000000e+01
+2.010000000000000000e+02
+1.100000000000000000e+02
+5.100000000000000000e+01
+2.770000000000000000e+02
+6.300000000000000000e+01
+1.180000000000000000e+02
+6.900000000000000000e+01
+2.730000000000000000e+02
+2.580000000000000000e+02
+4.300000000000000000e+01
+1.980000000000000000e+02
+2.420000000000000000e+02
+2.320000000000000000e+02
+1.750000000000000000e+02
+9.300000000000000000e+01
+1.680000000000000000e+02
+2.750000000000000000e+02
+2.930000000000000000e+02
+2.810000000000000000e+02
+7.200000000000000000e+01
+1.400000000000000000e+02
+1.890000000000000000e+02
+1.810000000000000000e+02
+2.090000000000000000e+02
+1.360000000000000000e+02
+2.610000000000000000e+02
+1.130000000000000000e+02
+1.310000000000000000e+02
+1.740000000000000000e+02
+2.570000000000000000e+02
+5.500000000000000000e+01
+8.400000000000000000e+01
+4.200000000000000000e+01
+1.460000000000000000e+02
+2.120000000000000000e+02
+2.330000000000000000e+02
+9.100000000000000000e+01
+1.110000000000000000e+02
+1.520000000000000000e+02
+1.200000000000000000e+02
+6.700000000000000000e+01
+3.100000000000000000e+02
+9.400000000000000000e+01
+1.830000000000000000e+02
+6.600000000000000000e+01
+1.730000000000000000e+02
+7.200000000000000000e+01
+4.900000000000000000e+01
+6.400000000000000000e+01
+4.800000000000000000e+01
+1.780000000000000000e+02
+1.040000000000000000e+02
+1.320000000000000000e+02
+2.200000000000000000e+02
+5.700000000000000000e+01
\ No newline at end of file
diff --git a/test/support/scholar_case.ex b/test/support/scholar_case.ex
index 62ba6ba5..2d355249 100644
--- a/test/support/scholar_case.ex
+++ b/test/support/scholar_case.ex
@@ -194,4 +194,33 @@ defmodule Scholar.Case do
     {y_train, y_test} = Nx.split(y, 120)
     {x_train, x_test, y_train, y_test}
   end
+
+  def diabetes_data do
+    x =
+      File.read!("./test/support/diabetes_data_raw.csv")
+      |> String.split("\n")
+      |> Enum.map(&String.split(&1, " "))
+      |> Enum.map(&list_of_string_to_floats(&1))
+      |> Nx.tensor()
+
+    y =
+      File.read!("./test/support/diabetes_target.csv")
+      |> String.split("\n")
+      |> Enum.map(&String.split(&1, " "))
+      |> Enum.map(&list_of_string_to_floats(&1))
+      |> Nx.tensor()
+
+    {x, y}
+  end
+
+  defp list_of_string_to_floats(list) do
+    list
+    |> Enum.map(&Float.parse/1)
+    |> Enum.map(&keep_floats/1)
+  end
+
+  defp keep_floats(parsed_float) do
+    {float, _} = parsed_float
+    float
+  end
 end

From e4cd5b091bd424062bd575ef625a7d57bf9efd3e Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sat, 30 Mar 2024 12:23:41 +0100
Subject: [PATCH 50/90] add test using diabetes data

---
 .../linear/bayesian_ridge_regression_test.exs | 52 +++++++++----------
 1 file changed, 25 insertions(+), 27 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 26ffe204..207b158e 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -1,5 +1,5 @@
 defmodule Scholar.Linear.BayesianRidgeRegressionTest do
-  import Nx.Defn  
+  import Nx.Defn
   use Scholar.Case, async: true
   alias Scholar.Linear.BayesianRidgeRegression
   alias Scholar.Linear.RidgeRegression
@@ -34,10 +34,11 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
 
-  @tag :wip
   test "compute scores" do
-    {x, y, noise} = data()
-
+    {x, y} = diabetes_data()
+    n_samples = 50 - 1
+    x = x[[0..n_samples, ..]]
+    y = y[[0..n_samples, ..]]
     eps = Nx.Constants.smallest_positive_normal({:f, 64})
     alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
     lambda = Nx.tensor(1.0)
@@ -47,35 +48,32 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     lambda_2 = 0.1
     # compute score
     score = compute_score(x, y, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2)
-    IO.inspect(score)
-    brr = BayesianRidgeRegression.fit(x, y,
-      alpha_1: alpha_1, alpha_2: alpha_2, lambda_1: lambda_1, lambda_2: lambda_2,
-      fit_intercept?: false, iterations: 1)
-    assert false
+    brr =
+      BayesianRidgeRegression.fit(x, y,
+        alpha_1: alpha_1,
+        alpha_2: alpha_2,
+        lambda_1: lambda_1,
+        lambda_2: lambda_2,
+        fit_intercept?: true,
+        iterations: 1
+      )
+    compare_scores = Nx.divide(Nx.subtract(score, brr.score), score)
+    check = Nx.less(compare_scores, 0.05) |> Nx.flatten()
+    assert check == Nx.tensor([1], type: {:u, 8})
   end
 
   defnp compute_score(x, y, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2) do
     {n_samples, _} = Nx.shape(x)
-    lambda_score = lambda_1 + Nx.log(lambda) - lambda_2 * lambda
-    alpha_score =  alpha_1 + Nx.log(alpha_2) - alpha_2 * alpha
-    m =  (1.0 / alpha) * Nx.eye(n_samples) + (1.0 / lambda) * Nx.dot(x, Nx.transpose(x))
+    lambda_score = lambda_1 * Nx.log(lambda) - lambda_2 * lambda
+    alpha_score = alpha_1 * Nx.log(alpha) - alpha_2 * alpha
+    m = (1.0 / alpha * Nx.eye(n_samples)) + (1.0 / lambda * Nx.dot(x, Nx.transpose(x)))
     m_inv_dot_y = Nx.LinAlg.solve(m, y)
-    logdet = m |> Nx.LinAlg.determinant |> Nx.log()
-    y_score = -0.5 * (logdet + Nx.dot(y, m_inv_dot_y) +
-                        n_samples * Nx.log(2 * Nx.Constants.pi()))
-    score_alpha + score_lambda + y_score
-  end
+    logdet = m |> Nx.LinAlg.determinant() |> Nx.log()
+
+    y_score =
+      -0.5 * (logdet + Nx.dot(Nx.transpose(y), m_inv_dot_y) + n_samples * Nx.log(2 * Nx.Constants.pi()))
 
-  # adapted from the linear regression notebook
-  # https://hexdocs.pm/scholar/linear_regression.html
-  defnp data do
-    key = Nx.Random.key(42)
-    size = 30
-    {x, new_key} = Nx.Random.normal(key, 0, 2, shape: {size, 3}, type: :f64)
-    {noise, _} = Nx.Random.normal(new_key, 0, 1, shape: {size}, type: :f64)
-    w = Nx.tensor([1, 2, 3])
-    y = Nx.dot(x, w) + 4
-    {x, y, noise}
+    alpha_score + lambda_score + y_score
   end
 
   test "constant inputs: prediction" do

From f7b9a5760a7d793d91e6b90b810db312722b41f7 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sat, 30 Mar 2024 12:23:53 +0100
Subject: [PATCH 51/90] fix score computation

---
 .../linear/bayesian_ridge_regression.ex       | 75 ++++++++++++-------
 1 file changed, 46 insertions(+), 29 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 918589e1..c4dfba1a 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -4,8 +4,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   import Scholar.Shared
 
   @derive {Nx.Container,
-           containers: [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations]}
-  defstruct [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations]
+           containers: [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations, :score]}
+  defstruct [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations, :score]
 
   opts = [
     iterations: [
@@ -113,9 +113,10 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       if Nx.is_tensor(sample_weights),
         do: Nx.as_type(sample_weights, x_type),
         else: Nx.tensor(sample_weights, type: x_type)
+
     # handle vector types
     # handle default alpha value, add eps to avoid division by 0
-    eps = Nx.Constants.smallest_positive_normal({:f, 64})
+    eps = Nx.Constants.smallest_positive_normal(x_type)
     default_alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
     alpha = Keyword.get(opts, :alpha_init, default_alpha)
     alpha = Nx.tensor(alpha, type: x_type)
@@ -125,7 +126,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     lambda = Nx.tensor(lambda, type: x_type)
     opts = Keyword.put(opts, :lambda_init, lambda)
 
-    {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged} =
+    {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged, score} =
       fit_n(x, y, sample_weights, opts)
 
     if Nx.to_number(has_converged) == 1 do
@@ -138,7 +139,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       alpha: Nx.to_number(alpha),
       lambda: Nx.to_number(lambda),
       rmse: Nx.to_number(rmse),
-      iterations: Nx.to_number(iterations)
+      iterations: Nx.to_number(iterations),
+      score: score
     }
   end
 
@@ -166,6 +168,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       else
         {x, y}
       end
+
     alpha = opts[:alpha_init]
     lambda = opts[:lambda_init]
 
@@ -181,40 +184,31 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     eigenvals = Nx.pow(s, 2)
     {n_samples, n_features} = Nx.shape(x)
     {coef, rmse} = update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
-
-    {{coef, alpha, lambda, rmse, iter, has_converged}, _} =
-      while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0)},
+    score =
+      log_marginal_likelihood(
+        coef, rmse, n_samples, n_features, eigenvals, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2)
+    {{coef, alpha, lambda, rmse, iter, has_converged, score}, _} =
+      while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0), score = score},
              {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}},
             iter < iterations and not has_converged do
-        gamma =
-          (alpha * eigenvals / (lambda + alpha * eigenvals))
-          |> Nx.sum()
 
+        gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
         lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
         alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
-
         {coef_new, rmse} =
-          update_coef(
-            x,
-            y,
-            n_samples,
-            n_features,
-            xt_y,
-            u,
-            vh,
-            eigenvals,
-            alpha,
-            lambda
-          )
-
+          update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
+        score =
+          log_marginal_likelihood(
+            coef, rmse, n_samples, n_features, eigenvals,
+            alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2)
         has_converged = Nx.sum(Nx.abs(coef - coef_new)) < 1.0e-8
 
-        {{coef_new, alpha, lambda, rmse, iter + 1, has_converged},
+        {{coef_new, alpha, lambda, rmse, iter + 1, has_converged, score},
          {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}}
       end
-
+    
     intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
-    {coef, intercept, alpha, lambda, rmse, iter, has_converged}
+    {coef, intercept, alpha, lambda, rmse, iter, has_converged, score}
   end
 
   defn update_coef(
@@ -241,6 +235,29 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {coef, rmse}
   end
 
+  defn log_marginal_likelihood(
+         coef,
+         rmse,
+         n_samples,
+         n_features,
+         eigenvals,
+         alpha,
+         lambda,
+         alpha_1,
+         alpha_2,
+         lambda_1,
+         lambda_2
+       ) do
+    logdet_sigma = -1 * Nx.sum(Nx.log(lambda + alpha * eigenvals))
+    score_lambda = lambda_1 * Nx.log(lambda) - lambda_2 * lambda
+    score_alpha = alpha_1 * Nx.log(alpha) - alpha_2 * alpha
+    score_parameters =
+      n_features * Nx.log(lambda) + n_samples * Nx.log(alpha) - alpha * rmse - lambda * Nx.sum(coef ** 2)
+    score =
+        0.5 * (score_parameters + logdet_sigma  - n_samples * Nx.log(2 * Nx.Constants.pi()))
+    score_alpha + score_lambda + score
+  end
+
   defn predict(%__MODULE__{coefficients: coeff, intercept: intercept} = _model, x) do
     Nx.dot(x, [-1], coeff, [-1]) + intercept
   end
@@ -261,7 +278,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
 
   defnp set_intercept(coeff, x_offset, y_offset, fit_intercept?) do
     if fit_intercept? do
-      y_offset - Nx.dot(coeff, x_offset)
+      y_offset - Nx.dot(x_offset, coeff)
     else
       Nx.tensor(0.0, type: Nx.type(coeff))
     end

From 60004c74de9b2b220f0ad3f0855054d592ea9566 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sat, 30 Mar 2024 12:25:00 +0100
Subject: [PATCH 52/90] formatter

---
 .../linear/bayesian_ridge_regression.ex       | 93 ++++++++++++-------
 .../linear/bayesian_ridge_regression_test.exs |  7 +-
 2 files changed, 66 insertions(+), 34 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index c4dfba1a..8bf3bd16 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -184,45 +184,70 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     eigenvals = Nx.pow(s, 2)
     {n_samples, n_features} = Nx.shape(x)
     {coef, rmse} = update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
+
     score =
       log_marginal_likelihood(
-        coef, rmse, n_samples, n_features, eigenvals, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2)
+        coef,
+        rmse,
+        n_samples,
+        n_features,
+        eigenvals,
+        alpha,
+        lambda,
+        alpha_1,
+        alpha_2,
+        lambda_1,
+        lambda_2
+      )
+
     {{coef, alpha, lambda, rmse, iter, has_converged, score}, _} =
       while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0), score = score},
              {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}},
             iter < iterations and not has_converged do
-
         gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
         lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
         alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
+
         {coef_new, rmse} =
           update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
+
         score =
           log_marginal_likelihood(
-            coef, rmse, n_samples, n_features, eigenvals,
-            alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2)
+            coef,
+            rmse,
+            n_samples,
+            n_features,
+            eigenvals,
+            alpha,
+            lambda,
+            alpha_1,
+            alpha_2,
+            lambda_1,
+            lambda_2
+          )
+
         has_converged = Nx.sum(Nx.abs(coef - coef_new)) < 1.0e-8
 
         {{coef_new, alpha, lambda, rmse, iter + 1, has_converged, score},
          {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}}
       end
-    
+
     intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
     {coef, intercept, alpha, lambda, rmse, iter, has_converged, score}
   end
 
-  defn update_coef(
-         x,
-         y,
-         n_samples,
-         n_features,
-         xt_y,
-         u,
-         vh,
-         eigenvals,
-         alpha,
-         lambda
-       ) do
+  defnp update_coef(
+          x,
+          y,
+          n_samples,
+          n_features,
+          xt_y,
+          u,
+          vh,
+          eigenvals,
+          alpha,
+          lambda
+        ) do
     scaled_eigens = eigenvals + lambda / alpha
     regularization = vh / Nx.new_axis(scaled_eigens, -1)
     reg_transpose = Nx.dot(regularization, xt_y)
@@ -235,26 +260,30 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {coef, rmse}
   end
 
-  defn log_marginal_likelihood(
-         coef,
-         rmse,
-         n_samples,
-         n_features,
-         eigenvals,
-         alpha,
-         lambda,
-         alpha_1,
-         alpha_2,
-         lambda_1,
-         lambda_2
-       ) do
+  defnp log_marginal_likelihood(
+          coef,
+          rmse,
+          n_samples,
+          n_features,
+          eigenvals,
+          alpha,
+          lambda,
+          alpha_1,
+          alpha_2,
+          lambda_1,
+          lambda_2
+        ) do
     logdet_sigma = -1 * Nx.sum(Nx.log(lambda + alpha * eigenvals))
     score_lambda = lambda_1 * Nx.log(lambda) - lambda_2 * lambda
     score_alpha = alpha_1 * Nx.log(alpha) - alpha_2 * alpha
+
     score_parameters =
-      n_features * Nx.log(lambda) + n_samples * Nx.log(alpha) - alpha * rmse - lambda * Nx.sum(coef ** 2)
+      n_features * Nx.log(lambda) + n_samples * Nx.log(alpha) - alpha * rmse -
+        lambda * Nx.sum(coef ** 2)
+
     score =
-        0.5 * (score_parameters + logdet_sigma  - n_samples * Nx.log(2 * Nx.Constants.pi()))
+      0.5 * (score_parameters + logdet_sigma - n_samples * Nx.log(2 * Nx.Constants.pi()))
+
     score_alpha + score_lambda + score
   end
 
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 207b158e..d7104042 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -48,6 +48,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     lambda_2 = 0.1
     # compute score
     score = compute_score(x, y, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2)
+
     brr =
       BayesianRidgeRegression.fit(x, y,
         alpha_1: alpha_1,
@@ -57,6 +58,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
         fit_intercept?: true,
         iterations: 1
       )
+
     compare_scores = Nx.divide(Nx.subtract(score, brr.score), score)
     check = Nx.less(compare_scores, 0.05) |> Nx.flatten()
     assert check == Nx.tensor([1], type: {:u, 8})
@@ -66,12 +68,13 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     {n_samples, _} = Nx.shape(x)
     lambda_score = lambda_1 * Nx.log(lambda) - lambda_2 * lambda
     alpha_score = alpha_1 * Nx.log(alpha) - alpha_2 * alpha
-    m = (1.0 / alpha * Nx.eye(n_samples)) + (1.0 / lambda * Nx.dot(x, Nx.transpose(x)))
+    m = 1.0 / alpha * Nx.eye(n_samples) + 1.0 / lambda * Nx.dot(x, Nx.transpose(x))
     m_inv_dot_y = Nx.LinAlg.solve(m, y)
     logdet = m |> Nx.LinAlg.determinant() |> Nx.log()
 
     y_score =
-      -0.5 * (logdet + Nx.dot(Nx.transpose(y), m_inv_dot_y) + n_samples * Nx.log(2 * Nx.Constants.pi()))
+      -0.5 *
+        (logdet + Nx.dot(Nx.transpose(y), m_inv_dot_y) + n_samples * Nx.log(2 * Nx.Constants.pi()))
 
     alpha_score + lambda_score + y_score
   end

From 024874965f66bfd99a6b241bf138087472857c79 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Wed, 3 Apr 2024 18:36:28 +0200
Subject: [PATCH 53/90] add mini test to check linear regression timeout

---
 .../scholar/linear/bayesian_ridge_regression_test.exs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index d7104042..5b8756b4 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -3,6 +3,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
   use Scholar.Case, async: true
   alias Scholar.Linear.BayesianRidgeRegression
   alias Scholar.Linear.RidgeRegression
+  alias Scholar.Linear.LinearRegression
   doctest BayesianRidgeRegression
 
   test "toy bayesian ridge" do
@@ -34,6 +35,16 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
 
+  @tag :wip
+  test "Other linear methods struggle with inversion of x's outer product" do
+    {x, y} = diabetes_data()
+    lr = LinearRegression.fit(x, y)
+    IO.inspect(lr)
+    rr = RidgeRegression.fit(x, y)
+    IO.inspect(rr)
+    assert false
+  end
+
   test "compute scores" do
     {x, y} = diabetes_data()
     n_samples = 50 - 1

From 9f114f40c3ce92ac534fc7db6c26ea0ce101e85e Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Wed, 3 Apr 2024 22:49:05 +0200
Subject: [PATCH 54/90] fixed multiple scores

---
 .../linear/bayesian_ridge_regression.ex       | 63 +++++++++----------
 .../linear/bayesian_ridge_regression_test.exs | 12 ++--
 2 files changed, 36 insertions(+), 39 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 8bf3bd16..5d61e457 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -4,8 +4,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   import Scholar.Shared
 
   @derive {Nx.Container,
-           containers: [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations, :score]}
-  defstruct [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations, :score]
+           containers: [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations, :scores]}
+  defstruct [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations, :scores]
 
   opts = [
     iterations: [
@@ -125,9 +125,16 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {lambda, opts} = Keyword.pop!(opts, :lambda_init)
     lambda = Nx.tensor(lambda, type: x_type)
     opts = Keyword.put(opts, :lambda_init, lambda)
+    zeros_list = for k <- 0..opts[:iterations], do: 0
+    scores = Nx.tensor(zeros_list, type: x_type)
+    IO.inspect(scores)
 
-    {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged, score} =
-      fit_n(x, y, sample_weights, opts)
+    {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged, scores} =
+      fit_n(x, y, sample_weights, scores, opts)
+    iterations = Nx.to_number(iterations)
+    scores = scores
+    |> Nx.to_list()
+    |> Enum.take(iterations)
 
     if Nx.to_number(has_converged) == 1 do
       IO.puts("Convergence after #{Nx.to_number(iterations)} iterations")
@@ -139,12 +146,12 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       alpha: Nx.to_number(alpha),
       lambda: Nx.to_number(lambda),
       rmse: Nx.to_number(rmse),
-      iterations: Nx.to_number(iterations),
-      score: score
+      iterations: iterations,
+      scores: scores
     }
   end
 
-  defnp fit_n(x, y, sample_weights, opts) do
+  defnp fit_n(x, y, sample_weights, scores, opts) do
     x = to_float(x)
     y = to_float(y)
 
@@ -185,33 +192,11 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {n_samples, n_features} = Nx.shape(x)
     {coef, rmse} = update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
 
-    score =
-      log_marginal_likelihood(
-        coef,
-        rmse,
-        n_samples,
-        n_features,
-        eigenvals,
-        alpha,
-        lambda,
-        alpha_1,
-        alpha_2,
-        lambda_1,
-        lambda_2
-      )
-
-    {{coef, alpha, lambda, rmse, iter, has_converged, score}, _} =
-      while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0), score = score},
+    {{coef, alpha, lambda, rmse, iter, has_converged, scores}, _} =
+      while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0), scores = scores},
              {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}},
-            iter < iterations and not has_converged do
-        gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
-        lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
-        alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
-
-        {coef_new, rmse} =
-          update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
-
-        score =
+            iter <= iterations and not has_converged do
+        new_score =
           log_marginal_likelihood(
             coef,
             rmse,
@@ -225,15 +210,23 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
             lambda_1,
             lambda_2
           )
+        scores = Nx.put_slice(scores, [iter], Nx.new_axis(new_score, -1))
+        
+        gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
+        lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
+        alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
+
+        {coef_new, rmse} =
+          update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
 
         has_converged = Nx.sum(Nx.abs(coef - coef_new)) < 1.0e-8
 
-        {{coef_new, alpha, lambda, rmse, iter + 1, has_converged, score},
+        {{coef_new, alpha, lambda, rmse, iter + 1, has_converged, scores},
          {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}}
       end
 
     intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
-    {coef, intercept, alpha, lambda, rmse, iter, has_converged, score}
+    {coef, intercept, alpha, lambda, rmse, iter, has_converged, scores}
   end
 
   defnp update_coef(
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 5b8756b4..ee127f8b 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -35,7 +35,6 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
 
-  @tag :wip
   test "Other linear methods struggle with inversion of x's outer product" do
     {x, y} = diabetes_data()
     lr = LinearRegression.fit(x, y)
@@ -44,11 +43,13 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     IO.inspect(rr)
     assert false
   end
-
+  
+  @tag :wip
   test "compute scores" do
     {x, y} = diabetes_data()
     n_samples = 50 - 1
     x = x[[0..n_samples, ..]]
+    IO.inspect(x)
     y = y[[0..n_samples, ..]]
     eps = Nx.Constants.smallest_positive_normal({:f, 64})
     alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
@@ -69,8 +70,11 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
         fit_intercept?: true,
         iterations: 1
       )
-
-    compare_scores = Nx.divide(Nx.subtract(score, brr.score), score)
+    first_score = brr.scores
+    |> List.first()
+    |> Nx.tensor()
+    
+    compare_scores = Nx.divide(Nx.subtract(score, first_score), score)
     check = Nx.less(compare_scores, 0.05) |> Nx.flatten()
     assert check == Nx.tensor([1], type: {:u, 8})
   end

From 92e434e99eae910fbfbe783202bc8de907488ddf Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Wed, 3 Apr 2024 23:26:01 +0200
Subject: [PATCH 55/90] added sigma

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 12 +++++++-----
 .../linear/bayesian_ridge_regression_test.exs   | 17 +++++++++++++++--
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 5d61e457..521f5aa7 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -4,8 +4,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   import Scholar.Shared
 
   @derive {Nx.Container,
-           containers: [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations, :scores]}
-  defstruct [:coefficients, :intercept, :alpha, :lambda, :rmse, :iterations, :scores]
+           containers: [:coefficients, :intercept, :alpha, :lambda, :sigma, :rmse, :iterations, :scores]}
+  defstruct [:coefficients, :intercept, :alpha, :lambda, :sigma, :rmse, :iterations, :scores]
 
   opts = [
     iterations: [
@@ -127,9 +127,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     opts = Keyword.put(opts, :lambda_init, lambda)
     zeros_list = for k <- 0..opts[:iterations], do: 0
     scores = Nx.tensor(zeros_list, type: x_type)
-    IO.inspect(scores)
 
-    {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged, scores} =
+    {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged, scores, sigma} =
       fit_n(x, y, sample_weights, scores, opts)
     iterations = Nx.to_number(iterations)
     scores = scores
@@ -145,6 +144,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       intercept: intercept,
       alpha: Nx.to_number(alpha),
       lambda: Nx.to_number(lambda),
+      sigma: sigma,
       rmse: Nx.to_number(rmse),
       iterations: iterations,
       scores: scores
@@ -226,7 +226,9 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       end
 
     intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
-    {coef, intercept, alpha, lambda, rmse, iter, has_converged, scores}
+    scaled_sigma = Nx.dot(Nx.transpose(vh), vh / Nx.new_axis(eigenvals + lambda / alpha, -1))
+    sigma = scaled_sigma / alpha
+    {coef, intercept, alpha, lambda, rmse, iter, has_converged, scores, sigma}
   end
 
   defnp update_coef(
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index ee127f8b..ed5dbf70 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -44,7 +44,6 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert false
   end
   
-  @tag :wip
   test "compute scores" do
     {x, y} = diabetes_data()
     n_samples = 50 - 1
@@ -98,8 +97,22 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert false
   end
 
+  @tag :wip
   test "constant inputs: variance" do
-    assert false
+    key = Nx.Random.key(42)
+    n_samples = 15
+    n_features = 10
+    {constant_value, new_key} = Nx.Random.uniform(key)
+    {x, _} = Nx.Random.uniform(new_key, shape: {n_samples, n_features}, type: :f64)
+    y = Nx.tensor(for k <- 0..(n_samples - 1), do: Nx.to_number(constant_value))
+    brr = BayesianRidgeRegression.fit(x, y)
+    check = Nx.less_equal(brr.sigma, 0.01)
+    ones = Nx.tensor(
+             for i <- 0..(n_features - 1) do
+               for k <- 0..(n_features - 1), do: 1
+             end,
+      type: {:u, 8})    
+    assert ones == check
   end
 
   test "n_features > n_samples" do

From 7a53d4df3ffd0f52ef3790615928d5576d62f944 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Wed, 3 Apr 2024 23:32:52 +0200
Subject: [PATCH 56/90] add required underscores

---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index ed5dbf70..b0c25bfa 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -97,7 +97,6 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert false
   end
 
-  @tag :wip
   test "constant inputs: variance" do
     key = Nx.Random.key(42)
     n_samples = 15
@@ -108,8 +107,8 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     brr = BayesianRidgeRegression.fit(x, y)
     check = Nx.less_equal(brr.sigma, 0.01)
     ones = Nx.tensor(
-             for i <- 0..(n_features - 1) do
-               for k <- 0..(n_features - 1), do: 1
+             for _i <- 0..(n_features - 1) do
+               for _k <- 0..(n_features - 1), do: 1
              end,
       type: {:u, 8})    
     assert ones == check

From cd3822c4f76b9fec9e5fa91ae6e41f761179b229 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Wed, 3 Apr 2024 23:33:03 +0200
Subject: [PATCH 57/90] fix predict function. scores are list and cannot be
 defn

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 521f5aa7..b405b1e0 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -282,10 +282,12 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     score_alpha + score_lambda + score
   end
 
-  defn predict(%__MODULE__{coefficients: coeff, intercept: intercept} = _model, x) do
-    Nx.dot(x, [-1], coeff, [-1]) + intercept
+  deftransform predict(%__MODULE__{coefficients: coeff, intercept: intercept} = _model, x) do
+    predict_n(coeff, intercept, x)
   end
 
+  defnp predict_n(coeff, intercept, x), do: Nx.dot(x, [-1], coeff, [-1]) + intercept
+
   # Implements sample weighting by rescaling inputs and
   # targets by sqrt(sample_weight).
   defnp rescale(x, y, sample_weights) do

From fdd0324725d2aac388ffdbee9134598f6d1983a4 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:45:13 +0200
Subject: [PATCH 58/90] Update lib/scholar/linear/bayesian_ridge_regression.ex

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index b405b1e0..f6b3dd65 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -226,7 +226,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       end
 
     intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
-    scaled_sigma = Nx.dot(Nx.transpose(vh), vh / Nx.new_axis(eigenvals + lambda / alpha, -1))
+    scaled_sigma = Nx.dot(vh, [0],  vh / Nx.new_axis(eigenvals + lambda / alpha, -1), [0])
     sigma = scaled_sigma / alpha
     {coef, intercept, alpha, lambda, rmse, iter, has_converged, scores, sigma}
   end

From 287eb6a4fd1f60cf736b116aa441be5fb84a6f63 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:45:25 +0200
Subject: [PATCH 59/90] Update lib/scholar/linear/bayesian_ridge_regression.ex

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index f6b3dd65..30fadcdf 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -246,7 +246,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     scaled_eigens = eigenvals + lambda / alpha
     regularization = vh / Nx.new_axis(scaled_eigens, -1)
     reg_transpose = Nx.dot(regularization, xt_y)
-    coef = Nx.dot(Nx.transpose(vh), reg_transpose)
+    coef = Nx.dot(vh, [0], reg_transpose, [0])
 
     error = y - Nx.dot(x, coef)
     squared_error = error ** 2

From 3694a0da8c17951652dcb412f76ba3a01bc144e9 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:45:44 +0200
Subject: [PATCH 60/90] Update lib/scholar/linear/bayesian_ridge_regression.ex
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Krsto Proroković <krstopro@yahoo.com>
---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 30fadcdf..7b369fad 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -186,7 +186,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
 
     iterations = opts[:iterations]
 
-    xt_y = Nx.dot(Nx.transpose(x), y)
+xt_y = Nx.dot(x, [0], y, [0])
     {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
     eigenvals = Nx.pow(s, 2)
     {n_samples, n_features} = Nx.shape(x)

From 199e1f2fec3bad1b5229b9e2a689ad21d82fe1d5 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:45:53 +0200
Subject: [PATCH 61/90] Update lib/scholar/linear/bayesian_ridge_regression.ex

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 7b369fad..74c1cb72 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -117,7 +117,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     # handle vector types
     # handle default alpha value, add eps to avoid division by 0
     eps = Nx.Constants.smallest_positive_normal(x_type)
-    default_alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
+    default_alpha = 1 / (Nx.variance(x) + eps)
     alpha = Keyword.get(opts, :alpha_init, default_alpha)
     alpha = Nx.tensor(alpha, type: x_type)
     opts = Keyword.put(opts, :alpha_init, alpha)

From c409e83363495d3e9dc45c9bc739fb3092708d84 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:46:09 +0200
Subject: [PATCH 62/90] Update lib/scholar/linear/bayesian_ridge_regression.ex

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 74c1cb72..45714567 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -188,7 +188,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
 
 xt_y = Nx.dot(x, [0], y, [0])
     {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
-    eigenvals = Nx.pow(s, 2)
+    eigenvals = s ** 2
     {n_samples, n_features} = Nx.shape(x)
     {coef, rmse} = update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
 

From d9628a6ad4e9ff984b748930aaef6255dcaaacb4 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:46:18 +0200
Subject: [PATCH 63/90] Update
 test/scholar/linear/bayesian_ridge_regression_test.exs

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index b0c25bfa..b47eae8f 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -105,7 +105,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     {x, _} = Nx.Random.uniform(new_key, shape: {n_samples, n_features}, type: :f64)
     y = Nx.tensor(for k <- 0..(n_samples - 1), do: Nx.to_number(constant_value))
     brr = BayesianRidgeRegression.fit(x, y)
-    check = Nx.less_equal(brr.sigma, 0.01)
+    check = brr.sigma <= 0.01
     ones = Nx.tensor(
              for _i <- 0..(n_features - 1) do
                for _k <- 0..(n_features - 1), do: 1

From a4c6728679673ca8ccbc7a7934632eb414e4b25f Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:46:25 +0200
Subject: [PATCH 64/90] Update
 test/scholar/linear/bayesian_ridge_regression_test.exs

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index b47eae8f..a542226a 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -106,12 +106,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     y = Nx.tensor(for k <- 0..(n_samples - 1), do: Nx.to_number(constant_value))
     brr = BayesianRidgeRegression.fit(x, y)
     check = brr.sigma <= 0.01
-    ones = Nx.tensor(
-             for _i <- 0..(n_features - 1) do
-               for _k <- 0..(n_features - 1), do: 1
-             end,
-      type: {:u, 8})    
-    assert ones == check
+    assert Nx.all(check) == Nx.u8(1)
   end
 
   test "n_features > n_samples" do

From f14f3fafb3bd475a8f42053412165b316ff01d64 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:46:50 +0200
Subject: [PATCH 65/90] Update
 test/scholar/linear/bayesian_ridge_regression_test.exs

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index a542226a..7fb7d2ba 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -50,9 +50,9 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     x = x[[0..n_samples, ..]]
     IO.inspect(x)
     y = y[[0..n_samples, ..]]
-    eps = Nx.Constants.smallest_positive_normal({:f, 64})
-    alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
-    lambda = Nx.tensor(1.0)
+    eps = Nx.Constants.smallest_positive_normal(:f64)
+    alpha = 1 / (Nx.variance(x) + eps)
+    lambda = 1.0
     alpha_1 = 0.1
     alpha_2 = 0.1
     lambda_1 = 0.1

From 8e0fb45b348ac3940690620c9b71798c51129e93 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:46:59 +0200
Subject: [PATCH 66/90] Update
 test/scholar/linear/bayesian_ridge_regression_test.exs

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 7fb7d2ba..61796c07 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -82,7 +82,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     {n_samples, _} = Nx.shape(x)
     lambda_score = lambda_1 * Nx.log(lambda) - lambda_2 * lambda
     alpha_score = alpha_1 * Nx.log(alpha) - alpha_2 * alpha
-    m = 1.0 / alpha * Nx.eye(n_samples) + 1.0 / lambda * Nx.dot(x, Nx.transpose(x))
+    m = 1.0 / alpha * Nx.eye(n_samples) + 1.0 / lambda * Nx.dot(x, [-1], x, [-1])
     m_inv_dot_y = Nx.LinAlg.solve(m, y)
     logdet = m |> Nx.LinAlg.determinant() |> Nx.log()
 

From 9ba1bc41c3e561620520fbde29d038ba82aed1e1 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:47:16 +0200
Subject: [PATCH 67/90] Update
 test/scholar/linear/bayesian_ridge_regression_test.exs

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 61796c07..b919bcff 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -73,9 +73,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     |> List.first()
     |> Nx.tensor()
     
-    compare_scores = Nx.divide(Nx.subtract(score, first_score), score)
-    check = Nx.less(compare_scores, 0.05) |> Nx.flatten()
-    assert check == Nx.tensor([1], type: {:u, 8})
+    assert_all_close(score, first_score, rtol: 0.05)
   end
 
   defnp compute_score(x, y, alpha, lambda, alpha_1, alpha_2, lambda_1, lambda_2) do

From 44d10e6bdd350acfae14123617eb41605c599c07 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:47:40 +0200
Subject: [PATCH 68/90] Update
 test/scholar/linear/bayesian_ridge_regression_test.exs

Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index b919bcff..4184f169 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -101,7 +101,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     n_features = 10
     {constant_value, new_key} = Nx.Random.uniform(key)
     {x, _} = Nx.Random.uniform(new_key, shape: {n_samples, n_features}, type: :f64)
-    y = Nx.tensor(for k <- 0..(n_samples - 1), do: Nx.to_number(constant_value))
+    y = Nx.broadcast(constant_value, {n_samples})
     brr = BayesianRidgeRegression.fit(x, y)
     check = brr.sigma <= 0.01
     assert Nx.all(check) == Nx.u8(1)

From 0006e71592812485cfd48808559d0f01259bc277 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 19:06:06 +0200
Subject: [PATCH 69/90] Update lib/scholar/linear/bayesian_ridge_regression.ex
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: José Valim <jose.valim@gmail.com>
---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 45714567..740a554b 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -125,7 +125,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {lambda, opts} = Keyword.pop!(opts, :lambda_init)
     lambda = Nx.tensor(lambda, type: x_type)
     opts = Keyword.put(opts, :lambda_init, lambda)
-    zeros_list = for k <- 0..opts[:iterations], do: 0
+    zeros_list = List.duplicate(0, opts[:iterations])
     scores = Nx.tensor(zeros_list, type: x_type)
 
     {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged, scores, sigma} =

From de7b9721beeece8c2c19e62fca6597bdbaf2c8fb Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 18:59:47 +0200
Subject: [PATCH 70/90] dot product without transpose

---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 4184f169..ed234b14 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -86,7 +86,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
 
     y_score =
       -0.5 *
-        (logdet + Nx.dot(Nx.transpose(y), m_inv_dot_y) + n_samples * Nx.log(2 * Nx.Constants.pi()))
+        (logdet + Nx.dot(y, [0], m_inv_dot_y, [0]) + n_samples * Nx.log(2 * Nx.Constants.pi()))
 
     alpha_score + lambda_score + y_score
   end

From 5d59f8715b88b545a6d3e831eb5b0239e55adb7b Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 19:00:13 +0200
Subject: [PATCH 71/90] remove show test

---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index ed234b14..80893a28 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -35,15 +35,6 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
 
-  test "Other linear methods struggle with inversion of x's outer product" do
-    {x, y} = diabetes_data()
-    lr = LinearRegression.fit(x, y)
-    IO.inspect(lr)
-    rr = RidgeRegression.fit(x, y)
-    IO.inspect(rr)
-    assert false
-  end
-  
   test "compute scores" do
     {x, y} = diabetes_data()
     n_samples = 50 - 1

From 3ecde56f982e9545ca3b62242e73105ec287471f Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 19:00:25 +0200
Subject: [PATCH 72/90] remove debug inspec

---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 80893a28..c9911a61 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -39,7 +39,6 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     {x, y} = diabetes_data()
     n_samples = 50 - 1
     x = x[[0..n_samples, ..]]
-    IO.inspect(x)
     y = y[[0..n_samples, ..]]
     eps = Nx.Constants.smallest_positive_normal(:f64)
     alpha = 1 / (Nx.variance(x) + eps)

From 0d43838f58844aa8a97c07e47db1069b5b83b8ea Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 19:04:51 +0200
Subject: [PATCH 73/90] linear regression dependency is no longer required

---
 test/scholar/linear/bayesian_ridge_regression_test.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index c9911a61..f4fe98e2 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -3,7 +3,6 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
   use Scholar.Case, async: true
   alias Scholar.Linear.BayesianRidgeRegression
   alias Scholar.Linear.RidgeRegression
-  alias Scholar.Linear.LinearRegression
   doctest BayesianRidgeRegression
 
   test "toy bayesian ridge" do

From 77624082ff617fb883cfa64b3767a68572316be0 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 20:19:10 +0200
Subject: [PATCH 74/90] minor fixes, and cleanup

---
 .../linear/bayesian_ridge_regression.ex       | 39 +++++++------------
 1 file changed, 14 insertions(+), 25 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 740a554b..9d9f4f52 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -4,8 +4,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   import Scholar.Shared
 
   @derive {Nx.Container,
-           containers: [:coefficients, :intercept, :alpha, :lambda, :sigma, :rmse, :iterations, :scores]}
-  defstruct [:coefficients, :intercept, :alpha, :lambda, :sigma, :rmse, :iterations, :scores]
+           containers: [:coefficients, :intercept, :alpha, :lambda, :sigma, :rmse, :iterations, :has_converged, :scores]}
+  defstruct [:coefficients, :intercept, :alpha, :lambda, :sigma, :rmse, :iterations, :has_converged, :scores]
 
   opts = [
     iterations: [
@@ -117,41 +117,33 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     # handle vector types
     # handle default alpha value, add eps to avoid division by 0
     eps = Nx.Constants.smallest_positive_normal(x_type)
-    default_alpha = 1 / (Nx.variance(x) + eps)
+    default_alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
     alpha = Keyword.get(opts, :alpha_init, default_alpha)
     alpha = Nx.tensor(alpha, type: x_type)
-    opts = Keyword.put(opts, :alpha_init, alpha)
 
     {lambda, opts} = Keyword.pop!(opts, :lambda_init)
     lambda = Nx.tensor(lambda, type: x_type)
-    opts = Keyword.put(opts, :lambda_init, lambda)
-    zeros_list = List.duplicate(0, opts[:iterations])
-    scores = Nx.tensor(zeros_list, type: x_type)
+
+    scores = Nx.broadcast(0, {opts[:iterations] + 1})
+    |> Nx.as_type(x_type)
 
     {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged, scores, sigma} =
-      fit_n(x, y, sample_weights, scores, opts)
-    iterations = Nx.to_number(iterations)
-    scores = scores
-    |> Nx.to_list()
-    |> Enum.take(iterations)
-
-    if Nx.to_number(has_converged) == 1 do
-      IO.puts("Convergence after #{Nx.to_number(iterations)} iterations")
-    end
+      fit_n(x, y, alpha, lambda, sample_weights, scores, opts)
 
     %__MODULE__{
       coefficients: coefficients,
       intercept: intercept,
-      alpha: Nx.to_number(alpha),
-      lambda: Nx.to_number(lambda),
+      alpha: alpha,
+      lambda: lambda,
       sigma: sigma,
-      rmse: Nx.to_number(rmse),
+      rmse: rmse,
       iterations: iterations,
-      scores: scores
+      has_converged: has_converged,
+      scores: scores,
     }
   end
 
-  defnp fit_n(x, y, sample_weights, scores, opts) do
+  defnp fit_n(x, y, alpha, lambda, sample_weights, scores, opts) do
     x = to_float(x)
     y = to_float(y)
 
@@ -176,9 +168,6 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         {x, y}
       end
 
-    alpha = opts[:alpha_init]
-    lambda = opts[:lambda_init]
-
     alpha_1 = opts[:alpha_1]
     alpha_2 = opts[:alpha_2]
     lambda_1 = opts[:lambda_1]
@@ -186,7 +175,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
 
     iterations = opts[:iterations]
 
-xt_y = Nx.dot(x, [0], y, [0])
+    xt_y = Nx.dot(x, [0], y, [0])
     {u, s, vh} = Nx.LinAlg.svd(x, full_matrices?: false)
     eigenvals = s ** 2
     {n_samples, n_features} = Nx.shape(x)

From 5ae61c4c9112c4f1f8c37225baa483d3f2e01d34 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 20:19:54 +0200
Subject: [PATCH 75/90] fix test types

---
 .../scholar/linear/bayesian_ridge_regression_test.exs | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index f4fe98e2..41618cdb 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -19,7 +19,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     x = Nx.tensor([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
     y = Nx.tensor([1, 2, 3, 2, 0, 4, 5])
     brr = BayesianRidgeRegression.fit(x, y)
-    rr = RidgeRegression.fit(x, y, alpha: brr.lambda / brr.alpha)
+    rr = RidgeRegression.fit(x, y, alpha: Nx.to_number(brr.lambda) / Nx.to_number(brr.alpha))
     assert_all_close(brr.coefficients, rr.coefficients, atol: 1.0e-2)
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
@@ -29,7 +29,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     y = Nx.tensor([1, 2, 3, 2, 0, 4, 5])
     w = Nx.tensor([4, 3, 3, 1, 1, 2, 3])
     brr = BayesianRidgeRegression.fit(x, y, sample_weights: w)
-    rr = RidgeRegression.fit(x, y, alpha: brr.lambda / brr.alpha, sample_weights: w)
+    rr = RidgeRegression.fit(x, y, alpha: Nx.to_number(brr.lambda) / Nx.to_number(brr.alpha), sample_weights: w)
     assert_all_close(brr.coefficients, rr.coefficients, atol: 1.0e-2)
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
@@ -40,7 +40,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     x = x[[0..n_samples, ..]]
     y = y[[0..n_samples, ..]]
     eps = Nx.Constants.smallest_positive_normal(:f64)
-    alpha = 1 / (Nx.variance(x) + eps)
+    alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
     lambda = 1.0
     alpha_1 = 0.1
     alpha_2 = 0.1
@@ -56,12 +56,11 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
         lambda_1: lambda_1,
         lambda_2: lambda_2,
         fit_intercept?: true,
+        compute_scores?: true,
         iterations: 1
       )
-    first_score = brr.scores
-    |> List.first()
-    |> Nx.tensor()
     
+    first_score = brr.scores[0]
     assert_all_close(score, first_score, rtol: 0.05)
   end
 

From 7ce464bbd318a9c26d71fa62e28d2b833a6160ec Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 20:36:48 +0200
Subject: [PATCH 76/90] fixed optional scores and n_features > n_samples

---
 .../linear/bayesian_ridge_regression.ex       | 80 +++++++++++++------
 .../linear/bayesian_ridge_regression_test.exs | 17 +++-
 2 files changed, 70 insertions(+), 27 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 9d9f4f52..e1bb0404 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -39,6 +39,14 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       of targets for a zero-vector on input.
       """
     ],
+    compute_scores?: [
+      type: :boolean,
+      default: false,
+      doc: """
+      If set to `true`, the log marginal likelihood will be computed
+      at each iteration of the algorithm.
+      """
+    ],    
     alpha_init: [
       type: {:custom, Scholar.Options, :non_negative_number, []},
       doc: ~S"""
@@ -124,12 +132,20 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {lambda, opts} = Keyword.pop!(opts, :lambda_init)
     lambda = Nx.tensor(lambda, type: x_type)
 
-    scores = Nx.broadcast(0, {opts[:iterations] + 1})
-    |> Nx.as_type(x_type)
+    scores = Nx.tensor(:nan)
+      |> Nx.broadcast({opts[:iterations] + 1})
+      |> Nx.as_type(x_type)
 
     {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged, scores, sigma} =
       fit_n(x, y, alpha, lambda, sample_weights, scores, opts)
 
+    scores =
+      if opts[:compute_scores?] do
+        scores
+      else
+        nil
+      end
+
     %__MODULE__{
       coefficients: coefficients,
       intercept: intercept,
@@ -185,21 +201,26 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0), scores = scores},
              {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}},
             iter <= iterations and not has_converged do
-        new_score =
-          log_marginal_likelihood(
-            coef,
-            rmse,
-            n_samples,
-            n_features,
-            eigenvals,
-            alpha,
-            lambda,
-            alpha_1,
-            alpha_2,
-            lambda_1,
-            lambda_2
-          )
-        scores = Nx.put_slice(scores, [iter], Nx.new_axis(new_score, -1))
+        scores =
+          if opts[:compute_scores?] do
+            new_score =
+              log_marginal_likelihood(
+                coef,
+                rmse,
+                n_samples,
+                n_features,
+                eigenvals,
+                alpha,
+                lambda,
+                alpha_1,
+                alpha_2,
+                lambda_1,
+                lambda_2
+              )
+            Nx.put_slice(scores, [iter], Nx.new_axis(new_score, -1))
+          else
+            scores
+          end
         
         gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
         lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
@@ -232,16 +253,23 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
           alpha,
           lambda
         ) do
-    scaled_eigens = eigenvals + lambda / alpha
-    regularization = vh / Nx.new_axis(scaled_eigens, -1)
-    reg_transpose = Nx.dot(regularization, xt_y)
-    coef = Nx.dot(vh, [0], reg_transpose, [0])
+    scaled_eigens = eigenvals + lambda / alpha    
+    coef =
+      if n_samples > n_features do
+        regularization = vh / Nx.new_axis(scaled_eigens, -1)
+        reg_transpose = Nx.dot(regularization, xt_y)
+        Nx.dot(vh, [0], reg_transpose, [0])
+      else
+        regularization = u / scaled_eigens
+        reg_transpose = Nx.dot(regularization, Nx.dot(u, [0], y, [0]))
+        Nx.dot(x, [0], reg_transpose, [0])
+    end
 
     error = y - Nx.dot(x, coef)
     squared_error = error ** 2
     rmse = Nx.sum(squared_error)
 
-    {coef, rmse}
+    {coef, rmse}    
   end
 
   defnp log_marginal_likelihood(
@@ -257,7 +285,13 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
           lambda_1,
           lambda_2
         ) do
-    logdet_sigma = -1 * Nx.sum(Nx.log(lambda + alpha * eigenvals))
+    logdet_sigma =
+      if n_samples > n_features do
+        -1 * Nx.sum(Nx.log(lambda + alpha * eigenvals))
+      else
+        broad_lambda = Nx.broadcast(lambda, {n_samples})
+        -1 * Nx.sum(Nx.log(broad_lambda + (alpha * eigenvals)))
+      end
     score_lambda = lambda_1 * Nx.log(lambda) - lambda_2 * lambda
     score_alpha = alpha_1 * Nx.log(alpha) - alpha_2 * alpha
 
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 41618cdb..78b22192 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -79,8 +79,17 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     alpha_score + lambda_score + y_score
   end
 
-  test "constant inputs: prediction" do
-    assert false
+  test "constant inputs: prediction. n_features > n_samples" do
+    key = Nx.Random.key(42)
+    n_samples = 4
+    n_features = 5
+    {constant_value, new_key} = Nx.Random.uniform(key)
+    {x, _} = Nx.Random.uniform(new_key, shape: {n_samples, n_features}, type: :f64)
+    y = Nx.broadcast(constant_value, {n_samples})
+    expected = Nx.broadcast(constant_value, {n_samples})
+    brr = BayesianRidgeRegression.fit(x, y)
+    predicted = BayesianRidgeRegression.predict(brr, x)
+    assert_all_close(expected, predicted, atol: 0.01)
   end
 
   test "constant inputs: variance" do
@@ -91,11 +100,11 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     {x, _} = Nx.Random.uniform(new_key, shape: {n_samples, n_features}, type: :f64)
     y = Nx.broadcast(constant_value, {n_samples})
     brr = BayesianRidgeRegression.fit(x, y)
-    check = brr.sigma <= 0.01
+    check = Nx.less_equal(brr.sigma, 0.01)
     assert Nx.all(check) == Nx.u8(1)
   end
 
-  test "n_features > n_samples" do
+  test "write docs" do
     assert false
   end
 end

From 0fe5c1acffe4db750db7e7d926495a4e75aa0b34 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 20:39:28 +0200
Subject: [PATCH 77/90] reduce diabetes data sample

---
 .../linear/bayesian_ridge_regression_test.exs |   3 -
 test/support/diabetes_data_raw.csv            | 394 +-----------------
 test/support/diabetes_target.csv              | 394 +-----------------
 test/support/scholar_case.ex                  |   1 +
 4 files changed, 3 insertions(+), 789 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 78b22192..687054b8 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -36,9 +36,6 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
 
   test "compute scores" do
     {x, y} = diabetes_data()
-    n_samples = 50 - 1
-    x = x[[0..n_samples, ..]]
-    y = y[[0..n_samples, ..]]
     eps = Nx.Constants.smallest_positive_normal(:f64)
     alpha = Nx.divide(1, Nx.add(Nx.variance(x), eps))
     lambda = 1.0
diff --git a/test/support/diabetes_data_raw.csv b/test/support/diabetes_data_raw.csv
index 95ed9eca..258261c2 100644
--- a/test/support/diabetes_data_raw.csv
+++ b/test/support/diabetes_data_raw.csv
@@ -47,396 +47,4 @@
 33 1 25.3 85.0 155 85.0 51.0 3.0 4.5539 70
 27 1 19.6 78.0 128 68.0 43.0 3.0 4.4427 71
 67 2 22.5 98.0 191 119.2 61.0 3.0 3.989 86
-37 2 27.7 93.0 180 119.4 30.0 6.0 5.0304 88
-58 1 25.7 99.0 157 91.6 49.0 3.0 4.4067 93
-65 2 27.9 103.0 159 96.8 42.0 4.0 4.6151 86
-34 1 25.5 93.0 218 144.0 57.0 4.0 4.4427 88
-46 1 24.9 115.0 198 129.6 54.0 4.0 4.2767 103
-35 1 28.7 97.0 204 126.8 64.0 3.0 4.1897 93
-37 1 21.8 84.0 184 101.0 73.0 3.0 3.912 93
-37 1 30.2 87.0 166 96.0 40.0 4.15 5.0106 87
-41 1 20.5 80.0 124 48.8 64.0 2.0 4.0254 75
-60 1 20.4 105.0 198 78.4 99.0 2.0 4.6347 79
-66 2 24.0 98.0 236 146.4 58.0 4.0 5.0626 96
-29 1 26.0 83.0 141 65.2 64.0 2.0 4.0775 83
-37 2 26.8 79.0 157 98.0 28.0 6.0 5.0434 96
-41 2 25.7 83.0 181 106.6 66.0 3.0 3.7377 85
-39 1 22.9 77.0 204 143.2 46.0 4.0 4.3041 74
-67 2 24.0 83.0 143 77.2 49.0 3.0 4.4308 94
-36 2 24.1 112.0 193 125.0 35.0 6.0 5.1059 95
-46 2 24.7 85.0 174 123.2 30.0 6.0 4.6444 96
-60 2 25.0 89.67 185 120.8 46.0 4.02 4.5109 92
-59 2 23.6 83.0 165 100.0 47.0 4.0 4.4998 92
-53 1 22.1 93.0 134 76.2 46.0 3.0 4.0775 96
-48 1 19.9 91.0 189 109.6 69.0 3.0 3.9512 101
-48 1 29.5 131.0 207 132.2 47.0 4.0 4.9345 106
-66 2 26.0 91.0 264 146.6 65.0 4.0 5.5683 87
-52 2 24.5 94.0 217 149.4 48.0 5.0 4.585 89
-52 2 26.6 111.0 209 126.4 61.0 3.0 4.6821 109
-46 2 23.5 87.0 181 114.8 44.0 4.0 4.7095 98
-40 2 29.0 115.0 97 47.2 35.0 2.77 4.3041 95
-22 1 23.0 73.0 161 97.8 54.0 3.0 3.8286 91
-50 1 21.0 88.0 140 71.8 35.0 4.0 5.112 71
-20 1 22.9 87.0 191 128.2 53.0 4.0 3.8918 85
-68 1 27.5 107.0 241 149.6 64.0 4.0 4.92 90
-52 2 24.3 86.0 197 133.6 44.0 5.0 4.5747 91
-44 1 23.1 87.0 213 126.4 77.0 3.0 3.8712 72
-38 1 27.3 81.0 146 81.6 47.0 3.0 4.4659 81
-49 1 22.7 65.33 168 96.2 62.0 2.71 3.8918 60
-61 1 33.0 95.0 182 114.8 54.0 3.0 4.1897 74
-29 2 19.4 83.0 152 105.8 39.0 4.0 3.5835 83
-61 1 25.8 98.0 235 125.8 76.0 3.0 5.112 82
-34 2 22.6 75.0 166 91.8 60.0 3.0 4.2627 108
-36 1 21.9 89.0 189 105.2 68.0 3.0 4.3694 96
-52 1 24.0 83.0 167 86.6 71.0 2.0 3.8501 94
-61 1 31.2 79.0 235 156.8 47.0 5.0 5.0499 96
-43 1 26.8 123.0 193 102.2 67.0 3.0 4.7791 94
-35 1 20.4 65.0 187 105.6 67.0 2.79 4.2767 78
-27 1 24.8 91.0 189 106.8 69.0 3.0 4.1897 69
-29 1 21.0 71.0 156 97.0 38.0 4.0 4.654 90
-64 2 27.3 109.0 186 107.6 38.0 5.0 5.3083 99
-41 1 34.6 87.33 205 142.6 41.0 5.0 4.6728 110
-49 2 25.9 91.0 178 106.6 52.0 3.0 4.5747 75
-48 1 20.4 98.0 209 139.4 46.0 5.0 4.7707 78
-53 1 28.0 88.0 233 143.8 58.0 4.0 5.0499 91
-53 2 22.2 113.0 197 115.2 67.0 3.0 4.3041 100
-23 1 29.0 90.0 216 131.4 65.0 3.0 4.585 91
-65 2 30.2 98.0 219 160.6 40.0 5.0 4.5218 84
-41 1 32.4 94.0 171 104.4 56.0 3.0 3.9703 76
-55 2 23.4 83.0 166 101.6 46.0 4.0 4.5218 96
-22 1 19.3 82.0 156 93.2 52.0 3.0 3.989 71
-56 1 31.0 78.67 187 141.4 34.0 5.5 4.0604 90
-54 2 30.6 103.33 144 79.8 30.0 4.8 5.1417 101
-59 2 25.5 95.33 190 139.4 35.0 5.43 4.3567 117
-60 2 23.4 88.0 153 89.8 58.0 3.0 3.2581 95
-54 1 26.8 87.0 206 122.0 68.0 3.0 4.382 80
-25 1 28.3 87.0 193 128.0 49.0 4.0 4.382 92
-54 2 27.7 113.0 200 128.4 37.0 5.0 5.1533 113
-55 1 36.6 113.0 199 94.4 43.0 4.63 5.7301 97
-40 2 26.5 93.0 236 147.0 37.0 7.0 5.5607 92
-62 2 31.8 115.0 199 128.6 44.0 5.0 4.8828 98
-65 1 24.4 120.0 222 135.6 37.0 6.0 5.5094 124
-33 2 25.4 102.0 206 141.0 39.0 5.0 4.8675 105
-53 1 22.0 94.0 175 88.0 59.0 3.0 4.9416 98
-35 1 26.8 98.0 162 103.6 45.0 4.0 4.2047 86
-66 1 28.0 101.0 195 129.2 40.0 5.0 4.8598 94
-62 2 33.9 101.0 221 156.4 35.0 6.0 4.9972 103
-50 2 29.6 94.33 300 242.4 33.0 9.09 4.8122 109
-47 1 28.6 97.0 164 90.6 56.0 3.0 4.4659 88
-47 2 25.6 94.0 165 74.8 40.0 4.0 5.5255 93
-24 1 20.7 87.0 149 80.6 61.0 2.0 3.6109 78
-58 2 26.2 91.0 217 124.2 71.0 3.0 4.6913 68
-34 1 20.6 87.0 185 112.2 58.0 3.0 4.3041 74
-51 1 27.9 96.0 196 122.2 42.0 5.0 5.0689 120
-31 2 35.3 125.0 187 112.4 48.0 4.0 4.8903 109
-22 1 19.9 75.0 175 108.6 54.0 3.0 4.1271 72
-53 2 24.4 92.0 214 146.0 50.0 4.0 4.4998 97
-37 2 21.4 83.0 128 69.6 49.0 3.0 3.8501 84
-28 1 30.4 85.0 198 115.6 67.0 3.0 4.3438 80
-47 1 31.6 84.0 154 88.0 30.0 5.1 5.1985 105
-23 1 18.8 78.0 145 72.0 63.0 2.0 3.912 86
-50 1 31.0 123.0 178 105.0 48.0 4.0 4.8283 88
-58 2 36.7 117.0 166 93.8 44.0 4.0 4.9488 109
-55 1 32.1 110.0 164 84.2 42.0 4.0 5.2417 90
-60 2 27.7 107.0 167 114.6 38.0 4.0 4.2767 95
-41 1 30.8 81.0 214 152.0 28.0 7.6 5.1358 123
-60 2 27.5 106.0 229 143.8 51.0 4.0 5.1417 91
-40 1 26.9 92.0 203 119.8 70.0 3.0 4.1897 81
-57 2 30.7 90.0 204 147.8 34.0 6.0 4.7095 93
-37 1 38.3 113.0 165 94.6 53.0 3.0 4.4659 79
-40 2 31.9 95.0 198 135.6 38.0 5.0 4.803999999999999 93
-33 1 35.0 89.0 200 130.4 42.0 4.76 4.9273 101
-32 2 27.8 89.0 216 146.2 55.0 4.0 4.3041 91
-35 2 25.9 81.0 174 102.4 31.0 6.0 5.3132 82
-55 1 32.9 102.0 164 106.2 41.0 4.0 4.4308 89
-49 1 26.0 93.0 183 100.2 64.0 3.0 4.5433 88
-39 2 26.3 115.0 218 158.2 32.0 7.0 4.9345 109
-60 2 22.3 113.0 186 125.8 46.0 4.0 4.2627 94
-67 2 28.3 93.0 204 132.2 49.0 4.0 4.7362 92
-41 2 32.0 109.0 251 170.6 49.0 5.0 5.0562 103
-44 1 25.4 95.0 162 92.6 53.0 3.0 4.4067 83
-48 2 23.3 89.33 212 142.8 46.0 4.61 4.7536 98
-45 1 20.3 74.33 190 126.2 49.0 3.88 4.3041 79
-47 1 30.4 120.0 199 120.0 46.0 4.0 5.1059 87
-46 1 20.6 73.0 172 107.0 51.0 3.0 4.2485 80
-36 2 32.3 115.0 286 199.4 39.0 7.0 5.4723 112
-34 1 29.2 73.0 172 108.2 49.0 4.0 4.3041 91
-53 2 33.1 117.0 183 119.0 48.0 4.0 4.382 106
-61 1 24.6 101.0 209 106.8 77.0 3.0 4.8363 88
-37 1 20.2 81.0 162 87.8 63.0 3.0 4.0254 88
-33 2 20.8 84.0 125 70.2 46.0 3.0 3.7842 66
-68 1 32.8 105.67 205 116.4 40.0 5.13 5.4931 117
-49 2 31.9 94.0 234 155.8 34.0 7.0 5.3982 122
-48 1 23.9 109.0 232 105.2 37.0 6.0 6.107 96
-55 2 24.5 84.0 179 105.8 66.0 3.0 3.5835 87
-43 1 22.1 66.0 134 77.2 45.0 3.0 4.0775 80
-60 2 33.0 97.0 217 125.6 45.0 5.0 5.4467 112
-31 2 19.0 93.0 137 73.0 47.0 3.0 4.4427 78
-53 2 27.3 82.0 119 55.0 39.0 3.0 4.8283 93
-67 1 22.8 87.0 166 98.6 52.0 3.0 4.3438 92
-61 2 28.2 106.0 204 132.0 52.0 4.0 4.6052 96
-62 1 28.9 87.33 206 127.2 33.0 6.24 5.4337 99
-60 1 25.6 87.0 207 125.8 69.0 3.0 4.1109 84
-42 1 24.9 91.0 204 141.8 38.0 5.0 4.7958 89
-38 2 26.8 105.0 181 119.2 37.0 5.0 4.8203 91
-62 1 22.4 79.0 222 147.4 59.0 4.0 4.3567 76
-61 2 26.9 111.0 236 172.4 39.0 6.0 4.8122 89
-61 2 23.1 113.0 186 114.4 47.0 4.0 4.8122 105
-53 1 28.6 88.0 171 98.8 41.0 4.0 5.0499 99
-28 2 24.7 97.0 175 99.6 32.0 5.0 5.3799 87
-26 2 30.3 89.0 218 152.2 31.0 7.0 5.1591 82
-30 1 21.3 87.0 134 63.0 63.0 2.0 3.6889 66
-50 1 26.1 109.0 243 160.6 62.0 4.0 4.625 89
-48 1 20.2 95.0 187 117.4 53.0 4.0 4.4188 85
-51 1 25.2 103.0 176 112.2 37.0 5.0 4.8978 90
-47 2 22.5 82.0 131 66.8 41.0 3.0 4.7536 89
-64 2 23.5 97.0 203 129.0 59.0 3.0 4.3175 77
-51 2 25.9 76.0 240 169.0 39.0 6.0 5.0752 96
-30 1 20.9 104.0 152 83.8 47.0 3.0 4.6634 97
-56 2 28.7 99.0 208 146.4 39.0 5.0 4.7274 97
-42 1 22.1 85.0 213 138.6 60.0 4.0 4.2767 94
-62 2 26.7 115.0 183 124.0 35.0 5.0 4.7875 100
-34 1 31.4 87.0 149 93.8 46.0 3.0 3.8286 77
-60 1 22.2 104.67 221 105.4 60.0 3.68 5.6276 93
-64 1 21.0 92.33 227 146.8 65.0 3.49 4.3307 102
-39 2 21.2 90.0 182 110.4 60.0 3.0 4.0604 98
-71 2 26.5 105.0 281 173.6 55.0 5.0 5.5683 84
-48 2 29.2 110.0 218 151.6 39.0 6.0 4.92 98
-79 2 27.0 103.0 169 110.8 37.0 5.0 4.6634 110
-40 1 30.7 99.0 177 85.4 50.0 4.0 5.3375 85
-49 2 28.8 92.0 207 140.0 44.0 5.0 4.7449 92
-51 1 30.6 103.0 198 106.6 57.0 3.0 5.1475 100
-57 1 30.1 117.0 202 139.6 42.0 5.0 4.625 120
-59 2 24.7 114.0 152 104.8 29.0 5.0 4.5109 88
-51 1 27.7 99.0 229 145.6 69.0 3.0 4.2767 77
-74 1 29.8 101.0 171 104.8 50.0 3.0 4.3944 86
-67 1 26.7 105.0 225 135.4 69.0 3.0 4.6347 96
-49 1 19.8 88.0 188 114.8 57.0 3.0 4.3944 93
-57 1 23.3 88.0 155 63.6 78.0 2.0 4.2047 78
-56 2 35.1 123.0 164 95.0 38.0 4.0 5.0434 117
-52 2 29.7 109.0 228 162.8 31.0 8.0 5.1417 103
-69 1 29.3 124.0 223 139.0 54.0 4.0 5.0106 102
-37 1 20.3 83.0 185 124.6 38.0 5.0 4.7185 88
-24 1 22.5 89.0 141 68.0 52.0 3.0 4.654 84
-55 2 22.7 93.0 154 94.2 53.0 3.0 3.5264 75
-36 1 22.8 87.0 178 116.0 41.0 4.0 4.654 82
-42 2 24.0 107.0 150 85.0 44.0 3.0 4.654 96
-21 1 24.2 76.0 147 77.0 53.0 3.0 4.4427 79
-41 1 20.2 62.0 153 89.0 50.0 3.0 4.2485 89
-57 2 29.4 109.0 160 87.6 31.0 5.0 5.3327 92
-20 2 22.1 87.0 171 99.6 58.0 3.0 4.2047 78
-67 2 23.6 111.33 189 105.4 70.0 2.7 4.2195 93
-34 1 25.2 77.0 189 120.6 53.0 4.0 4.3438 79
-41 2 24.9 86.0 192 115.0 61.0 3.0 4.382 94
-38 2 33.0 78.0 301 215.0 50.0 6.02 5.193 108
-51 1 23.5 101.0 195 121.0 51.0 4.0 4.7449 94
-52 2 26.4 91.33 218 152.0 39.0 5.59 4.9053 99
-67 1 29.8 80.0 172 93.4 63.0 3.0 4.3567 82
-61 1 30.0 108.0 194 100.0 52.0 3.73 5.3471 105
-67 2 25.0 111.67 146 93.4 33.0 4.42 4.585 103
-56 1 27.0 105.0 247 160.6 54.0 5.0 5.0876 94
-64 1 20.0 74.67 189 114.8 62.0 3.05 4.1109 91
-58 2 25.5 112.0 163 110.6 29.0 6.0 4.7622 86
-55 1 28.2 91.0 250 140.2 67.0 4.0 5.3660000000000005 103
-62 2 33.3 114.0 182 114.0 38.0 5.0 5.0106 96
-57 2 25.6 96.0 200 133.0 52.0 3.85 4.3175 105
-20 2 24.2 88.0 126 72.2 45.0 3.0 3.7842 74
-53 2 22.1 98.0 165 105.2 47.0 4.0 4.1589 81
-32 2 31.4 89.0 153 84.2 56.0 3.0 4.1589 90
-41 1 23.1 86.0 148 78.0 58.0 3.0 4.0943 60
-60 1 23.4 76.67 247 148.0 65.0 3.8 5.1358 77
-26 1 18.8 83.0 191 103.6 69.0 3.0 4.5218 69
-37 1 30.8 112.0 282 197.2 43.0 7.0 5.3423 101
-45 1 32.0 110.0 224 134.2 45.0 5.0 5.4116 93
-67 1 31.6 116.0 179 90.4 41.0 4.0 5.4723 100
-34 2 35.5 120.0 233 146.6 34.0 7.0 5.5683 101
-50 1 31.9 78.33 207 149.2 38.0 5.45 4.5951 84
-71 1 29.5 97.0 227 151.6 45.0 5.0 5.0239 108
-57 2 31.6 117.0 225 107.6 40.0 6.0 5.9584 113
-49 1 20.3 93.0 184 103.0 61.0 3.0 4.6052 93
-35 1 41.3 81.0 168 102.8 37.0 5.0 4.9488 94
-41 2 21.2 102.0 184 100.4 64.0 3.0 4.585 79
-70 2 24.1 82.33 194 149.2 31.0 6.26 4.2341 105
-52 1 23.0 107.0 179 123.7 42.5 4.21 4.1589 93
-60 1 25.6 78.0 195 95.4 91.0 2.0 3.7612 87
-62 1 22.5 125.0 215 99.0 98.0 2.0 4.4998 95
-44 2 38.2 123.0 201 126.6 44.0 5.0 5.0239 92
-28 2 19.2 81.0 155 94.6 51.0 3.0 3.8501 87
-58 2 29.0 85.0 156 109.2 36.0 4.0 3.989 86
-39 2 24.0 89.67 190 113.6 52.0 3.65 4.803999999999999 101
-34 2 20.6 98.0 183 92.0 83.0 2.0 3.6889 92
-65 1 26.3 70.0 244 166.2 51.0 5.0 4.8978 98
-66 2 34.6 115.0 204 139.4 36.0 6.0 4.9628 109
-51 1 23.4 87.0 220 108.8 93.0 2.0 4.5109 82
-50 2 29.2 119.0 162 85.2 54.0 3.0 4.7362 95
-59 2 27.2 107.0 158 102.0 39.0 4.0 4.4427 93
-52 1 27.0 78.33 134 73.0 44.0 3.05 4.4427 69
-69 2 24.5 108.0 243 136.4 40.0 6.0 5.8081 100
-53 1 24.1 105.0 184 113.4 46.0 4.0 4.8122 95
-47 2 25.3 98.0 173 105.6 44.0 4.0 4.7622 108
-52 1 28.8 113.0 280 174.0 67.0 4.0 5.273 86
-39 1 20.9 95.0 150 65.6 68.0 2.0 4.4067 95
-67 2 23.0 70.0 184 128.0 35.0 5.0 4.654 99
-59 2 24.1 96.0 170 98.6 54.0 3.0 4.4659 85
-51 2 28.1 106.0 202 122.2 55.0 4.0 4.8203 87
-23 2 18.0 78.0 171 96.0 48.0 4.0 4.9053 92
-68 1 25.9 93.0 253 181.2 53.0 5.0 4.5433 98
-44 1 21.5 85.0 157 92.2 55.0 3.0 3.8918 84
-60 2 24.3 103.0 141 86.6 33.0 4.0 4.6728 78
-52 1 24.5 90.0 198 129.0 29.0 7.0 5.2983 86
-38 1 21.3 72.0 165 60.2 88.0 2.0 4.4308 90
-61 1 25.8 90.0 280 195.4 55.0 5.0 4.9972 90
-68 2 24.8 101.0 221 151.4 60.0 4.0 3.8712 87
-28 2 31.5 83.0 228 149.4 38.0 6.0 5.3132 83
-65 2 33.5 102.0 190 126.2 35.0 5.0 4.9698 102
-69 1 28.1 113.0 234 142.8 52.0 4.0 5.2781 77
-51 1 24.3 85.33 153 71.6 71.0 2.15 3.9512 82
-29 1 35.0 98.33 204 142.6 50.0 4.08 4.0431 91
-55 2 23.5 93.0 177 126.8 41.0 4.0 3.8286 83
-34 2 30.0 83.0 185 107.2 53.0 3.0 4.8203 92
-67 1 20.7 83.0 170 99.8 59.0 3.0 4.0254 77
-49 1 25.6 76.0 161 99.8 51.0 3.0 3.9318 78
-55 2 22.9 81.0 123 67.2 41.0 3.0 4.3041 88
-59 2 25.1 90.0 163 101.4 46.0 4.0 4.3567 91
-53 1 33.2 82.67 186 106.8 46.0 4.04 5.112 102
-48 2 24.1 110.0 209 134.6 58.0 4.0 4.4067 100
-52 1 29.5 104.33 211 132.8 49.0 4.31 4.9836 98
-69 1 29.6 122.0 231 128.4 56.0 4.0 5.4510000000000005 86
-60 2 22.8 110.0 245 189.8 39.0 6.0 4.3944 88
-46 2 22.7 83.0 183 125.8 32.0 6.0 4.8363 75
-51 2 26.2 101.0 161 99.6 48.0 3.0 4.2047 88
-67 2 23.5 96.0 207 138.2 42.0 5.0 4.8978 111
-49 1 22.1 85.0 136 63.4 62.0 2.19 3.9703 72
-46 2 26.5 94.0 247 160.2 59.0 4.0 4.9345 111
-47 1 32.4 105.0 188 125.0 46.0 4.09 4.4427 99
-75 1 30.1 78.0 222 154.2 44.0 5.05 4.7791 97
-28 1 24.2 93.0 174 106.4 54.0 3.0 4.2195 84
-65 2 31.3 110.0 213 128.0 47.0 5.0 5.247000000000001 91
-42 1 30.1 91.0 182 114.8 49.0 4.0 4.5109 82
-51 1 24.5 79.0 212 128.6 65.0 3.0 4.5218 91
-53 2 27.7 95.0 190 101.8 41.0 5.0 5.4638 101
-54 1 23.2 110.67 238 162.8 48.0 4.96 4.9127 108
-73 1 27.0 102.0 211 121.0 67.0 3.0 4.7449 99
-54 1 26.8 108.0 176 80.6 67.0 3.0 4.9558 106
-42 1 29.2 93.0 249 174.2 45.0 6.0 5.0039 92
-75 1 31.2 117.67 229 138.8 29.0 7.9 5.7236 106
-55 2 32.1 112.67 207 92.4 25.0 8.28 6.1048 111
-68 2 25.7 109.0 233 112.6 35.0 7.0 6.0568 105
-57 1 26.9 98.0 246 165.2 38.0 7.0 5.3660000000000005 96
-48 1 31.4 75.33 242 151.6 38.0 6.37 5.5683 103
-61 2 25.6 85.0 184 116.2 39.0 5.0 4.9698 98
-69 1 37.0 103.0 207 131.4 55.0 4.0 4.6347 90
-38 1 32.6 77.0 168 100.6 47.0 4.0 4.625 96
-45 2 21.2 94.0 169 96.8 55.0 3.0 4.4543 102
-51 2 29.2 107.0 187 139.0 32.0 6.0 4.382 95
-71 2 24.0 84.0 138 85.8 39.0 4.0 4.1897 90
-57 1 36.1 117.0 181 108.2 34.0 5.0 5.2679 100
-56 2 25.8 103.0 177 114.4 34.0 5.0 4.9628 99
-32 2 22.0 88.0 137 78.6 48.0 3.0 3.9512 78
-50 1 21.9 91.0 190 111.2 67.0 3.0 4.0775 77
-43 1 34.3 84.0 256 172.6 33.0 8.0 5.5294 104
-54 2 25.2 115.0 181 120.0 39.0 5.0 4.7005 92
-31 1 23.3 85.0 190 130.8 43.0 4.0 4.3944 77
-56 1 25.7 80.0 244 151.6 59.0 4.0 5.118 95
-44 1 25.1 133.0 182 113.0 55.0 3.0 4.2485 84
-57 2 31.9 111.0 173 116.2 41.0 4.0 4.3694 87
-64 2 28.4 111.0 184 127.0 41.0 4.0 4.382 97
-43 1 28.1 121.0 192 121.0 60.0 3.0 4.0073 93
-19 1 25.3 83.0 225 156.6 46.0 5.0 4.7185 84
-71 2 26.1 85.0 220 152.4 47.0 5.0 4.6347 91
-50 2 28.0 104.0 282 196.8 44.0 6.0 5.3279 95
-59 2 23.6 73.0 180 107.4 51.0 4.0 4.6821 84
-57 1 24.5 93.0 186 96.6 71.0 3.0 4.5218 91
-49 2 21.0 82.0 119 85.4 23.0 5.0 3.9703 74
-41 2 32.0 126.0 198 104.2 49.0 4.0 5.4116 124
-25 2 22.6 85.0 130 71.0 48.0 3.0 4.0073 81
-52 2 19.7 81.0 152 53.4 82.0 2.0 4.4188 82
-34 1 21.2 84.0 254 113.4 52.0 5.0 6.0936 92
-42 2 30.6 101.0 269 172.2 50.0 5.0 5.4553 106
-28 2 25.5 99.0 162 101.6 46.0 4.0 4.2767 94
-47 2 23.3 90.0 195 125.8 54.0 4.0 4.3307 73
-32 2 31.0 100.0 177 96.2 45.0 4.0 5.1874 77
-43 1 18.5 87.0 163 93.6 61.0 2.67 3.7377 80
-59 2 26.9 104.0 194 126.6 43.0 5.0 4.803999999999999 106
-53 1 28.3 101.0 179 107.0 48.0 4.0 4.7875 101
-60 1 25.7 103.0 158 84.6 64.0 2.0 3.8501 97
-54 2 36.1 115.0 163 98.4 43.0 4.0 4.6821 101
-35 2 24.1 94.67 155 97.4 32.0 4.84 4.852 94
-49 2 25.8 89.0 182 118.6 39.0 5.0 4.803999999999999 115
-58 1 22.8 91.0 196 118.8 48.0 4.0 4.9836 115
-36 2 39.1 90.0 219 135.8 38.0 6.0 5.4205 103
-46 2 42.2 99.0 211 137.0 44.0 5.0 5.0106 99
-44 2 26.6 99.0 205 109.0 43.0 5.0 5.5797 111
-46 1 29.9 83.0 171 113.0 38.0 4.5 4.585 98
-54 1 21.0 78.0 188 107.4 70.0 3.0 3.9703 73
-63 2 25.5 109.0 226 103.2 46.0 5.0 5.9506 87
-41 2 24.2 90.0 199 123.6 57.0 4.0 4.5218 86
-28 1 25.4 93.0 141 79.0 49.0 3.0 4.1744 91
-19 1 23.2 75.0 143 70.4 52.0 3.0 4.6347 72
-61 2 26.1 126.0 215 129.8 57.0 4.0 4.9488 96
-48 1 32.7 93.0 276 198.6 43.0 6.42 5.1475 91
-54 2 27.3 100.0 200 144.0 33.0 6.0 4.7449 76
-53 2 26.6 93.0 185 122.4 36.0 5.0 4.8903 82
-48 1 22.8 101.0 110 41.6 56.0 2.0 4.1271 97
-53 1 28.8 111.67 145 87.2 46.0 3.15 4.0775 85
-29 2 18.1 73.0 158 99.0 41.0 4.0 4.4998 78
-62 1 32.0 88.0 172 69.0 38.0 4.0 5.7838 100
-50 2 23.7 92.0 166 97.0 52.0 3.0 4.4427 93
-58 2 23.6 96.0 257 171.0 59.0 4.0 4.9053 82
-55 2 24.6 109.0 143 76.4 51.0 3.0 4.3567 88
-54 1 22.6 90.0 183 104.2 64.0 3.0 4.3041 92
-36 1 27.8 73.0 153 104.4 42.0 4.0 3.4965 73
-63 2 24.1 111.0 184 112.2 44.0 4.0 4.9345 82
-47 2 26.5 70.0 181 104.8 63.0 3.0 4.1897 70
-51 2 32.8 112.0 202 100.6 37.0 5.0 5.7746 109
-42 1 19.9 76.0 146 83.2 55.0 3.0 3.6636 79
-37 2 23.6 94.0 205 138.8 53.0 4.0 4.1897 107
-28 1 22.1 82.0 168 100.6 54.0 3.0 4.2047 86
-58 1 28.1 111.0 198 80.6 31.0 6.0 6.0684 93
-32 1 26.5 86.0 184 101.6 53.0 4.0 4.9904 78
-25 2 23.5 88.0 143 80.8 55.0 3.0 3.5835 83
-63 1 26.0 85.67 155 78.2 46.0 3.37 5.037 97
-52 1 27.8 85.0 219 136.0 49.0 4.0 5.1358 75
-65 2 28.5 109.0 201 123.0 46.0 4.0 5.0752 96
-42 1 30.6 121.0 176 92.8 69.0 3.0 4.2627 89
-53 1 22.2 78.0 164 81.0 70.0 2.0 4.1744 101
-79 2 23.3 88.0 186 128.4 33.0 6.0 4.8122 102
-43 1 35.4 93.0 185 100.2 44.0 4.0 5.3181 101
-44 1 31.4 115.0 165 97.6 52.0 3.0 4.3438 89
-62 2 37.8 119.0 113 51.0 31.0 4.0 5.0434 84
-33 1 18.9 70.0 162 91.8 59.0 3.0 4.0254 58
-56 1 35.0 79.33 195 140.8 42.0 4.64 4.1109 96
-66 1 21.7 126.0 212 127.8 45.0 4.71 5.2781 101
-34 2 25.3 111.0 230 162.0 39.0 6.0 4.9767 90
-46 2 23.8 97.0 224 139.2 42.0 5.0 5.3660000000000005 81
-50 1 31.8 82.0 136 69.2 55.0 2.0 4.0775 85
-69 1 34.3 113.0 200 123.8 54.0 4.0 4.7095 112
-34 1 26.3 87.0 197 120.0 63.0 3.0 4.2485 96
-71 2 27.0 93.33 269 190.2 41.0 6.56 5.2417 93
-47 1 27.2 80.0 208 145.6 38.0 6.0 4.803999999999999 92
-41 1 33.8 123.33 187 127.0 45.0 4.16 4.3175 100
-34 1 33.0 73.0 178 114.6 51.0 3.49 4.1271 92
-51 1 24.1 87.0 261 175.6 69.0 4.0 4.4067 93
-43 1 21.3 79.0 141 78.8 53.0 3.0 3.8286 90
-55 1 23.0 94.67 190 137.6 38.0 5.0 4.2767 106
-59 2 27.9 101.0 218 144.2 38.0 6.0 5.1874 95
-27 2 33.6 110.0 246 156.6 57.0 4.0 5.0876 89
-51 2 22.7 103.0 217 162.4 30.0 7.0 4.8122 80
-49 2 27.4 89.0 177 113.0 37.0 5.0 4.9053 97
-27 1 22.6 71.0 116 43.4 56.0 2.0 4.4188 79
-57 2 23.2 107.33 231 159.4 41.0 5.63 5.0304 112
-39 2 26.9 93.0 136 75.4 48.0 3.0 4.1431 99
-62 2 34.6 120.0 215 129.2 43.0 5.0 5.3660000000000005 123
-37 1 23.3 88.0 223 142.0 65.0 3.4 4.3567 82
-46 1 21.1 80.0 205 144.4 42.0 5.0 4.5326 87
-68 2 23.5 101.0 162 85.4 59.0 3.0 4.4773 91
-51 1 31.5 93.0 231 144.0 49.0 4.7 5.2523 117
-41 1 20.8 86.0 223 128.2 83.0 3.0 4.0775 89
-53 1 26.5 97.0 193 122.4 58.0 3.0 4.1431 99
-45 1 24.2 83.0 177 118.4 45.0 4.0 4.2195 82
-33 1 19.5 80.0 171 85.4 75.0 2.0 3.9703 80
-60 2 28.2 112.0 185 113.8 42.0 4.0 4.9836 93
-47 2 24.9 75.0 225 166.0 42.0 5.0 4.4427 102
-60 2 24.9 99.67 162 106.6 43.0 3.77 4.1271 95
-36 1 30.0 95.0 201 125.2 42.0 4.79 5.1299 85
-36 1 19.6 71.0 250 133.2 97.0 3.0 4.5951 92
\ No newline at end of file
+37 2 27.7 93.0 180 119.4 30.0 6.0 5.0304 88
\ No newline at end of file
diff --git a/test/support/diabetes_target.csv b/test/support/diabetes_target.csv
index ce5e4a5e..c6715b81 100644
--- a/test/support/diabetes_target.csv
+++ b/test/support/diabetes_target.csv
@@ -47,396 +47,4 @@
 1.900000000000000000e+02
 1.420000000000000000e+02
 7.500000000000000000e+01
-1.420000000000000000e+02
-1.550000000000000000e+02
-2.250000000000000000e+02
-5.900000000000000000e+01
-1.040000000000000000e+02
-1.820000000000000000e+02
-1.280000000000000000e+02
-5.200000000000000000e+01
-3.700000000000000000e+01
-1.700000000000000000e+02
-1.700000000000000000e+02
-6.100000000000000000e+01
-1.440000000000000000e+02
-5.200000000000000000e+01
-1.280000000000000000e+02
-7.100000000000000000e+01
-1.630000000000000000e+02
-1.500000000000000000e+02
-9.700000000000000000e+01
-1.600000000000000000e+02
-1.780000000000000000e+02
-4.800000000000000000e+01
-2.700000000000000000e+02
-2.020000000000000000e+02
-1.110000000000000000e+02
-8.500000000000000000e+01
-4.200000000000000000e+01
-1.700000000000000000e+02
-2.000000000000000000e+02
-2.520000000000000000e+02
-1.130000000000000000e+02
-1.430000000000000000e+02
-5.100000000000000000e+01
-5.200000000000000000e+01
-2.100000000000000000e+02
-6.500000000000000000e+01
-1.410000000000000000e+02
-5.500000000000000000e+01
-1.340000000000000000e+02
-4.200000000000000000e+01
-1.110000000000000000e+02
-9.800000000000000000e+01
-1.640000000000000000e+02
-4.800000000000000000e+01
-9.600000000000000000e+01
-9.000000000000000000e+01
-1.620000000000000000e+02
-1.500000000000000000e+02
-2.790000000000000000e+02
-9.200000000000000000e+01
-8.300000000000000000e+01
-1.280000000000000000e+02
-1.020000000000000000e+02
-3.020000000000000000e+02
-1.980000000000000000e+02
-9.500000000000000000e+01
-5.300000000000000000e+01
-1.340000000000000000e+02
-1.440000000000000000e+02
-2.320000000000000000e+02
-8.100000000000000000e+01
-1.040000000000000000e+02
-5.900000000000000000e+01
-2.460000000000000000e+02
-2.970000000000000000e+02
-2.580000000000000000e+02
-2.290000000000000000e+02
-2.750000000000000000e+02
-2.810000000000000000e+02
-1.790000000000000000e+02
-2.000000000000000000e+02
-2.000000000000000000e+02
-1.730000000000000000e+02
-1.800000000000000000e+02
-8.400000000000000000e+01
-1.210000000000000000e+02
-1.610000000000000000e+02
-9.900000000000000000e+01
-1.090000000000000000e+02
-1.150000000000000000e+02
-2.680000000000000000e+02
-2.740000000000000000e+02
-1.580000000000000000e+02
-1.070000000000000000e+02
-8.300000000000000000e+01
-1.030000000000000000e+02
-2.720000000000000000e+02
-8.500000000000000000e+01
-2.800000000000000000e+02
-3.360000000000000000e+02
-2.810000000000000000e+02
-1.180000000000000000e+02
-3.170000000000000000e+02
-2.350000000000000000e+02
-6.000000000000000000e+01
-1.740000000000000000e+02
-2.590000000000000000e+02
-1.780000000000000000e+02
-1.280000000000000000e+02
-9.600000000000000000e+01
-1.260000000000000000e+02
-2.880000000000000000e+02
-8.800000000000000000e+01
-2.920000000000000000e+02
-7.100000000000000000e+01
-1.970000000000000000e+02
-1.860000000000000000e+02
-2.500000000000000000e+01
-8.400000000000000000e+01
-9.600000000000000000e+01
-1.950000000000000000e+02
-5.300000000000000000e+01
-2.170000000000000000e+02
-1.720000000000000000e+02
-1.310000000000000000e+02
-2.140000000000000000e+02
-5.900000000000000000e+01
-7.000000000000000000e+01
-2.200000000000000000e+02
-2.680000000000000000e+02
-1.520000000000000000e+02
-4.700000000000000000e+01
-7.400000000000000000e+01
-2.950000000000000000e+02
-1.010000000000000000e+02
-1.510000000000000000e+02
-1.270000000000000000e+02
-2.370000000000000000e+02
-2.250000000000000000e+02
-8.100000000000000000e+01
-1.510000000000000000e+02
-1.070000000000000000e+02
-6.400000000000000000e+01
-1.380000000000000000e+02
-1.850000000000000000e+02
-2.650000000000000000e+02
-1.010000000000000000e+02
-1.370000000000000000e+02
-1.430000000000000000e+02
-1.410000000000000000e+02
-7.900000000000000000e+01
-2.920000000000000000e+02
-1.780000000000000000e+02
-9.100000000000000000e+01
-1.160000000000000000e+02
-8.600000000000000000e+01
-1.220000000000000000e+02
-7.200000000000000000e+01
-1.290000000000000000e+02
-1.420000000000000000e+02
-9.000000000000000000e+01
-1.580000000000000000e+02
-3.900000000000000000e+01
-1.960000000000000000e+02
-2.220000000000000000e+02
-2.770000000000000000e+02
-9.900000000000000000e+01
-1.960000000000000000e+02
-2.020000000000000000e+02
-1.550000000000000000e+02
-7.700000000000000000e+01
-1.910000000000000000e+02
-7.000000000000000000e+01
-7.300000000000000000e+01
-4.900000000000000000e+01
-6.500000000000000000e+01
-2.630000000000000000e+02
-2.480000000000000000e+02
-2.960000000000000000e+02
-2.140000000000000000e+02
-1.850000000000000000e+02
-7.800000000000000000e+01
-9.300000000000000000e+01
-2.520000000000000000e+02
-1.500000000000000000e+02
-7.700000000000000000e+01
-2.080000000000000000e+02
-7.700000000000000000e+01
-1.080000000000000000e+02
-1.600000000000000000e+02
-5.300000000000000000e+01
-2.200000000000000000e+02
-1.540000000000000000e+02
-2.590000000000000000e+02
-9.000000000000000000e+01
-2.460000000000000000e+02
-1.240000000000000000e+02
-6.700000000000000000e+01
-7.200000000000000000e+01
-2.570000000000000000e+02
-2.620000000000000000e+02
-2.750000000000000000e+02
-1.770000000000000000e+02
-7.100000000000000000e+01
-4.700000000000000000e+01
-1.870000000000000000e+02
-1.250000000000000000e+02
-7.800000000000000000e+01
-5.100000000000000000e+01
-2.580000000000000000e+02
-2.150000000000000000e+02
-3.030000000000000000e+02
-2.430000000000000000e+02
-9.100000000000000000e+01
-1.500000000000000000e+02
-3.100000000000000000e+02
-1.530000000000000000e+02
-3.460000000000000000e+02
-6.300000000000000000e+01
-8.900000000000000000e+01
-5.000000000000000000e+01
-3.900000000000000000e+01
-1.030000000000000000e+02
-3.080000000000000000e+02
-1.160000000000000000e+02
-1.450000000000000000e+02
-7.400000000000000000e+01
-4.500000000000000000e+01
-1.150000000000000000e+02
-2.640000000000000000e+02
-8.700000000000000000e+01
-2.020000000000000000e+02
-1.270000000000000000e+02
-1.820000000000000000e+02
-2.410000000000000000e+02
-6.600000000000000000e+01
-9.400000000000000000e+01
-2.830000000000000000e+02
-6.400000000000000000e+01
-1.020000000000000000e+02
-2.000000000000000000e+02
-2.650000000000000000e+02
-9.400000000000000000e+01
-2.300000000000000000e+02
-1.810000000000000000e+02
-1.560000000000000000e+02
-2.330000000000000000e+02
-6.000000000000000000e+01
-2.190000000000000000e+02
-8.000000000000000000e+01
-6.800000000000000000e+01
-3.320000000000000000e+02
-2.480000000000000000e+02
-8.400000000000000000e+01
-2.000000000000000000e+02
-5.500000000000000000e+01
-8.500000000000000000e+01
-8.900000000000000000e+01
-3.100000000000000000e+01
-1.290000000000000000e+02
-8.300000000000000000e+01
-2.750000000000000000e+02
-6.500000000000000000e+01
-1.980000000000000000e+02
-2.360000000000000000e+02
-2.530000000000000000e+02
-1.240000000000000000e+02
-4.400000000000000000e+01
-1.720000000000000000e+02
-1.140000000000000000e+02
-1.420000000000000000e+02
-1.090000000000000000e+02
-1.800000000000000000e+02
-1.440000000000000000e+02
-1.630000000000000000e+02
-1.470000000000000000e+02
-9.700000000000000000e+01
-2.200000000000000000e+02
-1.900000000000000000e+02
-1.090000000000000000e+02
-1.910000000000000000e+02
-1.220000000000000000e+02
-2.300000000000000000e+02
-2.420000000000000000e+02
-2.480000000000000000e+02
-2.490000000000000000e+02
-1.920000000000000000e+02
-1.310000000000000000e+02
-2.370000000000000000e+02
-7.800000000000000000e+01
-1.350000000000000000e+02
-2.440000000000000000e+02
-1.990000000000000000e+02
-2.700000000000000000e+02
-1.640000000000000000e+02
-7.200000000000000000e+01
-9.600000000000000000e+01
-3.060000000000000000e+02
-9.100000000000000000e+01
-2.140000000000000000e+02
-9.500000000000000000e+01
-2.160000000000000000e+02
-2.630000000000000000e+02
-1.780000000000000000e+02
-1.130000000000000000e+02
-2.000000000000000000e+02
-1.390000000000000000e+02
-1.390000000000000000e+02
-8.800000000000000000e+01
-1.480000000000000000e+02
-8.800000000000000000e+01
-2.430000000000000000e+02
-7.100000000000000000e+01
-7.700000000000000000e+01
-1.090000000000000000e+02
-2.720000000000000000e+02
-6.000000000000000000e+01
-5.400000000000000000e+01
-2.210000000000000000e+02
-9.000000000000000000e+01
-3.110000000000000000e+02
-2.810000000000000000e+02
-1.820000000000000000e+02
-3.210000000000000000e+02
-5.800000000000000000e+01
-2.620000000000000000e+02
-2.060000000000000000e+02
-2.330000000000000000e+02
-2.420000000000000000e+02
-1.230000000000000000e+02
-1.670000000000000000e+02
-6.300000000000000000e+01
-1.970000000000000000e+02
-7.100000000000000000e+01
-1.680000000000000000e+02
-1.400000000000000000e+02
-2.170000000000000000e+02
-1.210000000000000000e+02
-2.350000000000000000e+02
-2.450000000000000000e+02
-4.000000000000000000e+01
-5.200000000000000000e+01
-1.040000000000000000e+02
-1.320000000000000000e+02
-8.800000000000000000e+01
-6.900000000000000000e+01
-2.190000000000000000e+02
-7.200000000000000000e+01
-2.010000000000000000e+02
-1.100000000000000000e+02
-5.100000000000000000e+01
-2.770000000000000000e+02
-6.300000000000000000e+01
-1.180000000000000000e+02
-6.900000000000000000e+01
-2.730000000000000000e+02
-2.580000000000000000e+02
-4.300000000000000000e+01
-1.980000000000000000e+02
-2.420000000000000000e+02
-2.320000000000000000e+02
-1.750000000000000000e+02
-9.300000000000000000e+01
-1.680000000000000000e+02
-2.750000000000000000e+02
-2.930000000000000000e+02
-2.810000000000000000e+02
-7.200000000000000000e+01
-1.400000000000000000e+02
-1.890000000000000000e+02
-1.810000000000000000e+02
-2.090000000000000000e+02
-1.360000000000000000e+02
-2.610000000000000000e+02
-1.130000000000000000e+02
-1.310000000000000000e+02
-1.740000000000000000e+02
-2.570000000000000000e+02
-5.500000000000000000e+01
-8.400000000000000000e+01
-4.200000000000000000e+01
-1.460000000000000000e+02
-2.120000000000000000e+02
-2.330000000000000000e+02
-9.100000000000000000e+01
-1.110000000000000000e+02
-1.520000000000000000e+02
-1.200000000000000000e+02
-6.700000000000000000e+01
-3.100000000000000000e+02
-9.400000000000000000e+01
-1.830000000000000000e+02
-6.600000000000000000e+01
-1.730000000000000000e+02
-7.200000000000000000e+01
-4.900000000000000000e+01
-6.400000000000000000e+01
-4.800000000000000000e+01
-1.780000000000000000e+02
-1.040000000000000000e+02
-1.320000000000000000e+02
-2.200000000000000000e+02
-5.700000000000000000e+01
\ No newline at end of file
+1.420000000000000000e+02
\ No newline at end of file
diff --git a/test/support/scholar_case.ex b/test/support/scholar_case.ex
index 2d355249..ad10762a 100644
--- a/test/support/scholar_case.ex
+++ b/test/support/scholar_case.ex
@@ -195,6 +195,7 @@ defmodule Scholar.Case do
     {x_train, x_test, y_train, y_test}
   end
 
+  # https://scikit-learn.org/stable/datasets/toy_dataset.html#diabetes-dataset
   def diabetes_data do
     x =
       File.read!("./test/support/diabetes_data_raw.csv")

From e4ed46a9b062c608f9df653f5f6cd61208c2001c Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 21:06:55 +0200
Subject: [PATCH 78/90] test if jit compilable

---
 .../linear/bayesian_ridge_regression_test.exs | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 687054b8..0b0101e6 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -5,6 +5,24 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
   alias Scholar.Linear.RidgeRegression
   doctest BayesianRidgeRegression
 
+  test "bayesian ridge is jit compilable" do
+    x = Nx.tensor([[1], [2], [6], [8], [10]])
+    y = Nx.tensor([1, 2, 6, 8, 10])
+    alpha_1 = 0.1
+    alpha_2 = 0.1
+    lambda_1 = 0.1
+    lambda_2 = 0.1    
+    Nx.Defn.jit(&BayesianRidgeRegression.fit/3).(x, y,
+      alpha_1: alpha_1,
+      alpha_2: alpha_2,
+      lambda_1: lambda_1,
+      lambda_2: lambda_2,
+      fit_intercept?: true,
+      compute_scores?: true,
+      iterations: 1)
+    assert true
+  end
+
   test "toy bayesian ridge" do
     x = Nx.tensor([[1], [2], [6], [8], [10]])
     y = Nx.tensor([1, 2, 6, 8, 10])
@@ -89,7 +107,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     assert_all_close(expected, predicted, atol: 0.01)
   end
 
-  test "constant inputs: variance" do
+  test "constant inputs: variance is constant" do
     key = Nx.Random.key(42)
     n_samples = 15
     n_features = 10

From 95d673fd9ca78f4818227147add6aeff8f206caf Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Fri, 5 Apr 2024 21:07:42 +0200
Subject: [PATCH 79/90] formatter run

---
 .../linear/bayesian_ridge_regression.ex       | 46 ++++++++++++++-----
 .../linear/bayesian_ridge_regression_test.exs | 17 +++++--
 2 files changed, 48 insertions(+), 15 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index e1bb0404..193f6633 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -4,8 +4,28 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   import Scholar.Shared
 
   @derive {Nx.Container,
-           containers: [:coefficients, :intercept, :alpha, :lambda, :sigma, :rmse, :iterations, :has_converged, :scores]}
-  defstruct [:coefficients, :intercept, :alpha, :lambda, :sigma, :rmse, :iterations, :has_converged, :scores]
+           containers: [
+             :coefficients,
+             :intercept,
+             :alpha,
+             :lambda,
+             :sigma,
+             :rmse,
+             :iterations,
+             :has_converged,
+             :scores
+           ]}
+  defstruct [
+    :coefficients,
+    :intercept,
+    :alpha,
+    :lambda,
+    :sigma,
+    :rmse,
+    :iterations,
+    :has_converged,
+    :scores
+  ]
 
   opts = [
     iterations: [
@@ -46,7 +66,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       If set to `true`, the log marginal likelihood will be computed
       at each iteration of the algorithm.
       """
-    ],    
+    ],
     alpha_init: [
       type: {:custom, Scholar.Options, :non_negative_number, []},
       doc: ~S"""
@@ -132,7 +152,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {lambda, opts} = Keyword.pop!(opts, :lambda_init)
     lambda = Nx.tensor(lambda, type: x_type)
 
-    scores = Nx.tensor(:nan)
+    scores =
+      Nx.tensor(:nan)
       |> Nx.broadcast({opts[:iterations] + 1})
       |> Nx.as_type(x_type)
 
@@ -155,7 +176,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       rmse: rmse,
       iterations: iterations,
       has_converged: has_converged,
-      scores: scores,
+      scores: scores
     }
   end
 
@@ -217,11 +238,12 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
                 lambda_1,
                 lambda_2
               )
+
             Nx.put_slice(scores, [iter], Nx.new_axis(new_score, -1))
           else
             scores
           end
-        
+
         gamma = Nx.sum(alpha * eigenvals / (lambda + alpha * eigenvals))
         lambda = (gamma + 2 * lambda_1) / (Nx.sum(coef ** 2) + 2 * lambda_2)
         alpha = (n_samples - gamma + 2 * alpha_1) / (rmse + 2 * alpha_2)
@@ -236,7 +258,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       end
 
     intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
-    scaled_sigma = Nx.dot(vh, [0],  vh / Nx.new_axis(eigenvals + lambda / alpha, -1), [0])
+    scaled_sigma = Nx.dot(vh, [0], vh / Nx.new_axis(eigenvals + lambda / alpha, -1), [0])
     sigma = scaled_sigma / alpha
     {coef, intercept, alpha, lambda, rmse, iter, has_converged, scores, sigma}
   end
@@ -253,7 +275,8 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
           alpha,
           lambda
         ) do
-    scaled_eigens = eigenvals + lambda / alpha    
+    scaled_eigens = eigenvals + lambda / alpha
+
     coef =
       if n_samples > n_features do
         regularization = vh / Nx.new_axis(scaled_eigens, -1)
@@ -263,13 +286,13 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         regularization = u / scaled_eigens
         reg_transpose = Nx.dot(regularization, Nx.dot(u, [0], y, [0]))
         Nx.dot(x, [0], reg_transpose, [0])
-    end
+      end
 
     error = y - Nx.dot(x, coef)
     squared_error = error ** 2
     rmse = Nx.sum(squared_error)
 
-    {coef, rmse}    
+    {coef, rmse}
   end
 
   defnp log_marginal_likelihood(
@@ -290,8 +313,9 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         -1 * Nx.sum(Nx.log(lambda + alpha * eigenvals))
       else
         broad_lambda = Nx.broadcast(lambda, {n_samples})
-        -1 * Nx.sum(Nx.log(broad_lambda + (alpha * eigenvals)))
+        -1 * Nx.sum(Nx.log(broad_lambda + alpha * eigenvals))
       end
+
     score_lambda = lambda_1 * Nx.log(lambda) - lambda_2 * lambda
     score_alpha = alpha_1 * Nx.log(alpha) - alpha_2 * alpha
 
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 0b0101e6..05c8a50a 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -11,7 +11,8 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     alpha_1 = 0.1
     alpha_2 = 0.1
     lambda_1 = 0.1
-    lambda_2 = 0.1    
+    lambda_2 = 0.1
+
     Nx.Defn.jit(&BayesianRidgeRegression.fit/3).(x, y,
       alpha_1: alpha_1,
       alpha_2: alpha_2,
@@ -19,7 +20,9 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
       lambda_2: lambda_2,
       fit_intercept?: true,
       compute_scores?: true,
-      iterations: 1)
+      iterations: 1
+    )
+
     assert true
   end
 
@@ -47,7 +50,13 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     y = Nx.tensor([1, 2, 3, 2, 0, 4, 5])
     w = Nx.tensor([4, 3, 3, 1, 1, 2, 3])
     brr = BayesianRidgeRegression.fit(x, y, sample_weights: w)
-    rr = RidgeRegression.fit(x, y, alpha: Nx.to_number(brr.lambda) / Nx.to_number(brr.alpha), sample_weights: w)
+
+    rr =
+      RidgeRegression.fit(x, y,
+        alpha: Nx.to_number(brr.lambda) / Nx.to_number(brr.alpha),
+        sample_weights: w
+      )
+
     assert_all_close(brr.coefficients, rr.coefficients, atol: 1.0e-2)
     assert_all_close(brr.intercept, rr.intercept, atol: 1.0e-2)
   end
@@ -74,7 +83,7 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
         compute_scores?: true,
         iterations: 1
       )
-    
+
     first_score = brr.scores[0]
     assert_all_close(score, first_score, rtol: 0.05)
   end

From 5823c97943a602f90d78ea8dbdc3c71378a0ef60 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sun, 14 Apr 2024 10:26:47 +0200
Subject: [PATCH 80/90] remove multi_weights duplication

---
 lib/scholar/options.ex | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/lib/scholar/options.ex b/lib/scholar/options.ex
index d47489af..e1ac99c9 100644
--- a/lib/scholar/options.ex
+++ b/lib/scholar/options.ex
@@ -93,16 +93,6 @@ defmodule Scholar.Options do
     end
   end
 
-  def multi_weights(weights) do
-    if is_nil(weights) or
-         (Nx.is_tensor(weights) and Nx.rank(weights) > 1) do
-      {:ok, weights}
-    else
-      {:error,
-       "expected weights to be a tensor with rank greater than 1, got: #{inspect(weights)}"}
-    end
-  end
-
   def key(key) do
     if Nx.is_tensor(key) and Nx.type(key) == {:u, 32} and Nx.shape(key) == {2} do
       {:ok, key}

From 58f36cec3b8ced428f7b6c1a5575a14cec979b28 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sun, 14 Apr 2024 11:47:44 +0200
Subject: [PATCH 81/90] wrote docs

---
 .../linear/bayesian_ridge_regression.ex       | 125 +++++++++++++++++-
 .../linear/bayesian_ridge_regression_test.exs |   4 -
 2 files changed, 119 insertions(+), 10 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 193f6633..b8f7a94d 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -1,4 +1,63 @@
 defmodule Scholar.Linear.BayesianRidgeRegression do
+  @moduledoc ~S"""
+  Bayesian ridge regression: A fully probabilistic linear model with parameter regularization.
+
+  In order to obtain a fully probabilistic linear model,
+  we declare the precision parameter in the model: $\alpha$,
+  This parameter describes the dispersion of the data around the mean.
+
+  $$
+  p(y | X, w, \alpha) = \mathcal{N}(y | Xw, \alpha^{-1})
+  $$
+
+  Where:
+  * $X$ is an input data
+
+  * $y$ is an input target
+
+  * $w$ is the model weights matrix
+
+  * $\alpha$ is the precision parameter of the target and $\alpha^{-1} = \sigma^{2}$, the variance.
+
+  In order to obtain a fully probabilistic regularized linear model,
+  we declare the distribution of the model weights matrix
+  with it's corresponding precision parameter:
+
+  $$
+  p(w | \lambda) = \mathcal{N}(w, \lambda^{-1})
+  $$
+
+  Where $\lambda$ is the precision parameter of the weights matrix.
+
+  Both $\alpha$ and $\lambda$ are choosen to have prior gamma distributions,
+  controlled through hyperparameters $\alpha_1$, $\alpha_2$, $\lambda_1$, $\lambda_2$.
+  These parameters are set by default to non-informative
+  $\alpha_1 = \alpha_2 = \lambda_1 = \lambda_2 = 1^{-6}$.
+
+  This model is similar to the classical ridge regression.
+  Confusingly the classical ridge regression's $\alpha$ parameter is the Bayesian ridge's $\lambda$ parameter.
+
+  Other than that, the differences between alorithms are:
+  * The matrix weight regularization parameter is estimated from data,
+  * The precision of the target is estimated.
+
+  As such, Bayesian ridge is more flexible to the data at hand.
+  These features come at higher computational cost.
+
+  This implementation is ported from Python's scikit-learn.
+  It uses the algorithm described in (Tipping, 2001)
+  and regularization parameters are updated as by (MacKay, 1992).
+  References
+  ----------
+  D. J. C. MacKay, Bayesian Interpolation, Computation and Neural Systems,
+  Vol. 4, No. 3, 1992.
+
+  M. E. Tipping, Sparse Bayesian Learning and the Relevance Vector Machine,
+  Journal of Machine Learning Research, Vol. 1, 2001.
+
+  Pedregosa et al., Scikit-learn: Machine Learning in Python,
+  JMLR 12, pp. 2825-2830, 2011.
+  """
   require Nx
   import Nx.Defn
   import Scholar.Shared
@@ -10,7 +69,6 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
              :alpha,
              :lambda,
              :sigma,
-             :rmse,
              :iterations,
              :has_converged,
              :scores
@@ -21,7 +79,6 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     :alpha,
     :lambda,
     :sigma,
-    :rmse,
     :iterations,
     :has_converged,
     :scores
@@ -125,6 +182,51 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   ]
 
   @opts_schema NimbleOptions.new!(opts)
+
+  @doc """
+  Fits a Bayesian ridge model for sample inputs `x` and
+  sample targets `y`.
+
+  ## Options
+
+  #{NimbleOptions.docs(@opts_schema)}
+
+  ## Return Values
+
+    The function returns a struct with the following parameters:
+
+    * `:coefficients` - Estimated coefficients for the linear regression problem.
+
+    * `:intercept` - Independent term in the linear model.
+
+    * `:alpha` - Estimated precision of the noise.
+
+    * `:lambda` - Estimated precision of the weights.
+
+    * `:sigma` - Estimated variance covariance matrix of weights with shape (n_features, n_features).
+
+    * `:iterations` - How many times the optimization algorithm was computed.
+
+    * `:has_converged` - Whether the coefficients converged during the optimization algorithm.
+
+    * `:scores` - Value of the log marginal likelihood at each iteration during the optimization.
+
+  ## Examples
+
+      iex> x = Nx.tensor([[1], [2], [6], [8], [10]])
+      iex> y = Nx.tensor([1, 2, 6, 8, 10])
+      iex> model = Scholar.Linear.BayesianRidgeRegression.fit(x, y)
+      iex> model.coefficients
+      #Nx.Tensor<
+        f32[1]
+        [0.9932512044906616]
+      >
+      iex> model.intercept
+      #Nx.Tensor<
+        f32
+        0.03644371032714844
+      >
+  """
   deftransform fit(x, y, opts \\ []) do
     opts = NimbleOptions.validate!(opts, @opts_schema)
 
@@ -157,7 +259,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       |> Nx.broadcast({opts[:iterations] + 1})
       |> Nx.as_type(x_type)
 
-    {coefficients, intercept, alpha, lambda, rmse, iterations, has_converged, scores, sigma} =
+    {coefficients, intercept, alpha, lambda, iterations, has_converged, scores, sigma} =
       fit_n(x, y, alpha, lambda, sample_weights, scores, opts)
 
     scores =
@@ -173,7 +275,6 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
       alpha: alpha,
       lambda: lambda,
       sigma: sigma,
-      rmse: rmse,
       iterations: iterations,
       has_converged: has_converged,
       scores: scores
@@ -218,7 +319,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {n_samples, n_features} = Nx.shape(x)
     {coef, rmse} = update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
 
-    {{coef, alpha, lambda, rmse, iter, has_converged, scores}, _} =
+    {{coef, alpha, lambda, _rmse, iter, has_converged, scores}, _} =
       while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0), scores = scores},
              {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}},
             iter <= iterations and not has_converged do
@@ -260,7 +361,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     intercept = set_intercept(coef, x_offset, y_offset, opts[:fit_intercept?])
     scaled_sigma = Nx.dot(vh, [0], vh / Nx.new_axis(eigenvals + lambda / alpha, -1), [0])
     sigma = scaled_sigma / alpha
-    {coef, intercept, alpha, lambda, rmse, iter, has_converged, scores, sigma}
+    {coef, intercept, alpha, lambda, iter, has_converged, scores, sigma}
   end
 
   defnp update_coef(
@@ -329,6 +430,18 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     score_alpha + score_lambda + score
   end
 
+  @doc """
+  Makes predictions with the given `model` on input `x`.
+  ## Examples
+
+      iex> x = Nx.tensor([[1], [2], [6], [8], [10]])
+      iex> y = Nx.tensor([1, 2, 6, 8, 10])
+      iex> model = Scholar.Linear.BayesianRidgeRegression.fit(x, y)
+      iex> Scholar.Linear.BayesianRidgeRegression.predict(model, Nx.tensor([[1], [3], [4]]))
+      Nx.tensor(
+        [1.02969491481781, 3.0161972045898438, 4.009448528289795]  
+      )  
+  """
   deftransform predict(%__MODULE__{coefficients: coeff, intercept: intercept} = _model, x) do
     predict_n(coeff, intercept, x)
   end
diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 05c8a50a..4398828d 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -127,8 +127,4 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
     check = Nx.less_equal(brr.sigma, 0.01)
     assert Nx.all(check) == Nx.u8(1)
   end
-
-  test "write docs" do
-    assert false
-  end
 end

From 98c51cc5fd9923ad24969ffccc7a2e8d908fc1d5 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Sun, 21 Apr 2024 10:44:46 +0200
Subject: [PATCH 82/90] Update lib/scholar/linear/bayesian_ridge_regression.ex
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Krsto Proroković <krstopro@yahoo.com>
---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index b8f7a94d..187ff44d 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -320,7 +320,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
     {coef, rmse} = update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
 
     {{coef, alpha, lambda, _rmse, iter, has_converged, scores}, _} =
-      while {{coef, rmse, alpha, lambda, iter = 0, has_converged = Nx.u8(0), scores = scores},
+while {{coef, rmse, alpha, lambda, iter = Nx.u64(0), has_converged = Nx.u8(0), scores = scores},
              {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}},
             iter <= iterations and not has_converged do
         scores =

From 52b208486a93a9eeffc57ff8cb5d5c2f3bf1f244 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Sun, 21 Apr 2024 10:44:57 +0200
Subject: [PATCH 83/90] Update lib/scholar/linear/bayesian_ridge_regression.ex
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Krsto Proroković <krstopro@yahoo.com>
---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 187ff44d..596bc7f5 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -47,7 +47,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   This implementation is ported from Python's scikit-learn.
   It uses the algorithm described in (Tipping, 2001)
   and regularization parameters are updated as by (MacKay, 1992).
-  References
+References:
   ----------
   D. J. C. MacKay, Bayesian Interpolation, Computation and Neural Systems,
   Vol. 4, No. 3, 1992.

From 4570fc559eb708b58fc7f0be94ce9d8ce84b9423 Mon Sep 17 00:00:00 2001
From: JoaquinIglesiasTurina <iglesiastj@gmail.com>
Date: Sun, 21 Apr 2024 10:45:04 +0200
Subject: [PATCH 84/90] Update lib/scholar/linear/bayesian_ridge_regression.ex
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Krsto Proroković <krstopro@yahoo.com>
---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 596bc7f5..3f7525a2 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -457,7 +457,7 @@ while {{coef, rmse, alpha, lambda, iter = Nx.u64(0), has_converged = Nx.u8(0), s
         {scalar * x, scalar * y}
 
       _ ->
-        scale = sample_weights |> Nx.sqrt() |> Nx.make_diagonal()
+scale = sample_weights |> Nx.sqrt() |> Nx.new_axis(1)
         {Nx.dot(scale, x), Nx.dot(scale, y)}
     end
   end

From 06eea91f81447ab55f7a886ddb213eeb17fb3427 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sun, 21 Apr 2024 10:44:02 +0200
Subject: [PATCH 85/90] fix references

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 3f7525a2..89140a5b 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -47,8 +47,9 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   This implementation is ported from Python's scikit-learn.
   It uses the algorithm described in (Tipping, 2001)
   and regularization parameters are updated as by (MacKay, 1992).
-References:
-  ----------
+  
+  References:
+
   D. J. C. MacKay, Bayesian Interpolation, Computation and Neural Systems,
   Vol. 4, No. 3, 1992.
 
@@ -320,7 +321,8 @@ References:
     {coef, rmse} = update_coef(x, y, n_samples, n_features, xt_y, u, vh, eigenvals, alpha, lambda)
 
     {{coef, alpha, lambda, _rmse, iter, has_converged, scores}, _} =
-while {{coef, rmse, alpha, lambda, iter = Nx.u64(0), has_converged = Nx.u8(0), scores = scores},
+      while {{coef, rmse, alpha, lambda, iter = Nx.u64(0), has_converged = Nx.u8(0),
+              scores = scores},
              {x, y, xt_y, u, s, vh, eigenvals, alpha_1, alpha_2, lambda_1, lambda_2, iterations}},
             iter <= iterations and not has_converged do
         scores =
@@ -457,7 +459,7 @@ while {{coef, rmse, alpha, lambda, iter = Nx.u64(0), has_converged = Nx.u8(0), s
         {scalar * x, scalar * y}
 
       _ ->
-scale = sample_weights |> Nx.sqrt() |> Nx.new_axis(1)
+        scale = sample_weights |> Nx.sqrt() |> Nx.new_axis(1)
         {Nx.dot(scale, x), Nx.dot(scale, y)}
     end
   end

From ccc2358ffe6472fdbb49fdc97df47712601536a3 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sun, 21 Apr 2024 11:20:12 +0200
Subject: [PATCH 86/90] undo scale suggestion

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 89140a5b..1ac1158e 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -459,7 +459,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
         {scalar * x, scalar * y}
 
       _ ->
-        scale = sample_weights |> Nx.sqrt() |> Nx.new_axis(1)
+        scale = sample_weights |> Nx.sqrt() |> Nx.make_diagonal()
         {Nx.dot(scale, x), Nx.dot(scale, y)}
     end
   end

From 8253e0bce25327f06575897afe317928fd72d849 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sun, 21 Apr 2024 11:20:57 +0200
Subject: [PATCH 87/90] remove jit compilable test case

---
 .../linear/bayesian_ridge_regression_test.exs | 21 -------------------
 1 file changed, 21 deletions(-)

diff --git a/test/scholar/linear/bayesian_ridge_regression_test.exs b/test/scholar/linear/bayesian_ridge_regression_test.exs
index 4398828d..7f4e36fc 100644
--- a/test/scholar/linear/bayesian_ridge_regression_test.exs
+++ b/test/scholar/linear/bayesian_ridge_regression_test.exs
@@ -5,27 +5,6 @@ defmodule Scholar.Linear.BayesianRidgeRegressionTest do
   alias Scholar.Linear.RidgeRegression
   doctest BayesianRidgeRegression
 
-  test "bayesian ridge is jit compilable" do
-    x = Nx.tensor([[1], [2], [6], [8], [10]])
-    y = Nx.tensor([1, 2, 6, 8, 10])
-    alpha_1 = 0.1
-    alpha_2 = 0.1
-    lambda_1 = 0.1
-    lambda_2 = 0.1
-
-    Nx.Defn.jit(&BayesianRidgeRegression.fit/3).(x, y,
-      alpha_1: alpha_1,
-      alpha_2: alpha_2,
-      lambda_1: lambda_1,
-      lambda_2: lambda_2,
-      fit_intercept?: true,
-      compute_scores?: true,
-      iterations: 1
-    )
-
-    assert true
-  end
-
   test "toy bayesian ridge" do
     x = Nx.tensor([[1], [2], [6], [8], [10]])
     y = Nx.tensor([1, 2, 6, 8, 10])

From 4670db63cd20f1dfdcdccd412bc9659bc63a39af Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sun, 21 Apr 2024 11:22:26 +0200
Subject: [PATCH 88/90] formatter

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 1ac1158e..2664a572 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -47,7 +47,7 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   This implementation is ported from Python's scikit-learn.
   It uses the algorithm described in (Tipping, 2001)
   and regularization parameters are updated as by (MacKay, 1992).
-  
+
   References:
 
   D. J. C. MacKay, Bayesian Interpolation, Computation and Neural Systems,

From f0e87a72c27a10790e4aed6aab99619bb4924f46 Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sun, 21 Apr 2024 16:59:56 +0200
Subject: [PATCH 89/90] refactored rescale fuction

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 2664a572..7d06dff9 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -453,15 +453,12 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   # Implements sample weighting by rescaling inputs and
   # targets by sqrt(sample_weight).
   defnp rescale(x, y, sample_weights) do
-    case Nx.shape(sample_weights) do
-      {} = scalar ->
-        scalar = Nx.sqrt(scalar)
-        {scalar * x, scalar * y}
-
-      _ ->
-        scale = sample_weights |> Nx.sqrt() |> Nx.make_diagonal()
-        {Nx.dot(scale, x), Nx.dot(scale, y)}
+    factor = Nx.sqrt(sample_weights)
+    x_scaled = case Nx.shape(factor) do
+      {} -> factor * x
+      _ -> Nx.new_axis(factor, 1) * x
     end
+    {x_scaled, factor * y}    
   end
 
   defnp set_intercept(coeff, x_offset, y_offset, fit_intercept?) do

From db92d3968b1c1a65e45e6de62d800d11a488e35a Mon Sep 17 00:00:00 2001
From: Joaquin Iglesias Turina <iglesiastj@gmail.com>
Date: Sun, 21 Apr 2024 17:00:39 +0200
Subject: [PATCH 90/90] formatter

---
 lib/scholar/linear/bayesian_ridge_regression.ex | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/scholar/linear/bayesian_ridge_regression.ex b/lib/scholar/linear/bayesian_ridge_regression.ex
index 7d06dff9..22ef0e4d 100644
--- a/lib/scholar/linear/bayesian_ridge_regression.ex
+++ b/lib/scholar/linear/bayesian_ridge_regression.ex
@@ -454,11 +454,14 @@ defmodule Scholar.Linear.BayesianRidgeRegression do
   # targets by sqrt(sample_weight).
   defnp rescale(x, y, sample_weights) do
     factor = Nx.sqrt(sample_weights)
-    x_scaled = case Nx.shape(factor) do
-      {} -> factor * x
-      _ -> Nx.new_axis(factor, 1) * x
-    end
-    {x_scaled, factor * y}    
+
+    x_scaled =
+      case Nx.shape(factor) do
+        {} -> factor * x
+        _ -> Nx.new_axis(factor, 1) * x
+      end
+
+    {x_scaled, factor * y}
   end
 
   defnp set_intercept(coeff, x_offset, y_offset, fit_intercept?) do