diff --git a/README.md b/README.md
index 50a4ffdf..ddf11304 100644
--- a/README.md
+++ b/README.md
@@ -587,13 +587,111 @@ trainer.fit(
 )
 ```
 
-**7. Tabular with a multi-target loss**
+**7. A two-tower model**
+
+This is a popular model in the context of recommendation systems. Let's say we
+have a tabular dataset formed my triples (user features, item features,
+target). We can create a two-tower model where the user and item features are
+passed through two separate models and then "fused" via a dot product.
+
+<p align="center">
+  <img width="350" src="docs/figures/arch_7.png">
+</p>
+
+
+```python
+import numpy as np
+import pandas as pd
+
+from pytorch_widedeep import Trainer
+from pytorch_widedeep.preprocessing import TabPreprocessor
+from pytorch_widedeep.models import TabMlp, WideDeep, ModelFuser
+
+# Let's create the interaction dataset
+# user_features dataframe
+np.random.seed(42)
+user_ids = np.arange(1, 101)
+ages = np.random.randint(18, 60, size=100)
+genders = np.random.choice(["male", "female"], size=100)
+locations = np.random.choice(["city_a", "city_b", "city_c", "city_d"], size=100)
+user_features = pd.DataFrame(
+    {"id": user_ids, "age": ages, "gender": genders, "location": locations}
+)
+
+# item_features dataframe
+item_ids = np.arange(1, 101)
+prices = np.random.uniform(10, 500, size=100).round(2)
+colors = np.random.choice(["red", "blue", "green", "black"], size=100)
+categories = np.random.choice(["electronics", "clothing", "home", "toys"], size=100)
+
+item_features = pd.DataFrame(
+    {"id": item_ids, "price": prices, "color": colors, "category": categories}
+)
+
+# Interactions dataframe
+interaction_user_ids = np.random.choice(user_ids, size=1000)
+interaction_item_ids = np.random.choice(item_ids, size=1000)
+purchased = np.random.choice([0, 1], size=1000, p=[0.7, 0.3])
+interactions = pd.DataFrame(
+    {
+        "user_id": interaction_user_ids,
+        "item_id": interaction_item_ids,
+        "purchased": purchased,
+    }
+)
+user_item_purchased = interactions.merge(
+    user_features, left_on="user_id", right_on="id"
+).merge(item_features, left_on="item_id", right_on="id")
+
+# Users
+tab_preprocessor_user = TabPreprocessor(
+    cat_embed_cols=["gender", "location"],
+    continuous_cols=["age"],
+)
+X_user = tab_preprocessor_user.fit_transform(user_item_purchased)
+tab_mlp_user = TabMlp(
+    column_idx=tab_preprocessor_user.column_idx,
+    cat_embed_input=tab_preprocessor_user.cat_embed_input,
+    continuous_cols=["age"],
+    mlp_hidden_dims=[16, 8],
+    mlp_dropout=[0.2, 0.2],
+)
+
+# Items
+tab_preprocessor_item = TabPreprocessor(
+    cat_embed_cols=["color", "category"],
+    continuous_cols=["price"],
+)
+X_item = tab_preprocessor_item.fit_transform(user_item_purchased)
+tab_mlp_item = TabMlp(
+    column_idx=tab_preprocessor_item.column_idx,
+    cat_embed_input=tab_preprocessor_item.cat_embed_input,
+    continuous_cols=["price"],
+    mlp_hidden_dims=[16, 8],
+    mlp_dropout=[0.2, 0.2],
+)
+
+two_tower_model = ModelFuser([tab_mlp_user, tab_mlp_item], fusion_method="dot")
+
+model = WideDeep(deeptabular=two_tower_model)
+
+trainer = Trainer(model, objective="binary")
+
+trainer.fit(
+    X_tab=[X_user, X_item],
+    target=interactions.purchased.values,
+    n_epochs=1,
+    batch_size=32,
+)
+```
+
+**8. Tabular with a multi-target loss**
 
 This one is "a bonus" to illustrate the use of multi-target losses, more than
 actually a different architecture.
 
 <p align="center">
-  <img width="200" src="docs/figures/arch_7.png">
+  <img width="200" src="docs/figures/arch_8.png">
 </p>
 
 
diff --git a/VERSION b/VERSION
index fdd3be6d..266146b8 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.6.2
+1.6.3
diff --git a/docs/examples.rst b/docs/examples.rst
index b78fd394..489560fa 100644
--- a/docs/examples.rst
+++ b/docs/examples.rst
@@ -17,5 +17,4 @@ them to address different problems
 * `HyperParameter Tuning With RayTune <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/10_hyperParameter_tuning_w_raytune_n_wnb.ipynb>`__
 * `Model Uncertainty Prediction <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/13_Model_Uncertainty_prediction.ipynb>`__
 * `Bayesian Models <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/14_bayesian_models.ipynb>`__
-* `Deep Imbalanced Regression <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/15_DIR-LDS_and_FDS.ipynb>`__
 
diff --git a/docs/figures/arch_7.png b/docs/figures/arch_7.png
index 16a0bfdd..c7780536 100644
Binary files a/docs/figures/arch_7.png and b/docs/figures/arch_7.png differ
diff --git a/docs/figures/arch_8.png b/docs/figures/arch_8.png
new file mode 100644
index 00000000..16a0bfdd
Binary files /dev/null and b/docs/figures/arch_8.png differ
diff --git a/examples/notebooks/15_DIR-LDS_and_FDS.ipynb b/examples/notebooks/15_DIR-LDS_and_FDS.ipynb
deleted file mode 100644
index 09770d99..00000000
--- a/examples/notebooks/15_DIR-LDS_and_FDS.ipynb
+++ /dev/null
@@ -1,847 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "01a4f48f",
-   "metadata": {},
-   "source": [
-    "# Label and Feature Distribution Smoothing for Deep Imbalanced Regression "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "92c81bca",
-   "metadata": {},
-   "source": [
-    "## Initial imports"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "2e7b9854",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "import torch\n",
-    "from torch.optim import SGD, lr_scheduler\n",
-    "\n",
-    "from pytorch_widedeep import Trainer\n",
-    "from pytorch_widedeep.preprocessing import TabPreprocessor\n",
-    "from pytorch_widedeep.models import TabMlp, WideDeep\n",
-    "from sklearn.metrics import mean_squared_error\n",
-    "from pytorch_widedeep.initializers import XavierNormal\n",
-    "from pytorch_widedeep.datasets import load_california_housing\n",
-    "\n",
-    "from sklearn.model_selection import train_test_split\n",
-    "\n",
-    "import matplotlib.pyplot as plt\n",
-    "from scipy.ndimage import convolve1d\n",
-    "from scipy.ndimage import gaussian_filter1d\n",
-    "from scipy.signal.windows import triang\n",
-    "from pytorch_widedeep.utils.deeptabular_utils import get_kernel_window, find_bin\n",
-    "from pytorch_widedeep.models import fds_layer\n",
-    "\n",
-    "# increase displayed columns in jupyter notebook\n",
-    "pd.set_option(\"display.max_columns\", 200)\n",
-    "pd.set_option(\"display.max_rows\", 300)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "3c0fc233",
-   "metadata": {},
-   "source": [
-    "# Load dataset"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "934cca51",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>MedInc</th>\n",
-       "      <th>HouseAge</th>\n",
-       "      <th>AveRooms</th>\n",
-       "      <th>AveBedrms</th>\n",
-       "      <th>Population</th>\n",
-       "      <th>AveOccup</th>\n",
-       "      <th>Latitude</th>\n",
-       "      <th>Longitude</th>\n",
-       "      <th>MedHouseVal</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>8.3252</td>\n",
-       "      <td>41.0</td>\n",
-       "      <td>6.984127</td>\n",
-       "      <td>1.023810</td>\n",
-       "      <td>322.0</td>\n",
-       "      <td>2.555556</td>\n",
-       "      <td>37.88</td>\n",
-       "      <td>-122.23</td>\n",
-       "      <td>4.526</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>8.3014</td>\n",
-       "      <td>21.0</td>\n",
-       "      <td>6.238137</td>\n",
-       "      <td>0.971880</td>\n",
-       "      <td>2401.0</td>\n",
-       "      <td>2.109842</td>\n",
-       "      <td>37.86</td>\n",
-       "      <td>-122.22</td>\n",
-       "      <td>3.585</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>7.2574</td>\n",
-       "      <td>52.0</td>\n",
-       "      <td>8.288136</td>\n",
-       "      <td>1.073446</td>\n",
-       "      <td>496.0</td>\n",
-       "      <td>2.802260</td>\n",
-       "      <td>37.85</td>\n",
-       "      <td>-122.24</td>\n",
-       "      <td>3.521</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>5.6431</td>\n",
-       "      <td>52.0</td>\n",
-       "      <td>5.817352</td>\n",
-       "      <td>1.073059</td>\n",
-       "      <td>558.0</td>\n",
-       "      <td>2.547945</td>\n",
-       "      <td>37.85</td>\n",
-       "      <td>-122.25</td>\n",
-       "      <td>3.413</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>3.8462</td>\n",
-       "      <td>52.0</td>\n",
-       "      <td>6.281853</td>\n",
-       "      <td>1.081081</td>\n",
-       "      <td>565.0</td>\n",
-       "      <td>2.181467</td>\n",
-       "      <td>37.85</td>\n",
-       "      <td>-122.25</td>\n",
-       "      <td>3.422</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "   MedInc  HouseAge  AveRooms  AveBedrms  Population  AveOccup  Latitude  \\\n",
-       "0  8.3252      41.0  6.984127   1.023810       322.0  2.555556     37.88   \n",
-       "1  8.3014      21.0  6.238137   0.971880      2401.0  2.109842     37.86   \n",
-       "2  7.2574      52.0  8.288136   1.073446       496.0  2.802260     37.85   \n",
-       "3  5.6431      52.0  5.817352   1.073059       558.0  2.547945     37.85   \n",
-       "4  3.8462      52.0  6.281853   1.081081       565.0  2.181467     37.85   \n",
-       "\n",
-       "   Longitude  MedHouseVal  \n",
-       "0    -122.23        4.526  \n",
-       "1    -122.22        3.585  \n",
-       "2    -122.24        3.521  \n",
-       "3    -122.25        3.413  \n",
-       "4    -122.25        3.422  "
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df = load_california_housing(as_frame=True)\n",
-    "df.head()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "753c111d",
-   "metadata": {},
-   "source": [
-    "# Effects of ks and sigma paramaters on kernel function"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "38837284",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[<matplotlib.lines.Line2D at 0x16a0a43a0>]"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    },
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGdCAYAAADqsoKGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcPUlEQVR4nO3deViU5cIG8HtmYGbYF5FVZBEVc0W2NEtLFM06erKOmqVSaostRpnynS/tfJ4OuBwz07RjxyVbtE7ZLmok2oKCIOaKoijIKiLDJgzMvN8fY9MhNwaBZ2a4f9c115XDM+/cb+Mwt++87/PIJEmSQERERGTh5KIDEBEREbUFlhoiIiKyCiw1REREZBVYaoiIiMgqsNQQERGRVWCpISIiIqvAUkNERERWgaWGiIiIrIKN6AAdRa/Xo6ioCE5OTpDJZKLjEBERUQtIkoTq6mr4+vpCLr/5sZhOU2qKiorg7+8vOgYRERG1QkFBAbp163bTMZ2m1Dg5OQEw/E9xdnYWnIaIiIhaoqqqCv7+/sbP8ZvpNKXmt6+cnJ2dWWqIiIgsTEtOHeGJwkRERGQVWGqIiIjIKrDUEBERkVVgqSEiIiKrwFJDREREVoGlhoiIiKwCSw0RERFZBZYaIiIisgosNURERGQVWGqIiIjIKrDUEBERkVVgqSEiIiKrwFJDRBYvt6wG7/14FjUNTaKjEJFAnWaVbiKyTmcv1uCRdb/gcl0j9uSUYeOMKCht+O81os6I73wisljlNQ2YsTEDl+saAQA/517Cgs9/hSRJgpMRkQgsNURkka5odXhy80HkV9Shm5sdlj8yEAq5DJ9nFeLN3adExyMiAfj1ExFZHJ1ewgtbD+FwQSVc7GyxKS4KIZ6OaNLpseDzI1j1Qy58Xe0wOaq76KhE1IF4pIaILIokSfjb18ew+3gplDZyvDc9AiGejgCAyVHd8fx9IQCAv35xFKk5ZSKjElEHY6khIouy/sezeD/tPABgxV8GIjLQvdnP40f1wkNhftDpJcz5MAtHCzUiYhKRACw1RGQxvj5chH98dxIA8Nf7++CBAb7XjJHJZEiaOAB3hXRBrVaHuE0ZuHC5rqOjEpEALDVEZBHS8yrw8ieHAQAzhgZi5t1BNxyrtJFj7WPh6O3lhIvVhiukNFevkCIi68VSQ0RmL7esGrPePwitTo/Rd3jhtQfugEwmu+ljnNW22BgXCW9nNXLLajB7y0E0NOk6KDERicBSQ0Rmray6HtM3ZEBzpRFh3V3x1uQwKOQ3LzS/8XW1w8a4SDiqbHAgrwLzPv0Vej3nsCGyViw1RGS2ahua8MSmDBRWXkFAF3u8Ny0CdkqFSdvo4+OMtY8Nho1chq8OF2HZrpx2SktEorHUEJFZatLp8dxHWThaWAV3ByU2xUWhi6OqVdu6u2dXJD7UHwCwNvUMPth/vi2jEpGZYKkhIrMjSRJe+/IY9uRchOrqXDRBHg63tc1HIvzxUkwvAMDCL4/i++OlbRGViMwISw0RmZ13Us/g4/R8yGTAW5PDMLi7W5ts94WRIZgU4Q+9BDz/sWFGYiKyHiw1RGRWvjhUiGU7Dee9LHrgDozp591m25bJZPj7n/vhnl5dcaVRhyc3ZyD/EuewIbIWLDVEZDZ+yS3HvP8Y5qKZOSwIM+668Vw0rWWrkOOdqYNxh48zymu0mLExHZdrtW3+PETU8VhqiMgs5JRU46kPMtGokzCuvw/+5/4+7fZcjiobbIyLhJ+rHc6W12LW+wdR38g5bIgsHUsNEQlXWlWPuI3pqK5vQkSAG/75l4GQt3AumtbyclZjY1wknNQ2OHj+Ml7+5DDnsCGycK0qNWvWrEFgYCDUajWio6ORnp5+w7Hr16/H3XffDTc3N7i5uSEmJuaa8Z9//jlGjx6NLl26QCaTITs7+5rtjBgxAjKZrNnt6aefbk18IjIj1fWNmLExA0WaegR3dcD6aRFQ25o2F01r9fJywruPh8NWIcO3R4rxj+9OdMjzElH7MLnUbNu2DfHx8Vi0aBGysrIwcOBAxMbGoqys7LrjU1NTMWXKFOzZswdpaWnw9/fH6NGjUVhYaBxTW1uLYcOGYcmSJTd97lmzZqG4uNh4W7p0qanxiciMNOr0ePbDLJworoKHoxKb46Lg5qDs0AxDe3hg+SMDAQDv/ZSHjT/ndejzE1HbkUmSZNLx1ujoaERGRmL16tUAAL1eD39/fzz//PNYsGDBLR+v0+ng5uaG1atXY9q0ac1+du7cOQQFBeHQoUMYNGhQs5+NGDECgwYNwsqVK02Ja1RVVQUXFxdoNBo4Ozu3ahtE1HYkScKr//kVn2ZegJ2tAtueuhMDurkKy/NOai6WJudAJgPWTg1v06uuiKj1TPn8NulIjVarRWZmJmJiYn7fgFyOmJgYpKWltWgbdXV1aGxshLu7uylPDQD48MMP4eHhgX79+iEhIQF1dTe+FLOhoQFVVVXNbkRkPt5KOY1PMy9ALgNWPxomtNAAwDPDe+DR6O6QJODFrYeQef6y0DxEZDqTSk15eTl0Oh28vLya3e/l5YWSkpIWbWP+/Pnw9fVtVoxa4tFHH8UHH3yAPXv2ICEhAVu2bMFjjz12w/GJiYlwcXEx3vz9/U16PiJqP58eLMDK708DAP5vfD+M7ON1i0e0P5lMhv/7U1+MDPVEQ5MeMzdnIK+8VnQsIjJBh179lJSUhK1bt2L79u1Qq9UmPXb27NmIjY1F//79MXXqVLz//vvYvn07zpw5c93xCQkJ0Gg0xltBQUFb7AIR3aZ9py4i4fMjAIBnRvTAY3cGCE70OxuFHG8/Gob+fi64XNeIGRvTcammQXQsImohk0qNh4cHFAoFSkubr5lSWloKb++bf/+8fPlyJCUlYdeuXRgwYIDpSf8gOjoaAJCbm3vdn6tUKjg7Oze7EZFYx4uq8OyHWWjSS/jTQF/MG91bdKRr2Ctt8O8ZEejmZofzl+rw5OaDuKLlHDZElsCkUqNUKhEeHo6UlBTjfXq9HikpKRgyZMgNH7d06VIsXrwYycnJiIiIaH3a//LbZd8+Pj5tsj0ial9FlVcQtykdNQ1NuDPYHcseGdDuc9G0lqeTGpviouBiZ4vsgkq8sPUQdJzDhsjsmfz1U3x8PNavX4/NmzfjxIkTeOaZZ1BbW4u4uDgAwLRp05CQkGAcv2TJErz22mvYsGEDAgMDUVJSgpKSEtTU1BjHVFRUIDs7G8ePHwcA5OTkIDs723iezpkzZ7B48WJkZmbi3Llz+OqrrzBt2jTcc889bXLUh4jal+aK4auc0qoG9PR0xLuPRUBl0zFz0bRWiKcj3pseAaWNHLuPl+L/vj4GEy8WJaIOZnKpmTRpEpYvX46FCxdi0KBByM7ORnJysvHk4fz8fBQXFxvHr127FlqtFg8//DB8fHyMt+XLlxvHfPXVVwgLC8O4ceMAAJMnT0ZYWBjWrVsHwHCE6Pvvv8fo0aMRGhqKl19+GRMnTsTXX399WztPRO1P26TH01sycaq0Bl2dVNgYFwkXe1vRsVokMtAdK/5imMNmc9p5vPcj57AhMmcmz1NjqThPDVHHkyQJ8Z8cxvZDhXBQKrDtqSHo5+ciOpbJ1u87izeuzja8+tEwPDDAV3Aios6j3eapISIyxT93ncL2Q4VQyGVYM3WwRRYaAJh5dxBmDA0EAMRvO4z0vAqxgYjoulhqiKhdfHQgH6v3GK5O/Mef+2FEb0/BiVpPJpPhtQfuwOg7vKDV6THr/YPILau59QOJqEOx1BBRm9tzsgyvfXkUAPDCfSGYFNldcKLbp5DL8NbkMIR1dzWe+FxWXS86FhH9F5YaImpTRy5oMOejLOj0Eh4a7IeXRvUSHanN2CkVeG9aBAK62OPC5St4ctNB1GmbRMcioqtYaoiozRRU1CFuUwbqtDoMC/FA0kMDIJOZ51w0rdXFUYVNcVFwd1DiSKEGz310CE06vehYRASWGiJqI5V1WszYmI7ymgaEejvhnccGQ2ljnb9igjwc8N70CKhs5PjhZBkWfsU5bIjMgXX+xiGiDtXQpMPsLZk4c7EW3s5qbIyLhLPaMuaiaa3B3d3w1uQwyGSGk6LX7r3+OnRE1HFYaojotuj1El7+xHCZs5PKBhvjIuHjYic6VocY088bCx+4AwCwNDkHX2YXCk5E1Lmx1BDRbVmSfBLf/FoMG7kM6x4PRx+fzjW5ZdxdQZg5LAgA8Mqnh/HLmXLBiYg6L5YaImq199PO4d19ZwEASyYOwF0hHoITifE/9/fBuP4+aNRJeGpLJk6VVouORNQpsdQQUavsPl6K1786BgB4eVQvTAzvJjiROHK5DP/8y0BEBLihur4JMzako7SKc9gQdTSWGiIy2aH8y3j+4yzoJWBypD+euy9EdCTh1LYKrJ8WgeCuDijS1CNuYwZqGjiHDVFHYqkhIpOcv1SLmZsPor5Rj+G9umLxhH5WNxdNa7k5KLE5LgoejkocL67Csx9moZFz2BB1GJYaImqxilotZmzMwKVaLfr6OmPN1MGwVfDXyH/zd7fHhhmRsLNVYN+pi/jr9iOcw4aog/C3ERG1SH2jDjM3ZyCvvBZ+rnbYOCMSjiob0bHM0oBurlj9aBjkMuCTgxewKiVXdCSiToGlhohuSaeXMHdrNrLyK+GstsGmuEh4OqtFxzJrI/t44f/G9wMAvPn9Kfwn84LgRETWj6WGiG7pjW9PIPlYCZQKOf41LQI9vZxER7IIj90ZgGdG9AAALPjsV/x4+qLgRETWjaWGiG7q3z/lYcPPeQCAZY8MwJ3BXQQnsizzRvfGnwb6okkv4ZkPsnC8qEp0JCKrxVJDRDe040gx/v7tcQDA/DGhGD/IT3AiyyOXy66WQXfUNDThiU0ZKNZcER2LyCqx1BDRdWWer8DcbdmQJOCxO7vj6eHBoiNZLJWNAu8+FoGeno4oqarHjA0ZqKpvFB2LyOqw1BDRNc5erMHMzQfR0KTHyFBPvP5gX85Fc5tc7G2xMS4SXZ1UyCmtxjMfZELbxDlsiNoSSw0RNVNe04AZGzNwua4RA7u54O1Hw2DDuWjaRDc3e2ycEQkHpQI/517Cgs9+5Rw2RG2Iv6mIyKhO24QnNx9EfkUd/N3t8N70SNgrORdNW+rn54I1UwdDIZfh80OFWLH7lOhIRFaDpYaIABjmonnh42wcLqiEq70tNsVFoauTSnQsqzSityf+8WfDHDZv/5CLj9PzBScisg4sNUQESZLw+lfH8P2JUiht5HhvWgR6dHUUHcuqTYrsjheuLgT6v18cxZ6cMsGJiCwfSw0R4V/7zmLL/vOQyYCVkwYhItBddKRO4aVRvfDQYD/o9BLmfJiFo4Ua0ZGILBpLDVEn99XhIiTuOAkA+Ov9fXB/fx/BiToPmUyGpIcGYFiIB+q0OsRtykBBRZ3oWEQWi6WGqBPbf/YSXvnkMABgxtBAPDksSHCizkdpI8c7jw1GqLcTLlY3IG5TBjR1nMOGqDVYaog6qdyyasx+/yC0Oj1i+3rhtQfu4Fw0gjirDXPYeDurkVtWg1lbDqKhSSc6FpHFYakh6oTKquoxfUMGquqbENbdFW9NDoNCzkIjko+LHTbGRcJJZYP0vAq88umv0Os5hw2RKVhqiDqZ2oYmPLE5A4WVVxDYxR7vTYuA2lYhOhYB6OPjjHWPh8NGLsPXh4uwZOdJ0ZGILApLDVEn0qTTY85HWThaWAV3ByU2xUWhiyPnojEnd4V4YMnEAQCAd/eexZa0c2IDEVkQlhqiTkKSJLz25VGk5lyE2laO96ZHINDDQXQsuo6J4d0QP6oXAGDRV8ew+3ip4EREloGlhqiTWLMnFx+nF0AmA96aHIbB3d1ER6KbeP6+EEyO9IdeAp7/OAvZBZWiIxGZPZYaok5g+6ELWL7LsMbQ6w/2RWxfb8GJ6FZkMhkWT+iH4b26or5Rjyc3ZSD/EuewIboZlhoiK/dLbjle/c+vAIDZ9wRj+tBAsYGoxWwVcqyZOhh9fZ1xqVaLGRvTcblWKzoWkdliqSGyYidLqvDUlkw06iSMG+CDBWNCRUciEzmqbLBxRiT8XO1wtrwWM98/iPpGzmFDdD0sNURWqkRTj7iNGahuaEJUoDv++chAyDkXjUXydFZjU1wknNU2yDx/GS9ty+YcNkTXwVJDZIWq6xsxY2M6ijX1CO7qgH9NC+dcNBaup5cT/jUtAkqFHDuOluCN706IjkRkdlhqiKxMo06PZz/MwsmSang4qrA5Lgqu9krRsagN3BncBcseMcxh8++f8rDhpzzBiYjMC0sNkRWRJAkJnx/Bj6fLYWerwIYZEfB3txcdi9rQ+EF+mH/13KjF3x5H8tFiwYmIzAdLDZEVWfn9afwn8wLkMmDN1DAM6OYqOhK1g6eHB+OxO7tDkoAXt2Yj83yF6EhEZoGlhshKfJJRgLdSTgMAFk/oh/tCvQQnovYik8nw+oN9MTLUEw1NeszcfBBnL9aIjkUkHEsNkRXYe+oiErYfAQA8O6IHpkYHCE5E7c1GIcfbj4ZhYDcXXK5rxIyNGSivaRAdi0ioVpWaNWvWIDAwEGq1GtHR0UhPT7/h2PXr1+Puu++Gm5sb3NzcEBMTc834zz//HKNHj0aXLl0gk8mQnZ19zXbq6+sxZ84cdOnSBY6Ojpg4cSJKS7keCtGxIg2e/SATOr2E8YN8MS+2t+hI1EHslTZ4b3ok/N3tkF9Rhyc3H8QVLeewoc7L5FKzbds2xMfHY9GiRcjKysLAgQMRGxuLsrKy645PTU3FlClTsGfPHqSlpcHf3x+jR49GYWGhcUxtbS2GDRuGJUuW3PB5X3rpJXz99df49NNPsXfvXhQVFeGhhx4yNT6RVSmsvIK4jRmo1epwZ7A7lj48ADIZ56LpTLo6qbApLgqu9rY4XFCJF7Yego5z2FAnJZMkyaS//dHR0YiMjMTq1asBAHq9Hv7+/nj++eexYMGCWz5ep9PBzc0Nq1evxrRp05r97Ny5cwgKCsKhQ4cwaNAg4/0ajQZdu3bFRx99hIcffhgAcPLkSfTp0wdpaWm48847b/m8VVVVcHFxgUajgbOzswl7TGSeNFca8ci6X3CqtAa9vBzx6dND4WJnKzoWCXLwXAUefe8AtE16TBsSgL/9qS8LLlkFUz6/TTpSo9VqkZmZiZiYmN83IJcjJiYGaWlpLdpGXV0dGhsb4e7u3uLnzczMRGNjY7PnDQ0NRffu3W/4vA0NDaiqqmp2I7IWDU06PL0lE6dKa+DppMLGuCgWmk4uItAdKycNgkwGvJ92Hut/PCs6ElGHM6nUlJeXQ6fTwcur+VUVXl5eKCkpadE25s+fD19f32YF5VZKSkqgVCrh6ura4udNTEyEi4uL8ebv79/i5yMyZ5IkYf5/fkXa2UtwUCqwMc6wLhDR/f198Nf7+wAA/vHdSXx9uEhwIqKO1aFXPyUlJWHr1q3Yvn071Gp1uz5XQkICNBqN8VZQUNCuz0fUUZbvysEX2UVQyGV457Fw9PV1ER2JzMiTw4Iw4+pK7C9/chjpeZzDhjoPk0qNh4cHFArFNVcdlZaWwtvb+6aPXb58OZKSkrBr1y4MGDDApJDe3t7QarWorKxs8fOqVCo4Ozs3uxFZuo8O5GPNnjMAgMQ/98fwXl0FJyJzI5PJ8NoDdyC2rxe0Oj1mvX8QuWXVomMRdQiTSo1SqUR4eDhSUlKM9+n1eqSkpGDIkCE3fNzSpUuxePFiJCcnIyIiwuSQ4eHhsLW1bfa8OTk5yM/Pv+nzElmTH06W4n+/MMxF88LInvhLJL9SpetTyGV4a3IYwrq7QnOlEdM3ZKCsul50LKJ2Z2PqA+Lj4zF9+nREREQgKioKK1euRG1tLeLi4gAA06ZNg5+fHxITEwEAS5YswcKFC/HRRx8hMDDQeA6Mo6MjHB0dAQAVFRXIz89HUZHh+9+cnBwAhiM03t7ecHFxwZNPPon4+Hi4u7vD2dkZzz//PIYMGdKiK5+ILN2vFyox58ND0EvAw+Hd8FJMT9GRyMypbRV4b1oEJq79Becu1eGJTRnYNnsIHFQm/9onshgmn1MzadIkLF++HAsXLsSgQYOQnZ2N5ORk48nD+fn5KC7+fYG1tWvXQqvV4uGHH4aPj4/xtnz5cuOYr776CmFhYRg3bhwAYPLkyQgLC8O6deuMY95880088MADmDhxIu655x54e3vj888/b/WOE1mKggrDB9KVRh3u7umBxIf681JdapEujoY5bNwdlDhaWIXnPspCk04vOhZRuzF5nhpLxXlqyBJV1mnx0NpfcPZiLUK9nfDp00PgpOal22SarPzLeHT9ftQ36jElqjv+8ed+LMZkMdptnhoi6jj1jTrMev8gzl6shY+LGpviolhoqFUGd3fDW5PDIJMBH6fn453UM6IjEbULlhoiM6TXS3j508PIOHcZTiobbIyLhLdL+06DQNYttq83Xn+wLwBg2c4cbD90QXAiorbHUkNkhpKST+LbX4thq5Dh3cfDEerNr0zp9k0fGojZ9wQDAF79z6/4JbdccCKitsVSQ2RmNv9yDv/aZ5jifsnEARga4iE4EVmTBWNCMW6ADxp1Ep7akomcEs5hQ9aDpYbIjOw6VoLXvz4GAHhldC88NLib4ERkbeRyGf75yEBEBbqjuqEJMzamo0TDOWzIOrDUEJmJQ/mX8cLWQ5AkYEqUP+bcGyI6Elkpta0C/5oWjuCuDijW1CNuUwaq6xtFxyK6bSw1RGbgXHktntx8EPWNeozo3RWLx/OSW2pfrvZKbI6LgoejCieKq/Dsh1lo5Bw2ZOFYaogEq6jVYsbGdFTUatHPzxlrHh0MGwXfmtT+/N3tsWFGBOxsFfjxdDn+5/Mj6CRTl5GV4m9OIoHqG3WYuTkD5y7Vwc/VDhtmRHIae+pQA7q5Ys3UMMhlwKeZF/BWymnRkYhajaWGSBCdXsKLWw8hK78SzmobbH4iEp5OnIuGOt59oV5YPKEfAGDl96fxycECwYmIWoelhkiQv397HDuPlUKpkGP9tAiEeDqJjkSd2NToADw7ogcA4H8+P4J9py4KTkRkOpYaIgHe+/EsNv58DgCw/C8DER3cRWwgIgDzYntj/CBfNOklPPthFo4XVYmORGQSlhqiDvbdkWK88d0JAMCCsaH400BfwYmIDGQyGZY+PAB3BrujpqEJcZvSUVR5RXQsohZjqSHqQAfPVWDutmxIEvD4nQF46uqU9UTmQmWjwLuPR6CXlyNKqxowY2M6NFc4hw1ZBpYaog5y5mINZr5/ENomPWL6eOL1P/XlXDRkllzsbLExLgqeTiqcKq3B01syoW3iHDZk/lhqiDrAxWrDv3gr6xoxsJsLVk0Jg0LOQkPmy8/VDhvjIuGgVCDt7CXM/+xXzmFDZo+lhqid1WmbMHNzBgoqrqC7uz3+PSMS9krORUPmr6+vC955LBwKuQzbDxXin7tOiY5EdFMsNUTtqEmnxwsfH8LhCxq42ttiU1wkPBxVomMRtdjwXl2R+Of+AIDVe3Lx0YF8wYmIboylhqidSJKE178+hu9PlEFpI8d70yIQ3NVRdCwik/0l0h8vjOwJAHjty6PYc7JMcCKi62OpIWon7+47iw/250MmA96aNAgRge6iIxG12ksxPfFweDfo9BLmfJSFIxc0oiMRXYOlhqgdfJldiKQdJwEA/zvuDozt7yM4EdHtkclkSHyoP+7u6YE6rQ5xmzJQUFEnOhZRMyw1RG1s/9lLmPfprwCAuLsC8eSwIMGJiNqGrUKOd6YORqi3E8prfruiTys6FpERSw1RGzpdWo3Z7x+EVqfHmL7e+N9xd4iORNSmnNS22BQXBR8XNc5crMXsLZloaNKJjkUEgKWGqM2UVdVjxsYMVNU3YXB3V6ycPIhz0ZBV8nZRY2NcJJxUNkjPq8DLnxyGXs85bEg8lhqiNmBYJycDhZVXEOThgPemR0JtqxAdi6jdhHo7Y93j4bBVyPDNr8VYknxSdCQilhqi29Wo02POh1k4VlSFLg5KbIqLhLuDUnQsonZ3V4gHlkwcAMBwtd/7aefEBqJOj6WG6DZIkoTXvjiKvacuQm0rx79nRCKgi4PoWEQd5qHB3fDK6F4AgNe/OoZdx0oEJ6LOjKWG6Das/iEXWzMKIJcBb08ZjEH+rqIjEXW4OfeGYEqUP/QS8MLWQziUf1l0JOqkWGqIWumzzAv4527DWjiv/6kvRt3hJTgRkRgymQyLx/fDiN5dUd+ox8zNB3H+Uq3oWNQJsdQQtcLPueWY/5lhLpqn7gnGtCGBYgMRCWajkGPNo4PRz88Zl2q1mLExAxW1nMOGOhZLDZGJTpZU4ektmWjSS3hggA/mjwkVHYnILDiobLBhRiT8XO2QV16LmZszUN/IOWyo47DUEJmgWHMFMzZkoLqhCVFB7lj+yEDIORcNkZGnkxqbn4iEs9oGWfmVmLs1GzrOYUMdhKWGqIWq6hsRtzEDJVX16NHVAf96PJxz0RBdR4inE9ZPi4BSIUfysRK88e0J0ZGok2CpIWoBbZMez36QhZMl1fBwVGFTXBRc7TkXDdGNRAd3wfK/DAQAbPg5D//+KU9wIuoMWGqIbkGSJCz4/Ff8lFsOe6UCG2dEwt/dXnQsIrP3p4G+WDDWcM7Z3789jh1HigUnImvHUkN0C29+fxqfZxVCIZdhzaOD0b+bi+hIRBbjqXuC8fidAZAk4MVt2cg8XyE6Elkxlhqim9iWkY9VKacBAIvH98O9oZ6CExFZFplMhtf/1BcxfTyhbTLMYXP2Yo3oWGSlWGqIbiA1pwz/s/0oAGDOvT3waHR3wYmILJNCLsOqKWEY2M0Fl+saMWNjBsprGkTHIivEUkN0HUcLNZjzYRZ0egl/DvPDK6N7i45EZNHslTb494xIdHe3R35FHZ7clIE6bZPoWGRlWGqI/qCw8gqe2JSBWq0OQ4K7YMnEAZDJOBcN0e0yXDkYCVd7Wxy+oMELH3MOG2pbLDVE/0VT14gZG9JRVt2AXl6OWPd4OJQ2fJsQtZXgro54b1oElDZyfH+iFK9/dQySxGJDbYO/rYmuamjS4akPDuJ0WQ28nA1z0bjY2YqORWR1IgLd8dakQZDJgC37z+Nf+86KjkRWgqWGCIBeL+HV//yK/Wcr4KiywcYZUfB1tRMdi8hqje3vg/8ddwcAIHHHSXx1uEhwIrIGrSo1a9asQWBgINRqNaKjo5Genn7DsevXr8fdd98NNzc3uLm5ISYm5prxkiRh4cKF8PHxgZ2dHWJiYnD69OlmYwIDAyGTyZrdkpKSWhOf6BrLduXgy+wi2MhleGfqYNzh6yw6EpHVe3JYEOLuCgQAvPLJYew/e0lsILJ4Jpeabdu2IT4+HosWLUJWVhYGDhyI2NhYlJWVXXd8amoqpkyZgj179iAtLQ3+/v4YPXo0CgsLjWOWLl2KVatWYd26dThw4AAcHBwQGxuL+vr6Ztv6v//7PxQXFxtvzz//vKnxia6x40gx1qaeAQAkPtQf9/TqKjgRUefxv+PuwJi+3tDq9HhqSyYv9abbIpNMPEMrOjoakZGRWL16NQBAr9fD398fzz//PBYsWHDLx+t0Ori5uWH16tWYNm0aJEmCr68vXn75ZbzyyisAAI1GAy8vL2zatAmTJ08GYDhSM3fuXMydO9fEXTSoqqqCi4sLNBoNnJ35r3Ay0DbpEbNiL/Ir6vDUPcFIuL+P6EhEnU59ow4PvfMLjhdXYfqQAPxtfD/RkciMmPL5bdKRGq1Wi8zMTMTExPy+AbkcMTExSEtLa9E26urq0NjYCHd3dwBAXl4eSkpKmm3TxcUF0dHR12wzKSkJXbp0QVhYGJYtW4amJs5xQLfnwwPnkV9RBw9HFV4Y2VN0HKJOSW2rwP+OM/yD4sMD+ThXXis4EVkqG1MGl5eXQ6fTwcvLq9n9Xl5eOHnyZIu2MX/+fPj6+hpLTElJiXEbf9zmbz8DgBdeeAGDBw+Gu7s7fvnlFyQkJKC4uBgrVqy47vM0NDSgoeH3w5hVVVUtykedR3V9I97+IRcAMDemJxxUJr0diKgNDQ3xwPBeXbH31EUs25WDNY8OFh2JLFCH/hZPSkrC1q1bkZqaCrVabdJj4+Pjjf89YMAAKJVKPPXUU0hMTIRKpbpmfGJiIv72t7/ddmayXu/uPYuKWi2CPRwwKdJfdByiTm/B2FDsO30R3/5ajNl3V2Kgv6voSGRhTPr6ycPDAwqFAqWlpc3uLy0thbe3900fu3z5ciQlJWHXrl0YMGCA8f7fHmfqNqOjo9HU1IRz585d9+cJCQnQaDTGW0FBwU3zUedSWlWP934yzI3x6pjesFVwdgMi0fr4OOPPYX4AgMQdJzgpH5nMpN/kSqUS4eHhSElJMd6n1+uRkpKCIUOG3PBxS5cuxeLFi5GcnIyIiIhmPwsKCoK3t3ezbVZVVeHAgQM33WZ2djbkcjk8Pa+/arJKpYKzs3OzG9FvVn5/CvWNegzu7orYvjcv5ETUcV4e3RtKGzn2n61Aas5F0XHIwpj89VN8fDymT5+OiIgIREVFYeXKlaitrUVcXBwAYNq0afDz80NiYiIAYMmSJVi4cCE++ugjBAYGGs+TcXR0hKOjI2QyGebOnYu///3v6NmzJ4KCgvDaa6/B19cXEyZMAACkpaXhwIEDuPfee+Hk5IS0tDS89NJLeOyxx+Dm5tZG/yuos8gtq8a2DMORu4T7+3BdJyIz4udqh7ihgXh331kk7TiJe3p1hULO9yi1jMmlZtKkSbh48SIWLlyIkpISDBo0CMnJycYTffPz8yGX/34AaO3atdBqtXj44YebbWfRokV4/fXXAQCvvvoqamtrMXv2bFRWVmLYsGFITk42nnejUqmwdetWvP7662hoaEBQUBBeeumlZufZELXUkuQc6CVg1B1eiAx0Fx2HiP7g2REh2JpRgJzSanyedQGPRPCcN2oZk+epsVScp4YA4OC5Cjy8Lg1yGbDrpXsQ4ukkOhIRXce/9p3BP747CR8XNfa8MgJqW4XoSCRIu81TQ2TJJEnCP747AQCYFOnPQkNkxqYNCYSfqx2KNfXY9Ms50XHIQrDUUKex81gpsvIrobaVY25ML9FxiOgm1LYKxI8yvE/f2ZOLyjqt4ERkCVhqqFNo0umxdKdhgsiZw4Lh5WzaPElE1PEmhPkh1NsJVfVNWLMnV3QcsgAsNdQpbDtYgLMXa+HuoMRTw4NFxyGiFlDIZVgwNhQAsPmX87hwuU5wIjJ3LDVk9eq0TVj5/WkAwPP3hcBJbSs4ERG11PBeXTG0RxdodXqs2HVKdBwycyw1ZPXe+zEPF6sb0N3dHlOjA0THISITyGQyJIw1LHa5PbsQx4u4jh/dGEsNWbXymga8u/cMAOCVWMNMpURkWfp3c8GDA30hSUBScssWT6bOib/hyaq9nXIatVod+vu54IH+PqLjEFErzRvdG7YKGfaduoifTpeLjkNmiqWGrNa58lp8eCAfAJAwNhRyTrVOZLG6d/n96+PEHSeg13eKeWPJRCw1ZLWW7cxBk14ynGgY4iE6DhHdpufvC4GjygbHiqrw9a9FouOQGWKpIauUXVCJb48UQyaD8ZJQIrJsXRxVePrqlAzLduagoUknOBGZG5YasjqSJCHx6nIIfw7zQx8frvVFZC2eGBYETycVLly+gg/254uOQ2aGpYaszp6cMhzIq4DSRo6XR/cWHYeI2pC90gYvXV0+YfUPp1FV3yg4EZkTlhqyKjq9hCU7cgAAM4YaFsQjIuvySHg39OjqgMt1jViXekZ0HDIjLDVkVT7LuoCc0mo4q23w7IgeouMQUTuwUcgxf4zhXLkNP+ehRFMvOBGZC5Yashr1jTq8udswjfqce0Pgaq8UnIiI2suoO7wQEeCG+ka98X1PxFJDVmPjz+dQrKmHr4sa04cGio5DRO1IJpMh4X7D0ZpPMwtwurRacCIyByw1ZBUu12rxTmouACB+dG+obRWCExFRewsPcEdsXy/oJWAJl08gsNSQlVizJxfV9U0I9XbCn8P8RMchog7y6phQKOQyfH+iDOl5FaLjkGAsNWTxCirq8H7aeQDA/LGGX3BE1Dn06OqISZH+AAzLJ0gSl0/ozFhqyOKt2H0KWp0eQ4K7YESvrqLjEFEHmzuyJ+xsFTiUX4nkoyWi45BALDVk0Y4VafBFdiEAIOH+UMhkPEpD1Nl4Oqsx6+4gAMDSnTlo1OkFJyJRWGrIoiXtOAlJAh4Y4IMB3VxFxyEiQWYP74EuDkrklddia0aB6DgkCEsNWayfTpfjx9PlsFXIMC+WyyEQdWaOKhu8MLInAOCt70+jtqFJcCISgaWGLJJeLyFxh2HRyqnRAQjo4iA4ERGJNiWqOwK62KO8pgHrfzwrOg4JwFJDFunrX4twrKgKjiobPH9fiOg4RGQGlDZyvBprmJDvX/vO4mJ1g+BE1NFYasjiNDTpsGynYdHKp4cHo4ujSnAiIjIX9/f3xkB/V9RpdViVclp0HOpgLDVkcT7Yn48Ll6/A00mFJ4YFiY5DRGZEJpMhYazhaM3H6fnIK68VnIg6EksNWZSq+kas/sHwr6+XRvWCvdJGcCIiMjd3BnfBfaGeaNJLWLaTyyd0Jiw1ZFHWpZ7B5bpG9OjqgEfCu4mOQ0Rmav6YUMhlwHdHSnAo/7LoONRBWGrIYpRo6rHh5zwAhl9YNgr+9SWi6+vt7YSJgw3/8EnccZLLJ3QS/FQgi/Hm7lOob9QjIsANo+7wEh2HiMxc/OheUNnIkZ5XgR9OlomOQx2ApYYswunSanyaaZgllMshEFFL+LjYIe4uw8UESTtOoonLJ1g9lhqyCEuST0IvAbF9vRAe4C46DhFZiGdG9ICrvS1Ol9Xgs6wLouNQO2OpIbOXnleB70+UQSGX4dUxoaLjEJEFcbGzxXP3GiboXLH7FK5odYITUXtiqSGzJkm/L4cwKdIfPbo6Ck5ERJbm8SEB8HO1Q2lVg/FiA7JOLDVk1pKPluBQfiXsbBWYe3WxOiIiU6hsFHglthcAw7QQFbVawYmovbDUkNlq1Omx9OpyCLPuDoKns1pwIiKyVOMH+uEOH2dUNzRh9Q+5ouNQO2GpIbO19eoU510clJg9vIfoOERkweRyGRZcXT5hy/5zKKioE5yI2gNLDZmlmoYmvHV1MboXRvaEo4rLIRDR7bmnV1cMC/FAo07C8l05ouNQO2CpIbO0ft9ZlNdoEdDFHlOiuouOQ0RW4rejNV9mF+FooUZwGmprLDVkdsqq67H+x7MAgHmxvaG04V9TImob/fxcMH6QLwDDhHxkXfhpQWZnVcpp1Gl1GNjNBeP6+4iOQ0RW5pXRvaFUyPFTbjn2nbooOg61IZYaMitnL9bg43TDcggLxvbhcghE1Ob83e3x2J0BAAxHa/R6LnZpLVpVatasWYPAwECo1WpER0cjPT39hmPXr1+Pu+++G25ubnBzc0NMTMw14yVJwsKFC+Hj4wM7OzvExMTg9OnTzcZUVFRg6tSpcHZ2hqurK5588knU1NS0Jj6ZsWU7c6DTS7i3d1cM6dFFdBwislLP3RcCJ5UNjhdX4cvDhaLjUBsxudRs27YN8fHxWLRoEbKysjBw4EDExsairOz6K6CmpqZiypQp2LNnD9LS0uDv74/Ro0ejsPD3v0RLly7FqlWrsG7dOhw4cAAODg6IjY1FfX29cczUqVNx7Ngx7N69G9988w327duH2bNnt2KXyVxl5V/GjqMlkMmA+WO5HAIRtR93ByWeHmGYKmL5zlOob+TyCVZBMlFUVJQ0Z84c4591Op3k6+srJSYmtujxTU1NkpOTk7R582ZJkiRJr9dL3t7e0rJly4xjKisrJZVKJX388ceSJEnS8ePHJQBSRkaGccyOHTskmUwmFRYWtuh5NRqNBEDSaDQtGk8dS6/XS4+s/UUKmP+N9PIn2aLjEFEnUNfQJEW/8b0UMP8baf2+M6Lj0A2Y8vlt0pEarVaLzMxMxMTEGO+Ty+WIiYlBWlpai7ZRV1eHxsZGuLsbVlrOy8tDSUlJs226uLggOjrauM20tDS4uroiIiLCOCYmJgZyuRwHDhy47vM0NDSgqqqq2Y3MV8qJMqSfq4DKRo74Ub1ExyGiTsBOqcBLowzLr6zekwvNlUbBieh2mVRqysvLodPp4OXl1ex+Ly8vlJSUtGgb8+fPh6+vr7HE/Pa4m22zpKQEnp6ezX5uY2MDd3f3Gz5vYmIiXFxcjDd/f/8W5aOO16TTY0my4dLKGXcFwtfVTnAiIuosJg7uhp6ejqisa8Ta1DOi49Bt6tCrn5KSkrB161Zs374danX7ruOTkJAAjUZjvBUUFLTr81HrfZZ1AafLauBiZ4tnh4eIjkNEnYiNQo75Ywzn8G38OQ9FlVcEJ6LbYVKp8fDwgEKhQGlpabP7S0tL4e3tfdPHLl++HElJSdi1axcGDBhgvP+3x91sm97e3teciNzU1ISKioobPq9KpYKzs3OzG5mfK1odVuw+BQB47t4QuNjbCk5ERJ3NyD6eiAp0R0OTHm9e/X1ElsmkUqNUKhEeHo6UlBTjfXq9HikpKRgyZMgNH7d06VIsXrwYycnJzc6LAYCgoCB4e3s322ZVVRUOHDhg3OaQIUNQWVmJzMxM45gffvgBer0e0dHRpuwCmZkNP+ehtKoBfq52eHxIgOg4RNQJyWQyLLjfcLTms6wLyCmpFpyIWsvkr5/i4+Oxfv16bN68GSdOnMAzzzyD2tpaxMXFAQCmTZuGhIQE4/glS5bgtddew4YNGxAYGIiSkhKUlJQY55iRyWSYO3cu/v73v+Orr77CkSNHMG3aNPj6+mLChAkAgD59+mDMmDGYNWsW0tPT8fPPP+O5557D5MmT4evr2wb/G0iEilot1l39Dvvl0b2gtlUITkREndXg7m4Y288begnGc/zI8pi89PGkSZNw8eJFLFy4ECUlJRg0aBCSk5ONJ/rm5+dDLv+9K61duxZarRYPP/xws+0sWrQIr7/+OgDg1VdfRW1tLWbPno3KykoMGzYMycnJzc67+fDDD/Hcc89h5MiRkMvlmDhxIlatWtWafSYzsfqHXFQ3NKGPjzMmDPITHYeIOrl5sb2x+3gpfjhZhv1nL+HOYE4AamlkkiR1ivmhq6qq4OLiAo1Gw/NrzEBBRR3u+2cqGnUS3n8iCvf06io6EhERXvviKLbsP4+B/q744tmhXKrFDJjy+c21n0iI5bty0KiTMCzEg4WGiMzGCyN7wkGpwOGCSnx3pGVTlZD5YKmhDne0UIMvs4sAAAu4HAIRmZGuTirMuicYALBs50lom/SCE5EpWGqowyXtMJyEN36QL/r5uQhOQ0TU3Ky7g+HhqMK5S3X4OD1fdBwyAUsNdah9py7ip9xyKBVyvDK6t+g4RETXcFDZ4MUYw/IJq1JOo7qeyydYCpYa6jB6vWQ8SvPYnQHwd7cXnIiI6PomR/oj2MMBl2q1WL/vrOg41EIsNdRhvjxciOPFVXBS2eC5+7gcAhGZL1uFHPNiDUeT1/+Yh7KqesGJqCVYaqhD1DfqsHynYfrxp0f0gLuDUnAiIqKbG9PPG2HdXXGlUYeVKadFx6EWYKmhDvHB/vMorLwCb2c1nrgrSHQcIqJbkslkSBjbBwCwLaMAZy7WCE5Et8JSQ+1Oc6URq/fkAgBeGtUTdkouh0BEliEqyB0xfTyh00tYyuUTzB5LDbW7talnUFnXiJ6ejpg4uJvoOEREJpk/JhRyGbDzWCkyz1eIjkM3wVJD7aqo8go2/pwHwPCLwUbBv3JEZFl6ejnhkXB/AEDidyfRSVYXskj8hKF2tWL3KTQ06REV6I6RfTxFxyEiapWXRvWC2laOg+cvY/fxUtFx6AZYaqjdnCypwmdZFwAAC+4P5cJwRGSxvF1+v8hhSfJJNOm4fII5YqmhdrNkx0lIEjC2nzcGd3cTHYeI6LY8PaIH3OxtceZiLT7NvCA6Dl0HSw21i7Qzl7An5yIUcplxAisiIkvmrLbFc/cZlk94c/cp1GmbBCeiP2KpoTYnSRKSdpwAAEyJ8kdwV0fBiYiI2sZjd3ZHNzc7lFU3YMNPeaLj0B+w1FCb+/ZIMQ5f0MBeqcCLI3uJjkNE1GZUNgrj0ed1e8/iUk2D4ET031hqqE1pm/RYtjMHADDr7mB0dVIJTkRE1LYeHOCLfn7OqGlowts/5IqOQ/+FpYba1Mfp+Th/qQ4ejkrMuidYdBwiojYnl8uwYIxh+YQPD5xH/qU6wYnoNyw11Gaq6xux6uqiby+O7AlHlY3gRERE7WNYTw/c3dMDjToJy3bliI5DV7HUUJtZv+8sLtVqEeThgMlR3UXHISJqVwvGhkImA74+XIRfL1SKjkNgqaE2UlZVj/U/Gq4EmBfbG7ZcDoGIrFxfXxdMGOQHAEjaweUTzAE/eahNrEw5jSuNOgzyd8XYft6i4xARdYj4Ub2gVMjxy5lL2Hvqoug4nR5LDd22MxdrsC2jAACQMJbLIRBR5+Hvbo9pQwIAGI7W6PQ8WiMSSw3dtqXJhjfyyFBPRAd3ER2HiKhDzbk3BE5qG5wsqcYXhwpFx+nUWGrotmSer8DOY6WQy4D5Y0NFxyEi6nBuDko8OyIEALBi9ynUN+oEJ+q8WGqo1SRJQuJ3JwEAj4T7o5eXk+BERERixN0VCB8XNQorr+D9tHOi43RaLDXUaruPl+Lg+ctQ28rx0iguh0BEnZfaVoH4q78HV/+Qi8o6reBEnRNLDbVKk06PJcmGozRP3BUEbxe14ERERGI9NLgbQr2dUFXfhHdSz4iO0ymx1FCrfJp5AWcu1sLN3hZPj+ghOg4RkXAKuQzzxxjOLdz0yzkUVl4RnKjzYakhk9Vpm/Dm7lMAgOfu6wlnta3gRERE5mFE7664M9gd2iY9/snlEzocSw2ZbMNPeSirbkA3Nzs8dieXQyAi+o1MJkPCWMNil9sPFeJ4UZXgRJ0LSw2Z5FJNA9btPQvAsByCykYhOBERkXkZ6O+KcQN8IEkwnntIHYOlhkzy9g+5qGloQj8/Zzw4wFd0HCIiszRvdG/YyGXYe+oifsktFx2n02CpoRbLv1SHDw+cBwAsGNMHcjmXQyAiup5ADwdMjTZ8PZ+44yT0XD6hQ7DUUIst25WDRp2Eu3t6YFhPD9FxiIjM2vMje8JBqcCRQg2+OVIsOk6nwFJDLfLrhUp8fbgIMhmwgMshEBHdkoejCk8NN0x5sXxnDrRNesGJrB9LDd2SJElI2mE42W3CID/09XURnIiIyDLMvDsIXZ1UyK/4/et7aj8sNXRLe09dxC9nLkGpkBunAScioluzV9pgbkxPAIYLLarrGwUnsm4sNXRTOv3vR2mmDQmAv7u94ERERJZlUoQ/grs6oKJWi3evTolB7YOlhm5q+6FCnCyphpPaBnPuDREdh4jI4tgo5Hg11nAu4ns/nUVpVb3gRNaLpYZuqL5RhxVXp/l+dkQI3ByUghMREVmm2L5eCA9wQ32jHiu/PyU6jtViqaEb2vzLORRp6uHjokbcXYGi4xARWSzD8gmGozXbMgqQW1YtOJF1Yqmh66qs02LNnlwAwEujekFty+UQiIhuR0SgO0bd4QW9BCxJ5mKX7aFVpWbNmjUIDAyEWq1GdHQ00tPTbzj22LFjmDhxIgIDAyGTybBy5cprxlRXV2Pu3LkICAiAnZ0dhg4dioyMjGZjZsyYAZlM1uw2ZsyY1sSnFngn9Qyq6pvQ28sJEwd3Ex2HiMgqzB/TG3IZsPt4KQ6eqxAdx+qYXGq2bduG+Ph4LFq0CFlZWRg4cCBiY2NRVlZ23fF1dXUIDg5GUlISvL29rztm5syZ2L17N7Zs2YIjR45g9OjRiImJQWFhYbNxY8aMQXFxsfH28ccfmxqfWqCw8go2/XIOADB/bG8ouBwCEVGbCPF0wqRIfwDAP747AUni8gltyeRSs2LFCsyaNQtxcXG44447sG7dOtjb22PDhg3XHR8ZGYlly5Zh8uTJUKlU1/z8ypUr+Oyzz7B06VLcc889CAkJweuvv46QkBCsXbu22ViVSgVvb2/jzc3NzdT41AL/3GWY+TI6yB339vYUHYeIyKrMjekFta0cWfmV2HmsVHQcq2JSqdFqtcjMzERMTMzvG5DLERMTg7S0tFYFaGpqgk6ng1qtbna/nZ0dfvrpp2b3paamwtPTE71798YzzzyDS5cu3XC7DQ0NqKqqanajWzteVIXthwxHyBLu7wOZjEdpiIjakpezGjOHBQMAlu48iSYdl09oKyaVmvLycuh0Onh5eTW738vLCyUlJa0K4OTkhCFDhmDx4sUoKiqCTqfDBx98gLS0NBQX/74A2JgxY/D+++8jJSUFS5Yswd69ezF27FjodLrrbjcxMREuLi7Gm7+/f6vydTZLkk9CkoBx/X0wyN9VdBwiIqv01PBguDsocfZiLbYdLBAdx2qYxdVPW7ZsgSRJ8PPzg0qlwqpVqzBlyhTI5b/Hmzx5Mv70pz+hf//+mDBhAr755htkZGQgNTX1uttMSEiARqMx3goK+JfmVn7JLcfeUxdhI5dhXmxv0XGIiKyWk9oWz99nmNB05fenUadtEpzIOphUajw8PKBQKFBa2vw7wNLS0hueBNwSPXr0wN69e1FTU4OCggKkp6ejsbERwcHBN3xMcHAwPDw8kJube92fq1QqODs7N7vRjen1EhKvLofwaHR3BHo4CE5ERGTdpkYHoLu7PS5WN+C9H/NEx7EKJpUapVKJ8PBwpKSkGO/T6/VISUnBkCFDbjuMg4MDfHx8cPnyZezcuRPjx4+/4dgLFy7g0qVL8PHxue3nJeCbI8U4UqiBg1KBF0b2FB2HiMjqKW3keOXqUfF3955BeU2D4ESWz+Svn+Lj47F+/Xps3rwZJ06cwDPPPIPa2lrExcUBAKZNm4aEhATjeK1Wi+zsbGRnZ0Or1aKwsBDZ2dnNjrDs3LkTycnJyMvLw+7du3HvvfciNDTUuM2amhrMmzcP+/fvx7lz55CSkoLx48cjJCQEsbGxt/v/oNPTNumxfKdhIqjZ9/SAh+O1V6kREVHbe6C/D/r7uaBWq8OqlNOi41g8k0vNpEmTsHz5cixcuBCDBg1CdnY2kpOTjScP5+fnNzvBt6ioCGFhYQgLC0NxcTGWL1+OsLAwzJw50zhGo9Fgzpw5CA0NxbRp0zBs2DDs3LkTtra2AACFQoFff/0Vf/rTn9CrVy88+eSTCA8Px48//njdy8TJNB8eOI/8ijp4OKow8+4g0XGIiDoNufz35RM+OpCPvPJawYksm0zqJDP/VFVVwcXFBRqNhufX/Jfq+kYMX5aKilot3vhzP0yNDhAdiYio05mxMR2pORcxrr8P1kwdLDqOWTHl89ssrn4icd7dexYVtVoEd3XApAhe9k5EJML8MaGQyYBvjxQju6BSdByLxVLTiZVW1eO9n84CAF6NDYWNgn8diIhE6OPjjIfCDOvsJXL5hFbjp1gntvL7U6hv1CM8wA2xfb1u/QAiImo38aN7QWkjx4G8CuzJuf56inRzLDWdVG5ZNbZlGCYkTBgbyuUQiIgE83O1Q9zQQADAkh050Ol5tMZULDWd1JLkHOglYNQdXogIdBcdh4iIADw7IgQudrbIKa3GZ1kXRMexOCw1ndDBcxXYfbwUchkwfwyXQyAiMhcu9raYc28PAMCbu0+hvvH66xvS9bHUdDKSJOEf350AAEyK9EeIp5PgRERE9N+mDQmEn6sdijX12PjzOdFxLApLTSez81gpsvIrobaVY25ML9FxiIjoD9S2CsSPMvx+fic1F5drtYITWQ6Wmk6kSafH0p2GRStnDguGl7NacCIiIrqeCWF+CPV2QnV9E9bsuf7CzXQtlppOZNvBApy9WAt3ByWeGn7jFdCJiEgshVyGBVeXT3g/7TwKKuoEJ7IMLDWdRJ22CSu/NyyW9vx9IXBS2wpORERENzO8V1cM7dEFWp0eK3afEh3HIrDUdBLv/ZiHi9UN6O5uz/WdiIgsgEwmQ8LYPgCAL7ILcaxIIziR+WOp6QTKaxrw7t4zAIBXYntDacOXnYjIEvTv5oIHB/pCkoCkHSdFxzF7/HTrBFalnEatVof+fi54oL+P6DhERGSCeaN7w1Yhw4+ny/HT6XLRccwaS42VyyuvxUcH8gEYlkOQy7kcAhGRJene5ffTBhJ3nICeyyfcEEuNlVu+MwdNeslwwlmIh+g4RETUCs/fFwJHlQ2OFVXh61+LRMcxWyw1Viy7oBLfHimGTAbjpYFERGR5ujiq8PTVqTiW7cxBQxOXT7gelhorJUkSEq8uh/DnMD/08XEWnIiIiG7HE8OC4OmkwoXLV/DB/nzRccwSS42V2pNThgN5FVDayPHyaC5aSURk6eyVNnjp6vIJq384jar6RsGJzA9LjRXS6SUs2ZEDAJgx1LAwGhERWb5HwruhR1cHXK5rxLrUM6LjmB2WGiv0WdYF5JRWw1ltg2dH9BAdh4iI2oiNQo75YwznSG74OQ8lmnrBicwLS42VqW/U4c2r02nPuTcErvZKwYmIiKgtjbrDCxEBbqhv1Bt/35MBS42V2fjzORRr6uHrosb0oYGi4xARURuTyWRIuN9wtObTzAKcKq0WnMh8sNRYkcu1WryTaliiPn50b6htFYITERFRewgPcEdsXy/oJWAJl08wYqmxImv25KK6vgmh3k74c5if6DhERNSOXh0TCoVchpSTZThw9pLoOGaBpcZKFFTU4f208wCA+WMNf9GJiMh69ejqiEmR/gCAxB0nIUlcPoGlxkqs2H0KWp0eQ4K7YESvrqLjEBFRB5g7sifsbBXILqjEjqMlouMIx1JjBY4VafBFdiEAIOH+UMhkPEpDRNQZeDqrMevuIACG5RMadXrBicRiqbECSTtOQpKABwf6YkA3V9FxiIioA80e3gNdHJTIK6/F1vTOvXwCS42F++l0OX48XQ5bhQzzuBwCEVGn46iywYsxPQEAb6WcRk1Dk+BE4rDUWDC9XkLiDsOilVOjA9C9i73gREREJMKUqO4I7GKP8hot1u87KzqOMCw1FuzrX4twrKgKjiobPH9fiOg4REQkiK1Cjnmxhgn51v94FmXVnXP5BJYaC9XQpMOynYZFK58eHowujirBiYiISKT7+3tjoL8r6rQ6rEo5LTqOECw1FuqD/fm4cPkKPJ1UeGJYkOg4REQkmEwmQ8JYw9Gaj9MLcPZijeBEHY+lxgJV1Tdi9Q+GFv7SqF6wV9oITkRERObgzuAuuC/UEzq9ZDya35mw1FigdalncLmuET26OuCR8G6i4xARkRmZPyYUchmw42gJsvIvi47ToVhqLEyJph4bfs4DYPiLa6PgS0hERL/r7e2EiYMN/+BN+q5zLZ/AT0QL8+buU6hv1CMiwA2j7vASHYeIiMxQ/OheUNnIkX6uAiknykTH6TAsNRbkdGk1Ps0sAMDlEIiI6MZ8XOwQd5fhIpIlySfR1EmWT2CpsSBLkk9CLwGxfb0QHuAuOg4REZmxZ0b0gKu9LU6X1eCzrAui43QIlhoLkZ5Xge9PlEEhl+HVMaGi4xARkZlzsbPFc/caJmZdsfsUrmh1ghO1P5YaCyBJEv7xnWE5hEmR/ujR1VFwIiIisgSPDwmAn6sdSqsajBeZWDOWGguw42gJsgsqYWerwNyRPUXHISIiC6GyUeCV2F4ADNOBVNRqBSdqX60qNWvWrEFgYCDUajWio6ORnp5+w7HHjh3DxIkTERgYCJlMhpUrV14zprq6GnPnzkVAQADs7OwwdOhQZGRkNBsjSRIWLlwIHx8f2NnZISYmBqdPW/800I06vXECpVl3B8HTWS04ERERWZLxA/1wh48zqhuasPqHXNFx2pXJpWbbtm2Ij4/HokWLkJWVhYEDByI2NhZlZde/ZKyurg7BwcFISkqCt7f3dcfMnDkTu3fvxpYtW3DkyBGMHj0aMTExKCwsNI5ZunQpVq1ahXXr1uHAgQNwcHBAbGws6uute9Guren5yCuvRRcHJWYP7yE6DhERWRi5XIYFV5dP2LL/HAoq6gQnakeSiaKioqQ5c+YY/6zT6SRfX18pMTHxlo8NCAiQ3nzzzWb31dXVSQqFQvrmm2+a3T948GDpr3/9qyRJkqTX6yVvb29p2bJlxp9XVlZKKpVK+vjjj1uUW6PRSAAkjUbTovHmoLq+UQpfvEsKmP+NtOnnPNFxiIjIgk1dv18KmP+N9MLHWaKjmMSUz2+TjtRotVpkZmYiJibGeJ9cLkdMTAzS0tJaVaqampqg0+mgVjf/WsXOzg4//fQTACAvLw8lJSXNntfFxQXR0dGtfl5LsH7fWZTXaBHQxR5TorqLjkNERBbst6M1X2YX4WihRnCa9mFSqSkvL4dOp4OXV/OZbL28vFBSUtKqAE5OThgyZAgWL16MoqIi6HQ6fPDBB0hLS0NxcTEAGLdtyvM2NDSgqqqq2c2SlFXXY/2PZwEA82J7Q2nDc7qJiKj1+vm5YPwgXwBA4o4TVrl8gll8Um7ZsgWSJMHPzw8qlQqrVq3ClClTIJe3Pl5iYiJcXFyMN39//zZM3P5WpZxGnVaHgd1cMK6/j+g4RERkBV4Z3RtKhRw/517CvtPlouO0OZNag4eHBxQKBUpLS5vdX1paesOTgFuiR48e2Lt3L2pqalBQUID09HQ0NjYiODgYAIzbNuV5ExISoNFojLeCgoJW5+toZy/W4ON0Q94FY/twOQQiImoT/u72eOzOAABA0o6T0Out62iNSaVGqVQiPDwcKSkpxvv0ej1SUlIwZMiQ2w7j4OAAHx8fXL58GTt37sT48eMBAEFBQfD29m72vFVVVThw4MANn1elUsHZ2bnZzVIs25kDnV7Cvb27YkiPLqLjEBGRFXnuvhA4qWxworgKX2QX3voBFsTk73fi4+Oxfv16bN68GSdOnMAzzzyD2tpaxMXFAQCmTZuGhIQE43itVovs7GxkZ2dDq9WisLAQ2dnZyM39/Vr5nTt3Ijk5GXl5edi9ezfuvfdehIaGGrcpk8kwd+5c/P3vf8dXX32FI0eOYNq0afD19cWECRNu83+BecnKv4wdR0sgkwHzx3I5BCIialvuDko8PcIwRcg/d51CfaP1LJ9gY+oDJk2ahIsXL2LhwoUoKSnBoEGDkJycbDyJNz8/v9m5MEVFRQgLCzP+efny5Vi+fDmGDx+O1NRUAIBGo0FCQgIuXLgAd3d3TJw4EW+88QZsbW2Nj3v11VdRW1uL2bNno7KyEsOGDUNycvI1V01ZMkmSkPTdSQDAxMHdEOptOUeXiIjIcjxxVxC2pJ1HYeUVbEk7j1n3BIuO1CZkkjWe/nwdVVVVcHFxgUajMduvor4/XoqZ7x+EykaOPa+MgK+rnehIRERkpbZl5GP+Z0fgYmeLffPuhYu97a0fJIApn99mcfUTAU06PZYkG47SzLgrkIWGiIja1cTB3dDT0xGaK414Z691LJ/AUmMmPsu6gNNlNXCxs8Wzw0NExyEiIitno5Bj/hjDuZsbfz6HosorghPdPpYaM3BFq8OK3acAAM/fF2K2hwCJiMi6jOzjiaggd2ib9MbPIUvGUmMGNvych9KqBvi52uHxIQGi4xARUSchk8mQcPVK28+yLuBkiWXNvv9HLDWCVdRqsS71DADgldheUNkoBCciIqLOJKy7G+7v7w1JApbsOCk6zm1hqRFs9Q+5qG5owh0+zhg/0E90HCIi6oTmxYbCRi7DnpyLSDtzSXScVmOpEaigog5b9p8DYFg9VS7ncghERNTxgjwcMCWqOwAgyYIXu2SpEWj5rhw06iQMC/HAPb26io5DRESd2Asje8JBqcDhCxp8e6RYdJxWYakR5GihBl9mFwEwHKUhIiISqauTyjiz8LKdOdA26QUnMh1LjSBJV0/GGj/IF/38XASnISIiAmbdHQwPRxXOX6rDx+n5ouOYjKVGgH2nLuKn3HIoFXK8Mrq36DhEREQAAAeVDV6M6QkAWJVyGtX1jYITmYalpoPp9ZLxKM1jdwbA391ecCIiIqLfTY70R7CHAy7VarF+31nRcUzCUtPBvjxciOPFVXBS2eC5+7gcAhERmRdbhRzzYg3fIqz/MQ9lVfWCE7UcS00Hqm/UYflOwzTUT4/oAXcHpeBERERE1xrTzxth3V1xpVGHlSmnRcdpMZaaDvTB/vMorLwCb2c1nrgrSHQcIiKi6zIsn9AHALAtowBnLtYITtQyLDUdRHOlEav3GJZ2f2lUT9gpuRwCERGZr6ggd8T08YROL2FpsmUsn8BS00HWpp5BZV0jeno6YuLgbqLjEBER3dL8MaGQy4Cdx0qReb5CdJxbYqnpAEWVV7Dh5zwAhr8gNgr+byciIvPX08sJj4T7AwASvztp9ssn8NO1A6zYfQraJj2iAt0xso+n6DhEREQt9tKoXlDbynHw/GXsPl4qOs5NsdS0s5MlVfgs6wIAYMH9oZDJuGglERFZDm+X3y9uWZJ8Ek06810+gaWmnS3ZcRKSBIzt543B3d1ExyEiIjLZ0yN6wM3eFmcu1uKTgxdEx7khlpp2lHbmEvbkXIRCLjNOZERERGRpnNW2eO4+w/IJb35/CnXaJsGJro+lpp1IkoSkHScAAFOi/BHc1VFwIiIiotZ77M7u6OZmh4vVDfj3j3mi41wXS007+fZIMQ5f0MBeqcCLI3uJjkNERHRbVDYK47cO7+47i0s1DYITXYulph1om/RYtjMHgGEZ965OKsGJiIiIbt+DA3zRz88ZNQ1NePuHXNFxrsFS0w4+Ts/H+Ut18HBUYtY9waLjEBERtQm5XIYFYwzLJ3x44DzOX6oVnKg5lpo2Vl3fiFVXF/96cWRPOKpsBCciIiJqO8N6euDunh5o1EnGbyXMBUtNG1u/7ywu1WoR5OGAyVHdRcchIiJqcwvGhkImA775tRiHCypFxzFiqWlDZVX1WH/1jPB5sb1hy+UQiIjICvX1dcGEQX4AgKQd5rN8Aj9129DKlNO40qjDIH9XjO3nLToOERFRu4kf1QtKhRxpZy8h9dRF0XEAsNS0mTMXa7AtowAAkDCWyyEQEZF183e3x/ShAQAMs+fr9OKP1rDUtJGlyYYXNKaPJ6KDu4iOQ0RE1O7m3BsCZ7UNTpZUY/uhQtFxWGraQub5Cuw8Vgq5DJg/JlR0HCIiog7haq/Es/eGAABW7MpBfaNOaB6WmtskSRISvzsJAHgk3B89vZwEJyIiIuo4M4YGwsdFjSJNPTb/ck5oFpaa27QnpwwHz1+G2laOl0ZxOQQiIupc1LYKxF/9/FuzJxdV9Y3CsnBmuNs0tIcH/np/HzTq9fB2UYuOQ0RE1OEeGtwN+06XY+JgPzgJnHRWJpnLxeXtrKqqCi4uLtBoNHB2dhYdh4iIiFrAlM9vfv1EREREVoGlhoiIiKwCSw0RERFZBZYaIiIisgosNURERGQVWGqIiIjIKrDUEBERkVVoValZs2YNAgMDoVarER0djfT09BuOPXbsGCZOnIjAwEDIZDKsXLnymjE6nQ6vvfYagoKCYGdnhx49emDx4sX47yl0ZsyYAZlM1uw2ZsyY1sQnIiIiK2TytH/btm1DfHw81q1bh+joaKxcuRKxsbHIycmBp6fnNePr6uoQHByMRx55BC+99NJ1t7lkyRKsXbsWmzdvRt++fXHw4EHExcXBxcUFL7zwgnHcmDFjsHHjRuOfVSqVqfGJiIjISplcalasWIFZs2YhLi4OALBu3Tp8++232LBhAxYsWHDN+MjISERGRgLAdX8OAL/88gvGjx+PcePGAQACAwPx8ccfX3MESKVSwdvb29TIRERE1AmY9PWTVqtFZmYmYmJift+AXI6YmBikpaW1OsTQoUORkpKCU6dOAQAOHz6Mn376CWPHjm02LjU1FZ6enujduzeeeeYZXLp06YbbbGhoQFVVVbMbERERWS+TjtSUl5dDp9PBy8ur2f1eXl44efJkq0MsWLAAVVVVCA0NhUKhgE6nwxtvvIGpU6cax4wZMwYPPfQQgoKCcObMGfzP//wPxo4di7S0NCgUimu2mZiYiL/97W+tzkRERESWxSxW6f7kk0/w4Ycf4qOPPkLfvn2RnZ2NuXPnwtfXF9OnTwcATJ482Ti+f//+GDBgAHr06IHU1FSMHDnymm0mJCQgPj7e+Oeqqir4+/u3/84QERGRECaVGg8PDygUCpSWlja7v7S09LbOdZk3bx4WLFhgLC79+/fH+fPnkZiYaCw1fxQcHAwPDw/k5uZet9SoVKpmJxL/diUVv4YiIiKyHL99bv/3FdE3YlKpUSqVCA8PR0pKCiZMmAAA0Ov1SElJwXPPPWd60qvq6uoglzc/vUehUECv19/wMRcuXMClS5fg4+PToueorq4GAB6tISIiskDV1dVwcXG56RiTv36Kj4/H9OnTERERgaioKKxcuRK1tbXGq6GmTZsGPz8/JCYmAjCcXHz8+HHjfxcWFiI7OxuOjo4ICQkBADz44IN444030L17d/Tt2xeHDh3CihUr8MQTTwAAampq8Le//Q0TJ06Et7c3zpw5g1dffRUhISGIjY1tUW5fX18UFBTAyckJMpnM1N2+qd++2iooKICzs3ObbtscWPv+Ada/j9w/y2ft+8j9s3zttY+SJKG6uhq+vr4tGmyyt99+W+revbukVCqlqKgoaf/+/cafDR8+XJo+fbrxz3l5eRKAa27Dhw83jqmqqpJefPFFqXv37pJarZaCg4Olv/71r1JDQ4MkSZJUV1cnjR49Wuratatka2srBQQESLNmzZJKSkpaE7/NaTQaCYCk0WhER2kX1r5/kmT9+8j9s3zWvo/cP8tnDvsok6QWfElFN1VVVQUXFxdoNBqrbODWvn+A9e8j98/yWfs+cv8snznsI9d+IiIiIqvAUtMGVCoVFi1aZLXLNlj7/gHWv4/cP8tn7fvI/bN85rCP/PqJiIiIrAKP1BAREZFVYKkhIiIiq8BSQ0RERFaBpYaIiIisAktNC61ZswaBgYFQq9WIjo5Genr6Tcd/+umnCA0NhVqtRv/+/fHdd991UNLWMWX/Nm3aBJlM1uymVqs7MK1p9u3bhwcffBC+vr6QyWT44osvbvmY1NRUDB48GCqVCiEhIdi0aVO757wdpu5jamrqNa+hTCZDSUlJxwQ2QWJiIiIjI+Hk5ARPT09MmDABOTk5t3ycJb0HW7OPlvQ+XLt2LQYMGABnZ2c4OztjyJAh2LFjx00fY0mvH2D6PlrS63c9SUlJkMlkmDt37k3HdfTryFLTAtu2bUN8fDwWLVqErKwsDBw4ELGxsSgrK7vu+F9++QVTpkzBk08+iUOHDmHChAmYMGECjh492sHJW8bU/QMAZ2dnFBcXG2/nz5/vwMSmqa2txcCBA7FmzZoWjc/Ly8O4ceNw7733GleMnzlzJnbu3NnOSVvP1H38TU5OTrPX0dPTs50Stt7evXsxZ84c7N+/H7t370ZjYyNGjx6N2traGz7G0t6DrdlHwHLeh926dUNSUhIyMzNx8OBB3HfffRg/fjyOHTt23fGW9voBpu8jYDmv3x9lZGTg3XffxYABA246TsjrKGwuYwsSFRUlzZkzx/hnnU4n+fr6SomJidcd/5e//EUaN25cs/uio6Olp556ql1ztpap+7dx40bJxcWlg9K1LQDS9u3bbzrm1Vdflfr27dvsvkmTJkmxsbHtmKzttGQf9+zZIwGQLl++3CGZ2lJZWZkEQNq7d+8Nx1jae/CPWrKPlvw+lCRJcnNzk957773r/szSX7/f3GwfLfX1q66ulnr27Cnt3r1bGj58uPTiiy/ecKyI15FHam5Bq9UiMzMTMTExxvvkcjliYmKQlpZ23cekpaU1Gw8AsbGxNxwvUmv2DzAsMhoQEAB/f/9b/mvE0ljS63e7Bg0aBB8fH4waNQo///yz6DgtotFoAADu7u43HGPpr2FL9hGwzPehTqfD1q1bUVtbiyFDhlx3jKW/fi3ZR8AyX785c+Zg3Lhx17w+1yPidWSpuYXy8nLodDp4eXk1u9/Ly+uG5x+UlJSYNF6k1uxf7969sWHDBnz55Zf44IMPoNfrMXToUFy4cKEjIre7G71+VVVVuHLliqBUbcvHxwfr1q3DZ599hs8++wz+/v4YMWIEsrKyREe7Kb1ej7lz5+Kuu+5Cv379bjjOkt6Df9TSfbS09+GRI0fg6OgIlUqFp59+Gtu3b8cdd9xx3bGW+vqZso+W9voBwNatW5GVlYXExMQWjRfxOtq025bJag0ZMqTZvz6GDh2KPn364N1338XixYsFJqOW6t27N3r37m3889ChQ3HmzBm8+eab2LJli8BkNzdnzhwcPXoUP/30k+go7aal+2hp78PevXsjOzsbGo0G//nPfzB9+nTs3bv3hh/6lsiUfbS016+goAAvvvgidu/ebdYnNLPU3IKHhwcUCgVKS0ub3V9aWgpvb+/rPsbb29uk8SK1Zv/+yNbWFmFhYcjNzW2PiB3uRq+fs7Mz7OzsBKVqf1FRUWZdFp577jl888032LdvH7p163bTsZb0HvxvpuzjH5n7+1CpVCIkJAQAEB4ejoyMDLz11lt49913rxlrqa+fKfv4R+b++mVmZqKsrAyDBw823qfT6bBv3z6sXr0aDQ0NUCgUzR4j4nXk10+3oFQqER4ejpSUFON9er0eKSkpN/yudMiQIc3GA8Du3btv+t2qKK3Zvz/S6XQ4cuQIfHx82itmh7Kk168tZWdnm+VrKEkSnnvuOWzfvh0//PADgoKCbvkYS3sNW7OPf2Rp70O9Xo+Ghobr/szSXr8budk+/pG5v34jR47EkSNHkJ2dbbxFRERg6tSpyM7OvqbQAIJex3Y7BdmKbN26VVKpVNKmTZuk48ePS7Nnz5ZcXV2lkpISSZIk6fHHH5cWLFhgHP/zzz9LNjY20vLly6UTJ05IixYtkmxtbaUjR46I2oWbMnX//va3v0k7d+6Uzpw5I2VmZkqTJ0+W1Gq1dOzYMVG7cFPV1dXSoUOHpEOHDkkApBUrVkiHDh2Szp8/L0mSJC1YsEB6/PHHjePPnj0r2dvbS/PmzZNOnDghrVmzRlIoFFJycrKoXbglU/fxzTfflL744gvp9OnT0pEjR6QXX3xRksvl0vfffy9qF27omWeekVxcXKTU1FSpuLjYeKurqzOOsfT3YGv20ZLehwsWLJD27t0r5eXlSb/++qu0YMECSSaTSbt27ZIkyfJfP0kyfR8t6fW7kT9e/WQOryNLTQu9/fbbUvfu3SWlUilFRUVJ+/fvN/5s+PDh0vTp05uN/+STT6RevXpJSqVS6tu3r/Ttt992cGLTmLJ/c+fONY718vKS7r//fikrK0tA6pb57fLlP95+26fp06dLw4cPv+YxgwYNkpRKpRQcHCxt3Lixw3ObwtR9XLJkidSjRw9JrVZL7u7u0ogRI6QffvhBTPhbuN5+AWj2mlj6e7A1+2hJ78MnnnhCCggIkJRKpdS1a1dp5MiRxg97SbL810+STN9HS3r9buSPpcYcXkeZJElS+x0HIiIiIuoYPKeGiIiIrAJLDREREVkFlhoiIiKyCiw1REREZBVYaoiIiMgqsNQQERGRVWCpISIiIqvAUkNERERWgaWGiIiIrAJLDREREVkFlhoiIiKyCiw1REREZBX+HySSUc/B8M55AAAAAElFTkSuQmCC",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "ks = 5\n",
-    "sigma = 2\n",
-    "half_ks = (ks - 1) // 2\n",
-    "base_kernel = [0.0] * half_ks + [1.0] + [0.0] * half_ks\n",
-    "kernel_window = gaussian_filter1d(base_kernel, sigma=sigma)\n",
-    "plt.plot(kernel_window)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8e6444b1",
-   "metadata": {},
-   "source": [
-    "# Label Distribution Smoothing - visualization\n",
-    "* visualization of  pytorch_widedeep.training._wd_dataset.WideDeepDataset._prepare_weights(...)\n",
-    "\n",
-    "**Assign weight to each sample by following procedure:**\n",
-    "1. creating histogram from label values with nuber of bins = granularity\n",
-    "2.[OPTIONAL] reweighting label frequencies by sqrt\n",
-    "3.[OPTIONAL] smoothing label frequencies by convolution of kernel function window with frequencies list\n",
-    "4. inverting values by n_samples / (n_classes * np.bincount(y)), [see](https://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_sample_weight.html)\n",
-    "5. assigning weight to each sample from closest bin value "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "45de4c6b",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAHVCAYAAAB8NLYkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABf8UlEQVR4nO3deVhUZf8G8HsAGZBlWGQRRcAlFRFRUEQFF0gyMi1zCw1NsxQ1wyWo3BdcSk1F0HpfNcvUFtRcUF5c0EJzSV+13EqTVMANRlARmef3hy/n58iwKTjD4f5c11w6z5zle+acOXPznGUUQggBIiIiIqr2jPRdABERERFVDgY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7omrA3d0dQ4YM0XcZsrdgwQI0bNgQxsbG8PHxeW7zHTJkCNzd3Z/b/Cpi7969UCgU2Lt3b5XPa9q0aVAoFLhx40aVz4tIrhjsiJ6z1atXQ6FQ4MiRIzpf79KlC7y8vJ55Ptu3b8e0adOeeTo1xa5duzBp0iR07NgRq1atwpw5c/RdkmzNmTMHmzZt0su8e/ToAVtbW2RmZhZ7LScnB3Xr1oW/vz80Go0eqiN6dib6LoCIynb27FkYGVXs77Dt27cjLi6O4a6cdu/eDSMjI/zrX/+CqampvssxGEFBQbh3716lvidz5szBG2+8gd69e1faNMtr+fLl8PLywgcffIB169ZpvfbRRx/hxo0bSEpKqvDnjchQcMslqgaUSiVq1aql7zIqJC8vT98lVEhWVhbMzc0Z6p5gZGQEMzMz2QQdDw8PTJ06Fd9++y127doltR8+fBgJCQmIiopCq1at9Fgh0bORxyeVSOaePMeuoKAA06dPR5MmTWBmZgZ7e3t06tQJycnJAB6dsxUXFwcAUCgU0qNIXl4exo8fD1dXVyiVSjRt2hSffvophBBa87137x7Gjh2LOnXqwMrKCq+++iquXLkChUKh1RNYdG7U77//jjfffBO2trbo1KkTAOC///0vhgwZgoYNG8LMzAzOzs54++23cfPmTa15FU3j3LlzGDRoEFQqFRwcHDB58mQIIZCeno5evXrB2toazs7O+Oyzz8r13j18+BAzZ85Eo0aNoFQq4e7ujo8++gj5+fnSMAqFAqtWrUJeXp70Xq1evVrn9EaPHg1LS0vcvXu32GsDBw6Es7MzCgsLAQCbN29GWFgYXFxcoFQq0ahRI8ycOVN6vSQlndd26dIlnbWdOXMGb7zxBuzs7GBmZgY/Pz9s2bJFa5iytpmK1FJ0usDvv/+Orl27onbt2qhXrx7mz59f6rSAR+91Xl4e1qxZI73XT54/mp2djSFDhsDGxgYqlQpDhw7V+X5//fXX8PX1hbm5Oezs7DBgwACkp6eXWUNUVBS8vb0xatQo3L9/H4WFhXjvvffg5uaGqVOnljk+kSHjoVgiPcnJydF5knhBQUGZ406bNg2xsbEYPnw42rVrB7VajSNHjuDYsWN48cUX8e677+Lq1atITk7G2rVrtcYVQuDVV1/Fnj17MGzYMPj4+GDnzp2YOHEirly5gkWLFknDDhkyBBs3bsTgwYPRvn177Nu3D2FhYSXW1bdvXzRp0gRz5syRQmJycjL++usvDB06FM7Ozjh9+jRWrlyJ06dP4+DBg1qBEwD69++P5s2bY+7cudi2bRtmzZoFOzs7rFixAt26dcO8efPwzTffYMKECWjbti2CgoJKfa+GDx+ONWvW4I033sD48eNx6NAhxMbG4o8//kBiYiIAYO3atVi5ciV+/fVXfPnllwCADh066Jxe//79ERcXh23btqFv375S+927d/HTTz9hyJAhMDY2BvDofEpLS0tERUXB0tISu3fvxpQpU6BWq7FgwYJS6y6v06dPo2PHjqhXrx6io6NhYWGBjRs3onfv3vjhhx/w2muvASh7m6mo27dv46WXXsLrr7+Ofv364fvvv8eHH36Ili1bokePHiWOt3btWqmGESNGAAAaNWqkNUy/fv3g4eGB2NhYHDt2DF9++SUcHR0xb948aZjZs2dj8uTJ6NevH4YPH47r169j6dKlCAoKwm+//QYbG5sSazAxMcHKlSvRoUMHzJw5E46Ojjh27BiSkpJQu3btCr8XRAZFENFztWrVKgGg1EeLFi20xnFzcxMRERHS81atWomwsLBS5xMZGSl0fcQ3bdokAIhZs2Zptb/xxhtCoVCICxcuCCGEOHr0qAAgxo0bpzXckCFDBAAxdepUqW3q1KkCgBg4cGCx+d29e7dY27fffisAiNTU1GLTGDFihNT28OFDUb9+faFQKMTcuXOl9tu3bwtzc3Ot90SX48ePCwBi+PDhWu0TJkwQAMTu3bultoiICGFhYVHq9IQQQqPRiHr16ok+ffpotW/cuLHYMula9nfffVfUrl1b3L9/X2vebm5u0vM9e/YIAGLPnj1a4168eFEAEKtWrZLagoODRcuWLbWmp9FoRIcOHUSTJk2ktvJsM7roqqVz584CgPjqq6+ktvz8fOHs7FzsfdHFwsJC57or2gbefvttrfbXXntN2NvbS88vXbokjI2NxezZs7WGO3nypDAxMSnWXpLRo0eLWrVqCUtLS53bLlF1xEOxRHoSFxeH5OTkYg9vb+8yx7WxscHp06dx/vz5Cs93+/btMDY2xtixY7Xax48fDyEEduzYAQBISkoCAIwaNUpruDFjxpQ47ffee69Ym7m5ufT/+/fv48aNG2jfvj0A4NixY8WGHz58uPR/Y2Nj+Pn5QQiBYcOGSe02NjZo2rQp/vrrrxJrAR4tK/Do0Nvjxo8fDwDYtm1bqeProlAo0LdvX2zfvh25ublS+4YNG1CvXj3pEDSgvex37tzBjRs3EBgYiLt37+LMmTMVnveTbt26hd27d6Nfv37S9G/cuIGbN28iNDQU58+fx5UrVwA82zaji6WlJQYNGiQ9NzU1Rbt27cpcJ+Xx5HYUGBiImzdvQq1WAwB+/PFHaDQa9OvXT1rmGzduwNnZGU2aNMGePXvKNZ/Zs2fD3t4eRkZGWj3VRNUZgx2RnrRr1w4hISHFHra2tmWOO2PGDGRnZ+OFF15Ay5YtMXHiRPz3v/8t13z//vtvuLi4wMrKSqu9efPm0utF/xoZGcHDw0NruMaNG5c47SeHBR6Fj/fffx9OTk4wNzeHg4ODNFxOTk6x4Rs0aKD1XKVSwczMDHXq1CnWfvv27RJreXwZnqzZ2dkZNjY20rJWVP/+/XHv3j3pPLbc3Fxs374dffv21Tq0fPr0abz22mtQqVSwtraGg4ODFIZ0LXtFXbhwAUIITJ48GQ4ODlqPonPFsrKyADzbNqNL/fr1ix1Gt7W1LXOdlMeT20DRZ6Jo2ufPn4cQAk2aNCm23H/88Ye0zGWxtrZG06ZN4erqCicnp2eum8gQ8Bw7omooKCgIf/75JzZv3oxdu3bhyy+/xKJFi5CQkKDV4/W8Pd5DVaRfv3745ZdfMHHiRPj4+MDS0hIajQYvvfSSznuFFZ2fVlYbgGIXe5TkyQDyrNq3bw93d3ds3LgRb775Jn766Sfcu3cP/fv3l4bJzs5G586dYW1tjRkzZqBRo0YwMzPDsWPH8OGHH5Z6n7SS6n3yoouiaUyYMAGhoaE6xykKtZW9zTzrOnmWaWs0GigUCuzYsUPnsJaWls9cA1F1xWBHVE3Z2dlh6NChGDp0KHJzcxEUFIRp06ZJX9IlhQM3Nzf85z//wZ07d7R67YoODbq5uUn/ajQaXLx4EU2aNJGGu3DhQrlrvH37NlJSUjB9+nRMmTJFaq+sw4FlKVqG8+fPSz2SAJCZmYns7GxpWZ9Gv3798Pnnn0OtVmPDhg1wd3eXDjEDj64mvXnzJn788UetCzwuXrxY5rSLeqiys7O12p/sYWzYsCEAoFatWggJCSlzumVtM8/LswbtRo0aQQgBDw8PvPDCC5VUFZE88FAsUTX05K1CLC0t0bhxY61beFhYWAAoHg5efvllFBYWYtmyZVrtixYtgkKhkK5oLOoBWr58udZwS5cuLXedRb0pT/biLF68uNzTeBYvv/yyzvktXLgQAEq9wrcs/fv3R35+PtasWYOkpCT069dP63Vdy/7gwYNi76cubm5uMDY2Rmpqqlb7k+M6OjqiS5cuWLFiBa5du1ZsOtevX5f+X55t5nmxsLAotl1WxOuvvw5jY2NMnz692LYlhCi2rEQ1CXvsiKohT09PdOnSBb6+vrCzs8ORI0fw/fffY/To0dIwvr6+AICxY8ciNDQUxsbGGDBgAHr27ImuXbvi448/xqVLl9CqVSvs2rULmzdvxrhx46RbT/j6+qJPnz5YvHgxbt68Kd3u5Ny5cwDK1+tibW2NoKAgzJ8/HwUFBahXrx527dpVrl6rytCqVStERERg5cqV0qHRX3/9FWvWrEHv3r3RtWvXp552mzZt0LhxY3z88cfIz8/XOgwLPLpdiq2tLSIiIjB27FgoFAqsXbu2XIcqVSoV+vbti6VLl0KhUKBRo0bYunWrznPH4uLi0KlTJ7Rs2RLvvPMOGjZsiMzMTKSlpeGff/7BiRMnAJRvm3lefH198Z///AcLFy6Ei4sLPDw84O/vX+7xGzVqhFmzZiEmJgaXLl1C7969YWVlhYsXLyIxMREjRozAhAkTqnAJiAwXgx1RNTR27Fhs2bIFu3btQn5+Ptzc3DBr1ixMnDhRGub111/HmDFjsH79enz99dcQQmDAgAEwMjLCli1bMGXKFGzYsAGrVq2Cu7s7FixYIF0tWuSrr76Cs7Mzvv32WyQmJiIkJAQbNmxA06ZNYWZmVq5a161bhzFjxiAuLg5CCHTv3h07duyAi4tLpb4nJfnyyy/RsGFDrF69GomJiXB2dkZMTEyl3Ii2f//+mD17Nho3bow2bdpovWZvb4+tW7di/Pjx+OSTT2Bra4tBgwYhODi4xPPhHrd06VIUFBQgISEBSqUS/fr1w4IFC4r9jrCnpyeOHDmC6dOnY/Xq1bh58yYcHR3RunVrrcPf5dlmnpeFCxdixIgR+OSTT3Dv3j1ERERUKNgBQHR0NF544QUsWrQI06dPBwC4urqie/fuePXVV6uibKJqQSEq40xXIqoxjh8/jtatW+Prr79GeHi4vsshIqLH8Bw7IirRvXv3irUtXrwYRkZGZf7iAxERPX88FEtEJZo/fz6OHj2Krl27wsTEBDt27MCOHTswYsQIuLq66rs8IiJ6Ag/FElGJkpOTMX36dPz+++/Izc1FgwYNMHjwYHz88ccwMeHfhUREhobBjoiIiEgmeI4dERERkUzI9liKRqPB1atXYWVlVek/J0RERET0vAghcOfOHbi4uMDIqPQ+OdkGu6tXr/LkbiIiIpKN9PR01K9fv9RhZBvsin4DMz09HdbW1nquhoiIiOjpqNVquLq6av2+d0lkG+yKDr9aW1sz2BEREVG1V55Ty3jxBBEREZFMMNgRERERyQSDHREREZFMMNgRERERyUSFg11qaip69uwJFxcXKBQKbNq0Set1IQSmTJmCunXrwtzcHCEhITh//rzWMLdu3UJ4eDisra1hY2ODYcOGITc3V2uY//73vwgMDISZmRlcXV0xf/78ii8dERERUQ1S4WCXl5eHVq1aIS4uTufr8+fPx5IlS5CQkIBDhw7BwsICoaGhuH//vjRMeHg4Tp8+jeTkZGzduhWpqakYMWKE9LparUb37t3h5uaGo0ePYsGCBZg2bRpWrlz5FItIRERE9Ozco7cVexiaZ/qtWIVCgcTERPTu3RvAo946FxcXjB8/HhMmTAAA5OTkwMnJCatXr8aAAQPwxx9/wNPTE4cPH4afnx8AICkpCS+//DL++ecfuLi4ID4+Hh9//DEyMjJgamoKAIiOjsamTZtw5swZnbXk5+cjPz9fel50z5ecnBze7oSIiIiema4gd2luWJXPV61WQ6VSlSvTVOo5dhcvXkRGRgZCQkKkNpVKBX9/f6SlpQEA0tLSYGNjI4U6AAgJCYGRkREOHTokDRMUFCSFOgAIDQ3F2bNncfv2bZ3zjo2NhUqlkh781QkiIiKqaSo12GVkZAAAnJyctNqdnJyk1zIyMuDo6Kj1uomJCezs7LSG0TWNx+fxpJiYGOTk5EiP9PT0Z18gIiIiompENr88oVQqoVQq9V0GERERkd5Uao+ds7MzACAzM1OrPTMzU3rN2dkZWVlZWq8/fPgQt27d0hpG1zQenwcRERERaavUYOfh4QFnZ2ekpKRIbWq1GocOHUJAQAAAICAgANnZ2Th69Kg0zO7du6HRaODv7y8Nk5qaioKCAmmY5ORkNG3aFLa2tpVZMhEREZFsVDjY5ebm4vjx4zh+/DiARxdMHD9+HJcvX4ZCocC4ceMwa9YsbNmyBSdPnsRbb70FFxcX6crZ5s2b46WXXsI777yDX3/9FT///DNGjx6NAQMGwMXFBQDw5ptvwtTUFMOGDcPp06exYcMGfP7554iKiqq0BSciIiKSmwqfY3fkyBF07dpVel4UtiIiIrB69WpMmjQJeXl5GDFiBLKzs9GpUyckJSXBzMxMGuebb77B6NGjERwcDCMjI/Tp0wdLliyRXlepVNi1axciIyPh6+uLOnXqYMqUKVr3uiMiIiIibc90HztDVpF7vhARERGVpcbdx46IiIiI9IfBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZKLSg11hYSEmT54MDw8PmJubo1GjRpg5cyaEENIwQghMmTIFdevWhbm5OUJCQnD+/Hmt6dy6dQvh4eGwtraGjY0Nhg0bhtzc3Moul4iIiEg2Kj3YzZs3D/Hx8Vi2bBn++OMPzJs3D/Pnz8fSpUulYebPn48lS5YgISEBhw4dgoWFBUJDQ3H//n1pmPDwcJw+fRrJycnYunUrUlNTMWLEiMoul4iIiEg2FOLxrrRK8Morr8DJyQn/+te/pLY+ffrA3NwcX3/9NYQQcHFxwfjx4zFhwgQAQE5ODpycnLB69WoMGDAAf/zxBzw9PXH48GH4+fkBAJKSkvDyyy/jn3/+gYuLS5l1qNVqqFQq5OTkwNraujIXkYiIiGog9+htxdouzQ2r8vlWJNNUeo9dhw4dkJKSgnPnzgEATpw4gQMHDqBHjx4AgIsXLyIjIwMhISHSOCqVCv7+/khLSwMApKWlwcbGRgp1ABASEgIjIyMcOnRI53zz8/OhVqu1HkREREQ1iUllTzA6OhpqtRrNmjWDsbExCgsLMXv2bISHhwMAMjIyAABOTk5a4zk5OUmvZWRkwNHRUbtQExPY2dlJwzwpNjYW06dPr+zFISIiIqo2Kr3HbuPGjfjmm2+wbt06HDt2DGvWrMGnn36KNWvWVPastMTExCAnJ0d6pKenV+n8iIiIiAxNpffYTZw4EdHR0RgwYAAAoGXLlvj7778RGxuLiIgIODs7AwAyMzNRt25dabzMzEz4+PgAAJydnZGVlaU13YcPH+LWrVvS+E9SKpVQKpWVvThERERE1Ual99jdvXsXRkbakzU2NoZGowEAeHh4wNnZGSkpKdLrarUahw4dQkBAAAAgICAA2dnZOHr0qDTM7t27odFo4O/vX9klExEREclCpffY9ezZE7Nnz0aDBg3QokUL/Pbbb1i4cCHefvttAIBCocC4ceMwa9YsNGnSBB4eHpg8eTJcXFzQu3dvAEDz5s3x0ksv4Z133kFCQgIKCgowevRoDBgwoFxXxBIRERHVRJUe7JYuXYrJkydj1KhRyMrKgouLC959911MmTJFGmbSpEnIy8vDiBEjkJ2djU6dOiEpKQlmZmbSMN988w1Gjx6N4OBgGBkZoU+fPliyZElll0tEREQkG5V+HztDwfvYERERUWWqkfexIyIiIiL9YLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZqJJgd+XKFQwaNAj29vYwNzdHy5YtceTIEel1IQSmTJmCunXrwtzcHCEhITh//rzWNG7duoXw8HBYW1vDxsYGw4YNQ25ublWUS0RERCQLlR7sbt++jY4dO6JWrVrYsWMHfv/9d3z22WewtbWVhpk/fz6WLFmChIQEHDp0CBYWFggNDcX9+/elYcLDw3H69GkkJydj69atSE1NxYgRIyq7XCIiIiLZUAghRGVOMDo6Gj///DP279+v83UhBFxcXDB+/HhMmDABAJCTkwMnJyesXr0aAwYMwB9//AFPT08cPnwYfn5+AICkpCS8/PLL+Oeff+Di4lJmHWq1GiqVCjk5ObC2tq68BSQiIqIayT16W7G2S3PDqny+Fck0ld5jt2XLFvj5+aFv375wdHRE69at8cUXX0ivX7x4ERkZGQgJCZHaVCoV/P39kZaWBgBIS0uDjY2NFOoAICQkBEZGRjh06JDO+ebn50OtVms9iIiIiGqSSg92f/31F+Lj49GkSRPs3LkTI0eOxNixY7FmzRoAQEZGBgDAyclJazwnJyfptYyMDDg6Omq9bmJiAjs7O2mYJ8XGxkKlUkkPV1fXyl40IiIiIoNW6cFOo9GgTZs2mDNnDlq3bo0RI0bgnXfeQUJCQmXPSktMTAxycnKkR3p6epXOj4iIiMjQVHqwq1u3Ljw9PbXamjdvjsuXLwMAnJ2dAQCZmZlaw2RmZkqvOTs7IysrS+v1hw8f4tatW9IwT1IqlbC2ttZ6EBEREdUklR7sOnbsiLNnz2q1nTt3Dm5ubgAADw8PODs7IyUlRXpdrVbj0KFDCAgIAAAEBAQgOzsbR48elYbZvXs3NBoN/P39K7tkIiIiIlkwqewJfvDBB+jQoQPmzJmDfv364ddff8XKlSuxcuVKAIBCocC4ceMwa9YsNGnSBB4eHpg8eTJcXFzQu3dvAI96+F566SXpEG5BQQFGjx6NAQMGlOuKWCK50tcVWUREVD1UerBr27YtEhMTERMTgxkzZsDDwwOLFy9GeHi4NMykSZOQl5eHESNGIDs7G506dUJSUhLMzMykYb755huMHj0awcHBMDIyQp8+fbBkyZLKLpeIiIhINir9PnaGgvexIzlijx0Rkf7UyPvYEREREZF+MNgRERERyQSDHREREZFMMNgRERERyQSDHREREZFMMNgRERERyQSDHREREZFMMNgRERERyUSl//IEUU3EGwcTEZEhYLAjMlC6wiIREVFpeCiWiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkghdPkEHh1aVERERPjz12RERERDLBYEdEREQkEwx2RERERDLBc+xINqrz+Xm8GTEREVUGBjuiKlJSWKsuYZOIiKofBjsiKqY6934SEdVkDHakFzz0SEREVPmq/OKJuXPnQqFQYNy4cVLb/fv3ERkZCXt7e1haWqJPnz7IzMzUGu/y5csICwtD7dq14ejoiIkTJ+Lhw4dVXS4RERFRtVWlPXaHDx/GihUr4O3trdX+wQcfYNu2bfjuu++gUqkwevRovP766/j5558BAIWFhQgLC4OzszN++eUXXLt2DW+99RZq1aqFOXPmVGXJVAXYO2fYuH6IiOSjynrscnNzER4eji+++AK2trZSe05ODv71r39h4cKF6NatG3x9fbFq1Sr88ssvOHjwIABg165d+P333/H111/Dx8cHPXr0wMyZMxEXF4cHDx7onF9+fj7UarXWg4iIiKgmqbJgFxkZibCwMISEhGi1Hz16FAUFBVrtzZo1Q4MGDZCWlgYASEtLQ8uWLeHk5CQNExoaCrVajdOnT+ucX2xsLFQqlfRwdXWtgqUiIiIiMlxVcih2/fr1OHbsGA4fPlzstYyMDJiamsLGxkar3cnJCRkZGdIwj4e6oteLXtMlJiYGUVFR0nO1Ws1wJxO8QvPp8H0jIqp5Kj3Ypaen4/3330dycjLMzMwqe/IlUiqVUCqVz21+RE+rss9p4zlyRERUpNKD3dGjR5GVlYU2bdpIbYWFhUhNTcWyZcuwc+dOPHjwANnZ2Vq9dpmZmXB2dgYAODs749dff9WabtFVs0XDEJUHbxJMREQ1SaWfYxccHIyTJ0/i+PHj0sPPzw/h4eHS/2vVqoWUlBRpnLNnz+Ly5csICAgAAAQEBODkyZPIysqShklOToa1tTU8PT0ru2QiIiIiWaj0HjsrKyt4eXlptVlYWMDe3l5qHzZsGKKiomBnZwdra2uMGTMGAQEBaN++PQCge/fu8PT0xODBgzF//nxkZGTgk08+QWRkJA+3UqWoqeefPcthW/Z+EhEZPr388sSiRYtgZGSEPn36ID8/H6GhoVi+fLn0urGxMbZu3YqRI0ciICAAFhYWiIiIwIwZM/RRLqFmBKHyLiPPaSMiIkP1XILd3r17tZ6bmZkhLi4OcXFxJY7j5uaG7du3V3FlRFRd1IQ/LoiInlWV/6QYERERET0fejkUS1Rd8LArERFVJ+yxIyIiIpIJ9thRtcSeNCIiouIY7IjIoDC0ExE9PQY70sIvVSIiouqL59gRERERyQR77KhSscePKoLbCxFR5WKPHREREZFMsMeOiKqt8vb48RcqiKimYLAjomfCn/oiIjIcDHZEJHsMn0RUUzDY1RD8YiMiIpI/BrsajFckEhERyQuDHRE9F4b2hwR7sYlIjni7EyIiIiKZYI8dPTVD64EhIiKq6RjsiKjSMfQTEekHgx0R0f+UFEh57h0RVRc8x46IiIhIJthjR0RUhSpy9S2v1CWiZ1XpwS42NhY//vgjzpw5A3Nzc3To0AHz5s1D06ZNpWHu37+P8ePHY/369cjPz0doaCiWL18OJycnaZjLly9j5MiR2LNnDywtLREREYHY2FiYmDCLEpFhKu+5hTwHkYiqSqUfit23bx8iIyNx8OBBJCcno6CgAN27d0deXp40zAcffICffvoJ3333Hfbt24erV6/i9ddfl14vLCxEWFgYHjx4gF9++QVr1qzB6tWrMWXKlMoul4iIiEg2FEIIUZUzuH79OhwdHbFv3z4EBQUhJycHDg4OWLduHd544w0AwJkzZ9C8eXOkpaWhffv22LFjB1555RVcvXpV6sVLSEjAhx9+iOvXr8PU1LTYfPLz85Gfny89V6vVcHV1RU5ODqytratyEasF9hAQVS5dh0ir4nPGQ7FEhkNfp0uo1WqoVKpyZZoqv3giJycHAGBnZwcAOHr0KAoKChASEiIN06xZMzRo0ABpaWkAgLS0NLRs2VLr0GxoaCjUajVOnz6tcz6xsbFQqVTSw9XVtaoWiYiIiMggVWmw02g0GDduHDp27AgvLy8AQEZGBkxNTWFjY6M1rJOTEzIyMqRhHg91Ra8XvaZLTEwMcnJypEd6enolLw0RERGRYavSKxEiIyNx6tQpHDhwoCpnAwBQKpVQKpVVPh8iIoCnNxCRYaqyHrvRo0dj69at2LNnD+rXry+1Ozs748GDB8jOztYaPjMzE87OztIwmZmZxV4veo2IiIiIiqv0HjshBMaMGYPExETs3bsXHh4eWq/7+vqiVq1aSElJQZ8+fQAAZ8+exeXLlxEQEAAACAgIwOzZs5GVlQVHR0cAQHJyMqytreHp6VnZJcsOexKIqKbgr4UQaav0YBcZGYl169Zh8+bNsLKyks6JU6lUMDc3h0qlwrBhwxAVFQU7OztYW1tjzJgxCAgIQPv27QEA3bt3h6enJwYPHoz58+cjIyMDn3zyCSIjI3m4lYiIiKgElR7s4uPjAQBdunTRal+1ahWGDBkCAFi0aBGMjIzQp08frRsUFzE2NsbWrVsxcuRIBAQEwMLCAhEREZgxY0Zll0tEZNDK2wPPHioiAqroUGxZzMzMEBcXh7i4uBKHcXNzw/bt2yuzNCIi2eLPkRER8BzuY0dEREREzweDHREREZFMMNgRERERyUSV3qCYiIjoceU9F5DnDBI9HQY7IiLSK957k6jyMNgREdUwz6M3jGGNSD8Y7IiI6JkwxBEZDgY7IiKZetbAZWiBrSqWh+ftkdww2BERkcGFOCJ6Ogx21Rx3xkRExZV338hePJIb3seOiIiISCbYY0dERFSGknoA2btHhoY9dkREREQywR47IiKix/DcZarOGOyIiIie0rNcfFGRAMlDvlReDHZERESViFfa/j++F88fgx0REVEV0+fNlRmuahYGu2qC53wQEdHjnuV7oSJhj8GwemGwIyIiogphZ4PhYrAjIiKi5xbWeNFI1WKwIyIiomqNh5b/n0EHu7i4OCxYsAAZGRlo1aoVli5dinbt2um7LCIiInoOKvs8wprAYIPdhg0bEBUVhYSEBPj7+2Px4sUIDQ3F2bNn4ejoqO/yiIiISCbkFAIVQgih7yJ08ff3R9u2bbFs2TIAgEajgaurK8aMGYPo6Ogyx1er1VCpVMjJyYG1tXVVl/vU5LQxERER1TTP4zBuRTKNQfbYPXjwAEePHkVMTIzUZmRkhJCQEKSlpekcJz8/H/n5+dLznJwcAI/ejKrmNXVnlc+DiIiIDM/zyBlF8yhPX5xBBrsbN26gsLAQTk5OWu1OTk44c+aMznFiY2Mxffr0Yu2urq5VUiMRERGRavHzm9edO3egUqlKHcYgg93TiImJQVRUlPRco9Hg1q1bsLe3h0KhKHN8tVoNV1dXpKenG/Sh25qO66l64HqqHrieqgeup+qhKteTEAJ37tyBi4tLmcMaZLCrU6cOjI2NkZmZqdWemZkJZ2dnneMolUoolUqtNhsbmwrP29ramh+caoDrqXrgeqoeuJ6qB66n6qGq1lNZPXVFjCp9zpXA1NQUvr6+SElJkdo0Gg1SUlIQEBCgx8qIiIiIDJdB9tgBQFRUFCIiIuDn54d27dph8eLFyMvLw9ChQ/VdGhEREZFBMthg179/f1y/fh1TpkxBRkYGfHx8kJSUVOyCisqiVCoxderUYodzybBwPVUPXE/VA9dT9cD1VD0Yynoy2PvYEREREVHFGOQ5dkRERERUcQx2RERERDLBYEdEREQkEwx2RERERDLBYPc/cXFxcHd3h5mZGfz9/fHrr7/quyR6TGpqKnr27AkXFxcoFAps2rRJ3yWRDrGxsWjbti2srKzg6OiI3r174+zZs/oui54QHx8Pb29v6UaqAQEB2LFjh77LolLMnTsXCoUC48aN03cp9IRp06ZBoVBoPZo1a6a3ehjsAGzYsAFRUVGYOnUqjh07hlatWiE0NBRZWVn6Lo3+Jy8vD61atUJcXJy+S6FS7Nu3D5GRkTh48CCSk5NRUFCA7t27Iy8vT9+l0WPq16+PuXPn4ujRozhy5Ai6deuGXr164fTp0/oujXQ4fPgwVqxYAW9vb32XQiVo0aIFrl27Jj0OHDigt1p4uxMA/v7+aNu2LZYtWwbg0a9cuLq6YsyYMYiOjtZzdfQkhUKBxMRE9O7dW9+lUBmuX78OR0dH7Nu3D0FBQfouh0phZ2eHBQsWYNiwYfouhR6Tm5uLNm3aYPny5Zg1axZ8fHywePFifZdFj5k2bRo2bdqE48eP67sUAOyxw4MHD3D06FGEhIRIbUZGRggJCUFaWpoeKyOq/nJycgA8Cg1kmAoLC7F+/Xrk5eXxJxsNUGRkJMLCwrS+o8jwnD9/Hi4uLmjYsCHCw8Nx+fJlvdVisL888bzcuHEDhYWFxX7RwsnJCWfOnNFTVUTVn0ajwbhx49CxY0d4eXnpuxx6wsmTJxEQEID79+/D0tISiYmJ8PT01HdZ9Jj169fj2LFjOHz4sL5LoVL4+/tj9erVaNq0Ka5du4bp06cjMDAQp06dgpWV1XOvp8YHOyKqGpGRkTh16pRezzWhkjVt2hTHjx9HTk4Ovv/+e0RERGDfvn0MdwYiPT0d77//PpKTk2FmZqbvcqgUPXr0kP7v7e0Nf39/uLm5YePGjXo5taHGB7s6derA2NgYmZmZWu2ZmZlwdnbWU1VE1dvo0aOxdetWpKamon79+vouh3QwNTVF48aNAQC+vr44fPgwPv/8c6xYsULPlREAHD16FFlZWWjTpo3UVlhYiNTUVCxbtgz5+fkwNjbWY4VUEhsbG7zwwgu4cOGCXuZf48+xMzU1ha+vL1JSUqQ2jUaDlJQUnm9CVEFCCIwePRqJiYnYvXs3PDw89F0SlZNGo0F+fr6+y6D/CQ4OxsmTJ3H8+HHp4efnh/DwcBw/fpyhzoDl5ubizz//RN26dfUy/xrfYwcAUVFRiIiIgJ+fH9q1a4fFixcjLy8PQ4cO1Xdp9D+5ublaf/1cvHgRx48fh52dHRo0aKDHyuhxkZGRWLduHTZv3gwrKytkZGQAAFQqFczNzfVcHRWJiYlBjx490KBBA9y5cwfr1q3D3r17sXPnTn2XRv9jZWVV7NxUCwsL2Nvb85xVAzNhwgT07NkTbm5uuHr1KqZOnQpjY2MMHDhQL/Uw2AHo378/rl+/jilTpiAjIwM+Pj5ISkoqdkEF6c+RI0fQtWtX6XlUVBQAICIiAqtXr9ZTVfSk+Ph4AECXLl202letWoUhQ4Y8/4JIp6ysLLz11lu4du0aVCoVvL29sXPnTrz44ov6Lo2o2vnnn38wcOBA3Lx5Ew4ODujUqRMOHjwIBwcHvdTD+9gRERERyUSNP8eOiIiISC4Y7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7AzU3r17oVAosHfvXn2XotOlS5egUCiwevXqcg/76aefPtW8DP29eJ5Wr14NhUKBI0eOVNo0hwwZAnd390qbnr5UZJusSkXr6NKlS089bmWu34o4f/48unfvDpVKBYVCgU2bNumljqrUpUsXeHl5PfX4CoUCo0ePLnO4Z9kOSpKbm4vhw4fD2dkZCoUC48aNq7RpV7aK7PenTZsGhULxHKqqGWpMsCv6kD3+cHR0RNeuXbFjxw59l1eptm/fjmnTptWY+RLVRMuXL6/0EBsREYGTJ09i9uzZWLt2Lfz8/Cp1+vRs5syZg9WrV2PkyJFYu3YtBg8erO+S9Obq1auYNm0ajh8/ru9SDE6NCXZFZsyYgbVr1+Krr77CpEmTcP36dbz88svYunWrvkurNNu3b8f06dOrdB5ubm64d++e1o6lquYbFBSEe/fuISgoqNKnTVTZBg8ejHv37sHNza1K51PZwe7evXtIS0vDsGHDMHr0aAwaNAj169evtOnXNFWxHezevRvt27fH1KlTMWjQIPj6+lbatPXpk08+wb179yo0ztWrVzF9+nQGOx1M9F3A89ajRw+tv0KHDRsGJycnfPvtt3jllVeeefoajQYPHjyAmZnZM0/LkCkUiue2jEZGRgb9fj58+BAajQampqb6LoUMgLGxMYyNjfVdRoVdv34dAGBjY1Np08zLy4OFhUWlTa86qYrtICsrC56enmUOd//+fZiamsLIqHr03ZiYmMDEpHrFEUPe71ePtV6FbGxsYG5uXmyj+vTTT9GhQwfY29vD3Nwcvr6++P7774uNX3S+xTfffIMWLVpAqVQiKSmpxPm5u7vjlVdewa5du+Dj4wMzMzN4enrixx9/LFe93333HXx9fWFubo46depg0KBBuHLlivT6kCFDEBcXJ9VW9ChJVFQU7O3tIYSQ2saMGQOFQoElS5ZIbZmZmVAoFIiPjwdQ/Hym8s535cqVaNSoEZRKJdq2bYvDhw+Xucy6zrErOk/m999/R9euXVG7dm3Uq1cP8+fP16rZxMREZy/i2bNnoVAosGzZMqktOzsb48aNg6urK5RKJRo3box58+ZBo9FIwzx+3sjixYulZfn9998BAEuXLkWLFi1Qu3Zt2Nraws/PD+vWrdOa95UrV/D222/DyckJSqUSLVq0wL///e8y34fH3b17F++++y7s7e1hbW2Nt956C7dv39YaZvPmzQgLC4OLiwuUSiUaNWqEmTNnorCwsMzpV3T737RpE7y8vKTl0fUZuHLlCoYNGybV4+HhgZEjR+LBgwfSMOVZB0XDDRkyBCqVCjY2NoiIiEB2dnaZy5WdnQ1jY2OtbfvGjRswMjIq9jkYOXIknJ2dtcY/dOgQXnrpJahUKtSuXRudO3fGzz//rDWMrnOrNBoNpk2bBhcXF9SuXRtdu3bF77//Dnd3dwwZMqRYnfn5+YiKioKDgwMsLCzw2muvScELeLQfOX36NPbt2yd91rp06QIAKCgowPTp09GkSROYmZnB3t4enTp1QnJyconvy7Rp06SepYkTJ0KhUGidd/nbb7+hR48esLa2hqWlJYKDg3Hw4EGdy71v3z6MGjUKjo6OZfb45efnY+rUqWjcuDGUSiVcXV0xadIk5Ofnaw23atUqdOvWDY6OjlAqlfD09JT2RU/asWMHOnfuDCsrK1hbW6Nt27bFPoMASt13lMc333yDpk2bwszMDL6+vkhNTdX5fjy+HRTt/w8cOIB27drBzMwMDRs2xFdffVXqvIr2gRcvXsS2bdukdX7p0iXptfXr1+OTTz5BvXr1ULt2bajVagBlf2cAj/bflpaWuHz5Ml555RVYWlqiXr160j795MmT6NatGywsLODm5qbz/SxNWft9XefYJScno1OnTrCxsYGlpSWaNm2Kjz76SHo/2rZtCwAYOnSo9H483oNdnuUuGs7T0xNmZmbw8vJCYmJisfOOS9vvP3jwAFOmTIGvry9UKhUsLCwQGBiIPXv2aM3n8WnExcWhYcOGqF27Nrp374709HQIITBz5kzUr18f5ubm6NWrF27dulWh91kiaohVq1YJAOI///mPuH79usjKyhKnTp0S7777rjAyMhK7du3SGr5+/fpi1KhRYtmyZWLhwoWiXbt2AoDYunWr1nAARPPmzYWDg4OYPn26iIuLE7/99luJdbi5uYkXXnhB2NjYiOjoaLFw4ULRsmXLYjXs2bNHABB79uwptgxt27YVixYtEtHR0cLc3Fy4u7uL27dvCyGE+OWXX8SLL74oAIi1a9dKj5L8+OOPAoA4efKk1NaqVSthZGQk3njjDantu+++EwDEqVOnhBBCXLx4UQAQq1atKnO+RcO2bt1aNG7cWMybN0/Mnz9f1KlTR9SvX188ePCgxPpKei86d+4sXFxchKurq3j//ffF8uXLRbdu3QQAsX37dmm4bt26CU9Pz2LTnD59ujA2NhYZGRlCCCHy8vKEt7e3sLe3Fx999JFISEgQb731llAoFOL999+XxitaFk9PT9GwYUMxd+5csWjRIvH333+LlStXCgDijTfeECtWrBCff/65GDZsmBg7dqw0fkZGhqhfv75wdXUVM2bMEPHx8eLVV18VAMSiRYtKfR+E+P9toGXLliIwMFAsWbJEREZGCiMjIxEUFCQ0Go00bO/evUW/fv3EggULRHx8vOjbt68AICZMmKA1zYiICOHm5qbVVpHtv1WrVqJu3bpi5syZYvHixaJhw4aidu3a4saNG9JwV65cES4uLqJ27dpi3LhxIiEhQUyePFk0b95c2nbLuw40Go0ICgoSRkZGYtSoUWLp0qWiW7duwtvbW2ubLIm3t7fo06eP9DwxMVEYGRlpbd9CCNGiRQutz0BKSoowNTUVAQEB4rPPPhOLFi0S3t7ewtTUVBw6dKjYOrp48aLUNmnSJAFA9OzZUyxbtky88847on79+qJOnToiIiKi2LitW7cW3bp1E0uXLhXjx48XxsbGol+/flo1169fXzRr1kz6rBXtPz766COhUCjEO++8I7744gvx2WefiYEDB4q5c+eW+J6cOHFCLFq0SAAQAwcOFGvXrhWJiYlCCCFOnTolLCwspHU8d+5c4eHhIZRKpTh48GCx2j09PUXnzp3F0qVLS51nYWGh6N69u7RNrFixQowePVqYmJiIXr16aQ3btm1bMWTIELFo0SKxdOlS0b17dwFALFu2TGu4VatWCYVCIby8vMTs2bNFXFycGD58uBg8eLA0THn3HSUBILy8vESdOnXEjBkzxLx584Sbm5swNzfX2o/q2g7c3NxE06ZNhZOTk/joo4/EsmXLRJs2bYRCodDa9p6UkZEh1q5dK+rUqSN8fHykdZ6bmyvtHz09PYWPj49YuHChiI2NFXl5eeX6zhDi0T7AzMxMeHp6ivfee0/ExcWJDh06SJ8nFxcXMXHiRLF06VLRokULYWxsLP76669S36eK7PenTp0qHo8jp06dEqampsLPz098/vnnIiEhQUyYMEEEBQVJ78eMGTMEADFixAjp/fjzzz+13vuylnvr1q1CoVAIb29vsXDhQjF58mRha2srvLy8tPaJpe33r1+/LurWrSuioqJEfHy8mD9/vmjatKmoVauWVhYomoaPj4/w9PQUCxcuFJ988okwNTUV7du3Fx999JHo0KGDWLJkiRg7dqxQKBRi6NChpb7HJalxwe7Jh1KpFKtXry42/N27d7WeP3jwQHh5eYlu3bpptQMQRkZG4vTp0+Wqw83NTQAQP/zwg9SWk5Mj6tatK1q3bi21PRlmHjx4IBwdHYWXl5e4d++eNNzWrVsFADFlyhSpLTIyUutDUpqsrCwBQCxfvlwIIUR2drYwMjISffv2FU5OTtJwY8eOFXZ2dlJweDLYlTbfomHt7e3FrVu3pPbNmzcLAOKnn34qtcaSgh0A8dVXX0lt+fn5wtnZWetLe8WKFcWCqxBCeHp6aq3LmTNnCgsLC3Hu3Dmt4aKjo4WxsbG4fPmy1rJYW1uLrKwsrWF79eolWrRoUeqyDBs2TNStW1cr9AghxIABA4RKpSq23T2paDv29fXV2jHOnz9fABCbN2+W2nRN69133xW1a9cW9+/fl9p0BbuKbP+mpqbiwoULUtuJEycEALF06VKp7a233hJGRkbi8OHDxWoq2qbKuw42bdokAIj58+dLwzx8+FAEBgaWK9hFRkZqbdtRUVEiKChIODo6ivj4eCGEEDdv3hQKhUJ8/vnnUo1NmjQRoaGhWuH57t27wsPDQ7z44otS25Nf6BkZGcLExET07t1bq45p06YJADqDXUhIiNZ8PvjgA2FsbCyys7OlthYtWojOnTsXW75WrVqJsLCwUt8DXYq27QULFmi19+7dW5iamkpfmkIIcfXqVWFlZSV90T5ee6dOncTDhw/LnN/atWuFkZGR2L9/v1Z7QkKCACB+/vlnqU3XthwaGioaNmwoPc/OzhZWVlbC399fax8phNB6L8u77yhJ0XfHkSNHpLa///5bmJmZiddee01qKynYARCpqalSW1ZWllAqlWL8+PFlztvNza3Yui3aPzZs2FDrfarId0ZERIQAIObMmSO13b59W5ibmwuFQiHWr18vtZ85c0YAEFOnTi211ors958MdkV/ZFy/fr3E6R8+fFjn570iy92yZUtRv359cefOHalt7969AoDOYKdrv//w4UORn5+v1Xb79m3h5OQk3n777WLTcHBw0Pocx8TESH8gFxQUSO0DBw4UpqamWvvq8qpxh2Lj4uKQnJyM5ORkfP311+jatSuGDx9e7FCoubm59P/bt28jJycHgYGBOHbsWLFpdu7cuVznPRRxcXHBa6+9Jj0vOpT222+/ISMjQ+c4R44cQVZWFkaNGqV1vllYWBiaNWuGbdu2lXv+j3NwcECzZs2kwwg///wzjI2NMXHiRGRmZuL8+fMAgP3796NTp07PdEl6//79YWtrKz0PDAwEAPz1119PNT1LS0sMGjRIem5qaop27dppTe/111+HiYkJNmzYILWdOnUKv//+O/r37y+1fffddwgMDIStrS1u3LghPUJCQlBYWFjsMEufPn3g4OCg1WZjY4N//vmnxMPLQgj88MMP6NmzJ4QQWvMJDQ1FTk6Ozu1LlxEjRqBWrVrS85EjR8LExATbt2+X2h7fhu/cuYMbN24gMDAQd+/exZkzZ0qdfkW2/5CQEDRq1Eh67u3tDWtra2k9aDQabNq0CT179tR5lWXRNlXedbB9+3aYmJhg5MiR0jSMjY0xZsyYUpepSGBgIDIzM3H27FkAj7btoKAgBAYGYv/+/QCAAwcOQAghbaPHjx/H+fPn8eabb+LmzZtSbXl5eQgODkZqamqxw8VFUlJS8PDhQ4waNUqrvbR6R4wYofVZCwwMRGFhIf7+++8yl8/GxganT5+WPrvPorCwELt27ULv3r3RsGFDqb1u3bp48803ceDAAemQX5F33nmnXOeWfffdd2jevDmaNWumtb67desGAFqHsh7fHnNycnDjxg107twZf/31F3JycgA8OnR3584dREdHFzsn98n9Vnn2HaUJCAjQunChQYMG6NWrF3bu3FnmqQ6enp7SdgU82gc3bdr0qfeDRSIiIrTep6f5zhg+fLj0fxsbGzRt2hQWFhbo16+f1N60aVPY2NiUu96n2e8Xnee5efPmEj9XJSnvcl+9ehUnT57EW2+9BUtLS2m4zp07o2XLljqnrWu/b2xsLJ1np9FocOvWLTx8+BB+fn4695d9+/aFSqWSnvv7+wMABg0apHVKmL+/Px48eKDz8HFZalywa9euHUJCQhASEoLw8HBs27YNnp6eGD16tNa5Plu3bkX79u1hZmYGOzs7ODg4ID4+XtqJPM7Dw6NCNTRu3LjYjuaFF14AgBLveVS0Q2/atGmx15o1a1auHX5JHv9C279/P/z8/ODn5wc7Ozvs378farUaJ06c0NoZPY0GDRpoPS/6sD95blh51a9fv9j7aGtrqzW9OnXqIDg4GBs3bpTaNmzYABMTE7z++utS2/nz55GUlAQHBwetR0hICIBHJy0/Ttc6//DDD2FpaYl27dqhSZMmiIyM1Dr/6vr168jOzsbKlSuLzWfo0KFa88nIyNB6PHnFWJMmTbSeW1paom7dulrbz+nTp/Haa69BpVLB2toaDg4O0peZru34cRXZ/p9cr4D2erh+/TrUanWZ9w4r7zr4+++/UbduXa2dMaD7s6FL0Xa8f/9+5OXl4bfffkNgYCCCgoK0PgfW1tZo1aqVVBvw6Mvzyfq+/PJL5Ofnl/ieFn02GzdurNVuZ2en9YX3uGf5rMyYMQPZ2dl44YUX0LJlS0ycOBH//e9/yxxPl+vXr+Pu3bs639vmzZtDo9EgPT1dq728+8Pz58/j9OnTxd7Pon3h45+5n3/+GSEhIbCwsICNjQ0cHByk862K3vc///wTAMp1j7ry7DtK8+TnD3i0D797967WuZC6lPV5eVpPvu8V/c4wMzMrFlpUKpXO90qlUpW73qfZlvv374+OHTti+PDhcHJywoABA7Bx48ZyhbzyLndJn8uS2oCSt+01a9bA29tbOqfVwcEB27ZtK9f+sijkubq66mx/mu2iel2GUgWMjIzQtWtXfP755zh//jxatGiB/fv349VXX0VQUBCWL1+OunXrolatWli1apXOk0Yf/yupOurUqRO++OIL/PXXX9i/fz8CAwOhUCjQqVMn7N+/Hy4uLtBoNM8c7Er6K148dsJ6VUxvwIABGDp0KI4fPw4fHx9s3LgRwcHBqFOnjjSMRqPBiy++iEmTJumcZtGXTRFd67x58+Y4e/Ystm7diqSkJPzwww9Yvnw5pkyZgunTp0s7pUGDBiEiIkLnfLy9vQE86hF53KpVq3SeZF+S7OxsdO7cGdbW1pgxYwYaNWoEMzMzHDt2DB9++GGpO8iKbv+VtV4rug6elouLCzw8PJCamgp3d3cIIRAQEAAHBwe8//77+Pvvv7F//3506NBBuqqw6P1asGABfHx8dE73yaD5LJ7lPQ0KCsKff/6JzZs3Y9euXfjyyy+xaNEiJCQkaPXIVJXy7g81Gg1atmyJhQsX6ny96Ivuzz//RHBwMJo1a4aFCxfC1dUVpqam2L59OxYtWlThHh2g8vdFhjDvZ/0eKqmuZ633acY3NzdHamoq9uzZg23btiEpKQkbNmxAt27dsGvXLr1dda7rPf76668xZMgQ9O7dGxMnToSjoyOMjY0RGxsr/bHxuKp6nx9X44Md8OiyZeDRXb0B4IcffoCZmRl27twJpVIpDbdq1apKmd+FCxcghND6K+jcuXMAUOIvABRdsXb27FnpUEWRs2fPat0rqaKHS4sCW3JyMg4fPozo6GgAj74g4uPj4eLiAgsLizLvmWSodw7v3bs33n33Xelw7Llz5xATE6M1TKNGjZCbmyv1Dj0tCwsL9O/fH/3798eDBw/w+uuvY/bs2YiJiYGDgwOsrKxQWFhY5nyevIKxRYsWWs/Pnz+Prl27Ss9zc3Nx7do1vPzyywAeXTV28+ZN/Pjjj1r3/7t48WKZy1DZ27+DgwOsra1x6tSpUocr7zpwc3NDSkoKcnNztcJU0aHV8ggMDERqaio8PDzg4+MDKysrtGrVCiqVCklJSTh27JjW1dRFh5qtra0rvI0UfTYvXLig9Rf/zZs3n6mXprTPm52dHYYOHYqhQ4ciNzcXQUFBmDZtWoWDnYODA2rXrq3zvT1z5gyMjIyK9TSUV6NGjXDixAkEBweXuiw//fQT8vPzsWXLFq3ejievOixaR6dOnSqxx6Wy6DrMfe7cOdSuXbtYr5e+VOQ7wxAZGRkhODgYwcHBWLhwIebMmYOPP/4Ye/bsQUhISInbTHmX+/HP5ZN0tZXk+++/R8OGDfHjjz9q1TR16tRyT6Oy1bhDsU8qKCjArl27YGpqiubNmwN4lJwVCoXWuRKXLl2qtJ/XuXr1KhITE6XnarUaX331FXx8fIrdXqGIn58fHB0dkZCQoHUrgB07duCPP/5AWFiY1FZ036jy3P4BeNS9XK9ePSxatAgFBQXo2LEjgEdffn/++Se+//57tG/fvsz7DFV0vs+LjY0NQkNDsXHjRqxfvx6mpqbo3bu31jD9+vVDWloadu7cWWz87OxsKfyX5ubNm1rPTU1N4enpCSEECgoKYGxsjD59+uCHH37QGXIeP4RTdLpA0ePJHryVK1eioKBAeh4fH4+HDx+iR48eAP7/r7/H/9p78OABli9fXuZyVPb2b2RkhN69e+Onn37S+VNZRTWWdx28/PLLePjwodbtLgoLC7F06dJy1xQYGIhLly5hw4YN0h82RkZG6NChAxYuXIiCggKtHmpfX180atQIn376qfQH4ONKO/wWHBwMExOTYrfnePxWO0/DwsJC52ftye3Q0tISjRs3LnYLkfIwNjZG9+7dsXnzZq3D/JmZmVi3bh06deoEa2vrCk8XeLS+r1y5gi+++KLYa/fu3UNeXp5UA6C9Lefk5BT7Q6N79+6wsrJCbGws7t+/r/VaZffEpaWlaZ0/lZ6ejs2bN6N79+4Gcw/DinxnGBpdt/ko6ikvWpaSvm/Ku9wuLi7w8vLCV199pfWZ3rdvH06ePFnuWnVtn4cOHUJaWlq5p1HZalyP3Y4dO6QTx7OysrBu3TqcP38e0dHR0g4qLCwMCxcuxEsvvYQ333wTWVlZiIuLQ+PGjZ/6XJXHvfDCCxg2bBgOHz4MJycn/Pvf/0ZmZmapPSK1atXCvHnzMHToUHTu3BkDBw5EZmYmPv/8c7i7u+ODDz6Qhi3qWRs7dixCQ0NhbGyMAQMGlFpTYGAg1q9fj5YtW0rnQLRp0wYWFhY4d+4c3nzzzTKX62nm+7z0798fgwYNwvLlyxEaGlrsJqwTJ07Eli1b8Morr2DIkCHw9fVFXl4eTp48ie+//x6XLl3SOnSrS/fu3eHs7IyOHTvCyckJf/zxB5YtW4awsDBYWVkBAObOnYs9e/bA398f77zzDjw9PXHr1i0cO3YM//nPf8p936IHDx4gODgY/fr1w9mzZ7F8+XJ06tQJr776KgCgQ4cOsLW1RUREBMaOHQuFQoG1a9eW6wuuKrb/OXPmYNeuXejcuTNGjBiB5s2b49q1a/juu+9w4MAB2NjYlHsd9OzZEx07dkR0dDQuXbok3QeyrPMGH1cU2s6ePYs5c+ZI7UFBQdixY4d0v60iRkZG+PLLL9GjRw+0aNECQ4cORb169XDlyhXs2bMH1tbW+Omnn3TOy8nJCe+//z4+++wzvPrqq3jppZdw4sQJ7NixA3Xq1Hnqnm5fX1/Ex8dj1qxZaNy4MRwdHdGtWzd4enqiS5cu8PX1hZ2dHY4cOYLvv/++XL9vqsusWbOke4qNGjUKJiYmWLFiBfLz8yt877fHDR48GBs3bsR7772HPXv2oGPHjigsLMSZM2ewceNG7Ny5E35+fujevTtMTU3Rs2dPvPvuu8jNzcUXX3wBR0dHXLt2TZqetbU1Fi1ahOHDh6Nt27Z48803YWtrixMnTuDu3btYs2bNU9f6JC8vL4SGhmLs2LFQKpXSH0xV/Ys/FVGR7wxDM2PGDKSmpiIsLAxubm7IysrC8uXLUb9+fXTq1AnAox5aGxsbJCQkwMrKChYWFvD394eHh0e5l3vOnDno1asXOnbsiKFDh+L27dtYtmwZvLy8dP4Bp8srr7yCH3/8Ea+99hrCwsJw8eJFJCQkwNPTs9zTqHQVvo62mtJ1uxMzMzPh4+Mj4uPjtS6HF0KIf/3rX6JJkyZCqVSKZs2aiVWrVhW7JFuIR5e+R0ZGlruOosvVd+7cKby9vaXpf/fdd1rD6brFhxBCbNiwQbRu3VoolUphZ2cnwsPDxT///KM1zMOHD8WYMWOEg4ODUCgU5br1SVxcnAAgRo4cqdUeEhIiAIiUlBStdl23OylpviXdRkEIUa7L5ku63YmuW4vounWHEEKo1Wphbm4uAIivv/5a53zu3LkjYmJiROPGjYWpqamoU6eO6NChg/j000+lW4uUtiwrVqwQQUFBwt7eXiiVStGoUSMxceJEkZOTozVcZmamiIyMFK6urqJWrVrC2dlZBAcHi5UrV5b6Pgjx/9vxvn37xIgRI4Stra2wtLQU4eHh4ubNm1rD/vzzz6J9+/bC3NxcuLi4iEmTJomdO3cWey91vWfPuv27ublp3cZDiEe3hHjrrbeEg4ODUCqVomHDhiIyMlLrVgHlWQdCPLodyeDBg4W1tbVQqVRi8ODB4rfffivX7U6KODo6CgAiMzNTajtw4IAAIAIDA3WO89tvv4nXX39dWsdubm6iX79+Wp8PXbe5ePjwoZg8ebJwdnYW5ubmolu3buKPP/4Q9vb24r333is27pO3hdH1GcjIyBBhYWHCyspKAJBufTJr1izRrl07YWNjI8zNzUWzZs3E7Nmzy7xfZGnb9rFjx0RoaKiwtLQUtWvXFl27dhW//PKL1jAl1V6aBw8eiHnz5okWLVoIpVIpbG1tha+vr5g+fbrW52bLli3C29tbmJmZCXd3dzFv3jzx73//u9j7XDRshw4dhLm5ubC2thbt2rUT3377rfR6RfcdTyra5r/++mvpM9K6deti++qSbnei61Y0nTt31nnrmieVdruTJ79DipTnOyMiIkJYWFjorEvXe1XScjyuIvv9J/ctKSkpolevXsLFxUWYmpoKFxcXMXDgwGK3Qtq8ebPw9PQUJiYmxT775VluIYRYv369aNasmVAqlcLLy0ts2bJF9OnTRzRr1qxcy6LRaMScOXOEm5ubtC1s3bq12PZU0jRKWn9P83kqohDiOZwtShJ3d3d4eXnJ6rdpiajisrOzYWtri1mzZuHjjz/WdzlE9D8+Pj5wcHAo9ddaDFmNP8eOiKiq6fqB88WLFwOA9FNgRPR8FRQUFDt/eu/evThx4kS1/lzWuHPsiIietw0bNmD16tV4+eWXYWlpiQMHDuDbb79F9+7dpYuViOj5unLlCkJCQjBo0CC4uLjgzJkzSEhIgLOzM9577z19l/fUGOyIiKqYt7c3TExMMH/+fKjVaumCilmzZum7NKIay9bWFr6+vvjyyy9x/fp1WFhYICwsDHPnzoW9vb2+y3tqPMeOiIiISCZ4jh0RERGRTDDYEREREcmEbM+x02g0uHr1KqysrAz2p66IiIiIyiKEwJ07d+Di4iL9hnVJZBvsrl69+tS/YUhERERkaNLT01G/fv1Sh5FtsCv6Caf09PSn/i1DIiIiIn1Tq9VwdXWVsk1p9BLsUlNTsWDBAhw9ehTXrl1DYmKi1o+yCyEwdepUfPHFF8jOzkbHjh0RHx+PJk2alHseRYdfra2tGeyIiIio2ivPqWV6uXgiLy8PrVq1QlxcnM7X58+fjyVLliAhIQGHDh2ChYUFQkNDcf/+/edcKREREVH1oZceux49eqBHjx46XxNCYPHixfjkk0/Qq1cvAMBXX30FJycnbNq0CQMGDHiepRIRERFVGwZ3u5OLFy8iIyMDISEhUptKpYK/vz/S0tJKHC8/Px9qtVrrQURERFSTGFywy8jIAAA4OTlptTs5OUmv6RIbGwuVSiU9eEUsERER1TQGF+yeVkxMDHJycqRHenq6vksiIiIieq4M7nYnzs7OAIDMzEzUrVtXas/MzISPj0+J4ymVSiiVyqouTyf36G3F2i7NDdNDJURERFSTGVyPnYeHB5ydnZGSkiK1qdVqHDp0CAEBAXqsjIiIiMiw6aXHLjc3FxcuXJCeX7x4EcePH4ednR0aNGiAcePGYdasWWjSpAk8PDwwefJkuLi4aN3rjoiIiIi06SXYHTlyBF27dpWeR0VFAQAiIiKwevVqTJo0CXl5eRgxYgSys7PRqVMnJCUlwczMTB/lEhEREVULCiGE0HcRVUGtVkOlUiEnJ6fKf3mC59gRERFRValIpjG4c+yIiIiI6Okw2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJhEEGu8LCQkyePBkeHh4wNzdHo0aNMHPmTAgh9F0aERERkcEy0XcBusybNw/x8fFYs2YNWrRogSNHjmDo0KFQqVQYO3asvssjIiIiMkgGGex++eUX9OrVC2FhYQAAd3d3fPvtt/j111/1XBkRERGR4TLIQ7EdOnRASkoKzp07BwA4ceIEDhw4gB49epQ4Tn5+PtRqtdaDiIiIqCYxyB676OhoqNVqNGvWDMbGxigsLMTs2bMRHh5e4jixsbGYPn36c6ySiIiIyLAYZI/dxo0b8c0332DdunU4duwY1qxZg08//RRr1qwpcZyYmBjk5ORIj/T09OdYMREREZH+GWSP3cSJExEdHY0BAwYAAFq2bIm///4bsbGxiIiI0DmOUqmEUql8nmUSERERGRSD7LG7e/cujIy0SzM2NoZGo9FTRURERESGzyB77Hr27InZs2ejQYMGaNGiBX777TcsXLgQb7/9tr5LIyIiIjJYBhnsli5dismTJ2PUqFHIysqCi4sL3n33XUyZMkXfpREREREZLIMMdlZWVli8eDEWL16s71KIiIiIqg2DPMeOiIiIiCqOwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGTCRN8FyJV79LZibZfmhumhEiIiIqop2GNHREREJBMMdkREREQywWBHREREJBMMdkREREQywWBHREREJBMMdkREREQyYbDB7sqVKxg0aBDs7e1hbm6Oli1b4siRI/oui4iIiMhgGeR97G7fvo2OHTuia9eu2LFjBxwcHHD+/HnY2trquzQiIiIig2WQwW7evHlwdXXFqlWrpDYPDw89VkRERERk+AzyUOyWLVvg5+eHvn37wtHREa1bt8YXX3xR6jj5+flQq9VaDyIiIqKaxCCD3V9//YX4+Hg0adIEO3fuxMiRIzF27FisWbOmxHFiY2OhUqmkh6ur63OsmIiIiEj/FEIIoe8inmRqago/Pz/88ssvUtvYsWNx+PBhpKWl6RwnPz8f+fn50nO1Wg1XV1fk5OTA2tq6SuvV9buwuvC3YomIiKii1Go1VCpVuTKNQfbY1a1bF56enlptzZs3x+XLl0scR6lUwtraWutBREREVJMYZLDr2LEjzp49q9V27tw5uLm56akiIiIiIsNnkMHugw8+wMGDBzFnzhxcuHAB69atw8qVKxEZGanv0oiIiIgMlkEGu7Zt2yIxMRHffvstvLy8MHPmTCxevBjh4eH6Lo2IiIjIYBnkfewA4JVXXsErr7yi7zKIiIiIqg2D7LEjIiIioopjsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCRN9F1CTuEdvK9Z2aW6YHiohIiIiOWKPHREREZFMMNgRERERyQSDHREREZFMMNgRERERyQSDHREREZFMMNgRERERyQSDHREREZFMMNgRERERyQSDHREREZFMMNgRERERyUS1CHZz586FQqHAuHHj9F0KERERkcEy+GB3+PBhrFixAt7e3vouhYiIiMigGXSwy83NRXh4OL744gvY2tqWOmx+fj7UarXWg4iIiKgmMehgFxkZibCwMISEhJQ5bGxsLFQqlfRwdXV9DhUSERERGQ6DDXbr16/HsWPHEBsbW67hY2JikJOTIz3S09OruEIiIiIiw2Ki7wJ0SU9Px/vvv4/k5GSYmZmVaxylUgmlUlnFlREREREZLoMMdkePHkVWVhbatGkjtRUWFiI1NRXLli1Dfn4+jI2N9VghERERkeExyGAXHByMkydParUNHToUzZo1w4cffshQR0RERKSDQQY7KysreHl5abVZWFjA3t6+WDsRERERPWKQwa4mcY/eVqzt0twwPVRCRERE1V21CXZ79+7VdwlEREREBs1gb3dCRERERBXDYEdEREQkEwx2RERERDLBYEdEREQkEwx2RERERDLBYEdEREQkEwx2RERERDLBYEdEREQkEwx2RERERDLBYEdEREQkEwx2RERERDLBYEdEREQkEwx2RERERDLBYEdEREQkEyb6LoCIiIjoeXGP3las7dLcMD1UUjUY7IiIiEh2dAW4moDBzgDJ/a8JIiKiylRTQ5wuDHZERERUbTDElY4XTxARERHJBHvsiIiIyCCxd67i2GNHREREJBMMdkREREQyYZCHYmNjY/Hjjz/izJkzMDc3R4cOHTBv3jw0bdpU36URERFRFeBh18phkD12+/btQ2RkJA4ePIjk5GQUFBSge/fuyMvL03dpRERERAbLIHvskpKStJ6vXr0ajo6OOHr0KIKCgnSOk5+fj/z8fOm5Wq2u0hqJiIio4tgzV7UMssfuSTk5OQAAOzu7EoeJjY2FSqWSHq6urs+rPCIiIiKDYJA9do/TaDQYN24cOnbsCC8vrxKHi4mJQVRUlPRcrVYz3BEREZVTRXrSyvtrSOyde/4MPthFRkbi1KlTOHDgQKnDKZVKKJXK51QVERFR9VAV4YqBzXAZdLAbPXo0tm7ditTUVNSvX1/f5RARERk0Bi4yyGAnhMCYMWOQmJiIvXv3wsPDQ98l6V1JH9bydocTEZG8MMSRLgYZ7CIjI7Fu3Tps3rwZVlZWyMjIAACoVCqYm5vruToiIqKqwbBGz8ogg118fDwAoEuXLlrtq1atwpAhQ55/QURERM+AgY2eF4MMdkIIfZdARERUJgY2MjQGGeyIiIgMCQMcAbq3A0M7171a3KCYiIiIiMrGHjsiIqLHsHeOgOq7HTDYERFRjVVdv7ypcslpO2Cwq+aqw/F+IiIiej4Y7IiI6Jl6LAzxj0k59cAQVQSDHRFRDVPZoacqQpSusMiwRlQ2BjsiIpmqzkGoOtdOpE8MdjLE8+6Iah4GISICGOyIiKodhjgiKgmDHRGRgWBgI6JnxWBXQ/DwLFHZGKyIqLpjsKvBGPaopmBgI6KagsGOtDzrFyCDYc3D0EREZDgY7KhSlfdLvioC4PMKGDUhvDKsERFVTwx2pBfVOThU59qJiEjejPRdABERERFVDgY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIpkw6GAXFxcHd3d3mJmZwd/fH7/++qu+SyIiIiIyWAYb7DZs2ICoqChMnToVx44dQ6tWrRAaGoqsrCx9l0ZERERkkAz2BsULFy7EO++8g6FDhwIAEhISsG3bNvz73/9GdHR0seHz8/ORn58vPc/JyQEAqNXqKq9Vk3+3yudBREREhud55IyieQghyhzWIIPdgwcPcPToUcTExEhtRkZGCAkJQVpams5xYmNjMX369GLtrq6uVVYnERER1Wyqxc9vXnfu3IFKpSp1GIMMdjdu3EBhYSGcnJy02p2cnHDmzBmd48TExCAqKkp6rtFocOvWLdjb20OhUJQ5T7VaDVdXV6Snp8Pa2vrZFoCqDNdT9cD1VD1wPVUPXE/VQ1WuJyEE7ty5AxcXlzKHNchg9zSUSiWUSqVWm42NTYWnY21tzQ9ONcD1VD1wPVUPXE/VA9dT9VBV66msnroiBnnxRJ06dWBsbIzMzEyt9szMTDg7O+upKiIiIiLDZpDBztTUFL6+vkhJSZHaNBoNUlJSEBAQoMfKiIiIiAyXwR6KjYqKQkREBPz8/NCuXTssXrwYeXl50lWylU2pVGLq1KnFDueSYeF6qh64nqoHrqfqgeupejCU9aQQ5bl2Vk+WLVuGBQsWICMjAz4+PliyZAn8/f31XRYRERGRQTLoYEdERERE5WeQ59gRERERUcUx2BERERHJBIMdERERkUww2BERERHJBIPd/8TFxcHd3R1mZmbw9/fHr7/+qu+S6DGpqano2bMnXFxcoFAosGnTJn2XRDrExsaibdu2sLKygqOjI3r37o2zZ8/quyx6Qnx8PLy9vaU75AcEBGDHjh36LotKMXfuXCgUCowbN07fpdATpk2bBoVCofVo1qyZ3uphsAOwYcMGREVFYerUqTh27BhatWqF0NBQZGVl6bs0+p+8vDy0atUKcXFx+i6FSrFv3z5ERkbi4MGDSE5ORkFBAbp37468vDx9l0aPqV+/PubOnYujR4/iyJEj6NatG3r16oXTp0/ruzTS4fDhw1ixYgW8vb31XQqVoEWLFrh27Zr0OHDggN5q4e1OAPj7+6Nt27ZYtmwZgEe/cuHq6ooxY8YgOjpaz9XRkxQKBRITE9G7d299l0JluH79OhwdHbFv3z4EBQXpuxwqhZ2dHRYsWIBhw4bpuxR6TG5uLtq0aYPly5dj1qxZ8PHxweLFi/VdFj1m2rRp2LRpE44fP67vUgCwxw4PHjzA0aNHERISIrUZGRkhJCQEaWlpeqyMqPrLyckB8Cg0kGEqLCzE+vXrkZeXx59sNECRkZEICwvT+o4iw3P+/Hm4uLigYcOGCA8Px+XLl/VWi8H+pNjzcuPGDRQWFsLJyUmr3cnJCWfOnNFTVUTVn0ajwbhx49CxY0d4eXnpuxx6wsmTJxEQEID79+/D0tISiYmJ8PT01HdZ9Jj169fj2LFjOHz4sL5LoVL4+/tj9erVaNq0Ka5du4bp06cjMDAQp06dgpWV1XOvp8YHOyKqGpGRkTh16pRezzWhkjVt2hTHjx9HTk4Ovv/+e0RERGDfvn0MdwYiPT0d77//PpKTk2FmZqbvcqgUPXr0kP7v7e0Nf39/uLm5YePGjXo5taHGB7s6derA2NgYmZmZWu2ZmZlwdnbWU1VE1dvo0aOxdetWpKamon79+vouh3QwNTVF48aNAQC+vr44fPgwPv/8c6xYsULPlREAHD16FFlZWWjTpo3UVlhYiNTUVCxbtgz5+fkwNjbWY4VUEhsbG7zwwgu4cOGCXuZf48+xMzU1ha+vL1JSUqQ2jUaDlJQUnm9CVEFCCIwePRqJiYnYvXs3PDw89F0SlZNGo0F+fr6+y6D/CQ4OxsmTJ3H8+HHp4efnh/DwcBw/fpyhzoDl5ubizz//RN26dfUy/xrfYwcAUVFRiIiIgJ+fH9q1a4fFixcjLy8PQ4cO1Xdp9D+5ublaf/1cvHgRx48fh52dHRo0aKDHyuhxkZGRWLduHTZv3gwrKytkZGQAAFQqFczNzfVcHRWJiYlBjx490KBBA9y5cwfr1q3D3r17sXPnTn2XRv9jZWVV7NxUCwsL2Nvb85xVAzNhwgT07NkTbm5uuHr1KqZOnQpjY2MMHDhQL/Uw2AHo378/rl+/jilTpiAjIwM+Pj5ISkoqdkEF6c+RI0fQtWtX6XlUVBQAICIiAqtXr9ZTVfSk+Ph4AECXLl202letWoUhQ4Y8/4JIp6ysLLz11lu4du0aVCoVvL29sXPnTrz44ov6Lo2o2vnnn38wcOBA3Lx5Ew4ODujUqRMOHjwIBwcHvdTD+9gRERERyUSNP8eOiIiISC4Y7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCb+D0xsoViBF0NVAAAAAElFTkSuQmCC",
-      "text/plain": [
-       "<Figure size 640x480 with 2 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "lds = True\n",
-    "kernel = \"gaussian\"\n",
-    "ks = 5\n",
-    "sigma = 2\n",
-    "reweight = \"sqrt\"\n",
-    "Y = df[\"MedHouseVal\"].values\n",
-    "lds_y_max = None\n",
-    "lds_y_min = None\n",
-    "granularity = 100\n",
-    "\n",
-    "fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1)\n",
-    "\n",
-    "y_max = max(Y) if lds_y_max is None else lds_y_max\n",
-    "y_min = min(Y) if lds_y_min is None else lds_y_min\n",
-    "bin_edges = np.linspace(y_min, y_max, num=granularity, endpoint=True)\n",
-    "value_dict = dict(zip(bin_edges[:-1], np.histogram(Y, bin_edges)[0]))\n",
-    "\n",
-    "ax1.set_title(\"Histogram of values in the Y\")\n",
-    "ax1.bar(\n",
-    "    value_dict.keys(),\n",
-    "    value_dict.values(),\n",
-    "    width=list(value_dict.keys())[1] - list(value_dict.keys())[0],\n",
-    ")\n",
-    "\n",
-    "if reweight:\n",
-    "    value_dict = dict(zip(value_dict.keys(), np.sqrt(list(value_dict.values()))))\n",
-    "\n",
-    "if kernel is not None:\n",
-    "    lds_kernel_window = get_kernel_window(kernel, ks, sigma)\n",
-    "    smoothed_values = convolve1d(\n",
-    "        list(value_dict.values()), weights=lds_kernel_window, mode=\"constant\"\n",
-    "    )\n",
-    "    weigths = sum(smoothed_values) / (len(smoothed_values) * smoothed_values)\n",
-    "else:\n",
-    "    values = list(value_dict.values())\n",
-    "    weigths = sum(values) / (len(values) * values)  # type: ignore[operator]\n",
-    "value_dict = dict(zip(value_dict.keys(), weigths))\n",
-    "\n",
-    "left_bin_edges = find_bin(bin_edges, Y)\n",
-    "weights = np.array([value_dict[edge] for edge in left_bin_edges], dtype=\"float32\")\n",
-    "\n",
-    "\n",
-    "ax2.set_title(\"Bar plot with inverse-balanced weights for each bin from histogram\")\n",
-    "ax2.bar(\n",
-    "    value_dict.keys(),\n",
-    "    value_dict.values(),\n",
-    "    width=list(value_dict.keys())[1] - list(value_dict.keys())[0],\n",
-    ")\n",
-    "fig.tight_layout()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b472dec0",
-   "metadata": {},
-   "source": [
-    "# Feature Distribution Smoothing\n",
-    "We use dataset feature values in this example, but during the training process features tensors are the ouput of last layer before FDS layer.\n",
-    "* labels are np.vstack-ed to reflect normal training scenario"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "36fe33d0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "features = torch.tensor(df.drop(columns=[\"MedHouseVal\"]).values)\n",
-    "labels = torch.tensor(np.vstack(df[\"MedHouseVal\"].values))\n",
-    "FDS = fds_layer.FDSLayer(feature_dim=features.size(1))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "ec0310c7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "for epoch in range(3):\n",
-    "    FDS.update_last_epoch_stats(epoch)\n",
-    "    FDS.update_running_stats(torch.clone(features).detach(), labels, epoch)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "802278aa",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkEAAAGzCAYAAAAotsMiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAACVZklEQVR4nO2dd3wUdfrHP1uym94bCQESUOkSQJoNBUVFTywoiifYEIXjRL1TzzvbHaJ4enY9z1P4iZ6iomc5C1KsCAgEpHcSAiEJIb1sm98fu9/Z2b6bbEvyeb9eeUFmZ2dmJ7s7n3mez/M8KkmSJBBCCCGEdDPUkT4AQgghhJBIQBFECCGEkG4JRRAhhBBCuiUUQYQQQgjpllAEEUIIIaRbQhFECCGEkG4JRRAhhBBCuiUUQYQQQgjpllAEEUIIIaRbQhFEIoJKpcIjjzwS6cPoEvTp0weXXnqpz/XWrFkDlUqFNWvWhP6gQsz48eMxfvz4SB+GRzZs2IBx48YhISEBKpUKJSUlkT6kqOKRRx6BSqUK2/7C8X2zd+9eXHjhhUhJSYFKpcLHH38c0v2R4EAR1IVYvHgxVCqV/KPVapGfn4+ZM2eivLw80odHSLfAaDRi6tSpqKmpwT/+8Q+89dZb6N27d8j2J8TtoUOH5GUzZ85EYmJiyPYZCh5//PGoEw6HDh3y+8ZhxowZ+PXXX7FgwQK89dZbGDlyZNCPp7m5GY888kiXuJGJFrSRPgASfB577DEUFhaitbUVP//8MxYvXowffvgB27ZtQ2xsbKQPDwDQ0tICrZZvv3ByzjnnoKWlBTqdLtKH0qXZv38/Dh8+jH/961+49dZbI304Ucmf//xn3H///Q7LHn/8cVx99dWYMmVKZA6qA7S0tGDt2rV48MEHMXfu3JDtp7m5GY8++igARHUktDPBq1AX5OKLL5bvQm699VZkZmbiySefxCeffIJrrrkmwkdnJVrEWHdCrVbzvIeByspKAEBqamrQttnU1ISEhISgbS/SaLXaLnUTVFVVBSC4f/NwYjKZYLFYuuUNEtNh3YCzzz4bgPUOVeDJUzFz5kz06dNH/l2Eg//+97/jtddeQ9++faHX63HGGWdgw4YNLs9NTExEeXk5pkyZgsTERGRlZeHee++F2Wx2WNc5Ry88Avv27cPMmTORmpqKlJQU3HTTTWhubnZ4bktLC+bNm4fMzEwkJSXhN7/5DcrLy/3K+4vUwbJly/Doo48iPz8fSUlJuPrqq1FXV4e2tjbcddddyM7ORmJiIm666Sa0tbW5bGfp0qUYMWIE4uLikJ6ejmnTpqGsrMxhne+//x5Tp05Fr169oNfrUVBQgPnz56OlpaXd580bX3/9NYYNG4bY2FgMHDgQy5cvd/valaH08ePHY/DgwdixYwfOO+88xMfHIz8/H4sWLXJ4rsFgwEMPPYQRI0YgJSUFCQkJOPvss7F69Wqfx3XppZeiqKjI7WNjx451SBu8+eabOP/885GdnQ29Xo+BAwfilVde8bkPkQpWpoQ8vWYAWLduHS666CKkpKQgPj4e5557Ln788UeHdRoaGnDXXXehT58+0Ov1yM7OxgUXXIBNmzZ5PI6ZM2fi3HPPBQBMnToVKpXK4XO2atUqnH322UhISEBqaiouv/xy7Ny502Eb4rOwY8cOXH/99UhLS8NZZ53l8xy0h/fff19+H2dmZuKGG25wSZ0H8v48ceIEfvvb3yI5ORmpqamYMWMGtmzZApVKhcWLF7u8RoFKpUJTUxOWLFkip/Nnzpwp71/5neRpGwDQ1taG+fPnIysrS/5uOHLkiNvXXl5ejptvvhk5OTnQ6/UYNGgQ3njjjQDOnv04RLrzD3/4A1QqlcPx+rMffz5fhw4dQlZWFgDg0Ucflc+T+M5rz3f6s88+K3+n79ixAwCwa9cuXH311UhPT0dsbCxGjhyJTz75JODz0lnoOlKceERcGNLS0tq9jXfeeQcNDQ24/fbboVKpsGjRIlx55ZU4cOAAYmJi5PXMZjMmTZqE0aNH4+9//zu++eYbPP300+jbty/uuOMOn/u55pprUFhYiIULF2LTpk14/fXXkZ2djSeffFJeZ+bMmVi2bBl++9vfYsyYMfj2228xefLkgF7PwoULERcXh/vvvx/79u3DCy+8gJiYGKjVapw8eRKPPPKInEosLCzEQw89JD93wYIF+Mtf/oJrrrkGt956K6qqqvDCCy/gnHPOwebNm+W7wffffx/Nzc244447kJGRgfXr1+OFF17AkSNH8P777zscT0fP2969e3Httddi9uzZmDFjBt58801MnToVX375JS644AKvzz158iQuuugiXHnllbjmmmvwwQcf4L777sOQIUNw8cUXAwDq6+vx+uuv47rrrsNtt92GhoYG/Pvf/8akSZOwfv16DBs2zOP2r732Wtx4443YsGEDzjjjDHn54cOH8fPPP+Opp56Sl73yyisYNGgQfvOb30Cr1eLTTz/FnXfeCYvFgjlz5vg8D/6watUqXHzxxRgxYgQefvhhqNVqWXx9//33GDVqFABg9uzZ+OCDDzB37lwMHDgQJ06cwA8//ICdO3di+PDhbrd9++23Iz8/H48//jjmzZuHM844Azk5OQCAb775BhdffDGKiorwyCOPoKWlBS+88ALOPPNMbNq0yeVCP3XqVJxyyil4/PHHIUlSUF67ksWLF+Omm27CGWecgYULF+L48eN47rnn8OOPPzq8jwH/3p8WiwWXXXYZ1q9fjzvuuAP9+/fHf//7X8yYMcPnsbz11lu49dZbMWrUKMyaNQsA0Ldv34Bf06233oqlS5fi+uuvx7hx47Bq1Sq33w3Hjx/HmDFjoFKpMHfuXGRlZeGLL77ALbfcgvr6etx1111+7/PKK69Eamoq5s+fj+uuuw6XXHKJ7Mfydz/+fL6ysrLwyiuv4I477sAVV1yBK6+8EgAwdOjQgM8TYL3haG1txaxZs6DX65Geno7t27fjzDPPRH5+Pu6//34kJCRg2bJlmDJlCj788ENcccUV7dpXVCORLsObb74pAZC++eYbqaqqSiorK5M++OADKSsrS9Lr9VJZWZm87rnnniude+65LtuYMWOG1Lt3b/n3gwcPSgCkjIwMqaamRl7+3//+VwIgffrppw7PBSA99thjDtssLi6WRowY4bAMgPTwww/Lvz/88MMSAOnmm292WO+KK66QMjIy5N83btwoAZDuuusuh/Vmzpzpsk13rF69WgIgDR48WDIYDPLy6667TlKpVNLFF1/ssP7YsWMdzsehQ4ckjUYjLViwwGG9X3/9VdJqtQ7Lm5ubXfa/cOFCSaVSSYcPH5aXBXLe3NG7d28JgPThhx/Ky+rq6qQePXpIxcXFLq999erV8rJzzz1XAiD93//9n7ysra1Nys3Nla666ip5mclkktra2hz2e/LkSSknJ8flb+ZMXV2dpNfrpXvuucdh+aJFi1zOhbtzNmnSJKmoqMhhmfP7V7z3Dx486LCe82u2WCzSKaecIk2aNEmyWCwO+y0sLJQuuOACeVlKSoo0Z84cr6/NHWKf77//vsPyYcOGSdnZ2dKJEyfkZVu2bJHUarV04403ysvEZ+G6664LeN+CGTNmSAkJCR4fNxgMUnZ2tjR48GCppaVFXv7ZZ59JAKSHHnrIYVv+vD8//PBDCYD07LPPysvMZrN0/vnnSwCkN9980+U1KklISJBmzJjh9rUoP4OetlFSUiIBkO68806H9a6//nqX74ZbbrlF6tGjh1RdXe2w7rRp06SUlBS370NviO/Jp556ymG5v/vx9/NVVVXl8Xsu0O/05ORkqbKy0mHdCRMmSEOGDJFaW1vlZRaLRRo3bpx0yimneD0HnRWmw7ogEydORFZWFgoKCnD11VcjISEBn3zyCXr27NnubV577bUOkSSRYjtw4IDLurNnz3b4/eyzz3a7njvcPffEiROor68HAHz55ZcAgDvvvNNhvd/97nd+bV9w4403OkSwRo8eDUmScPPNNzusN3r0aJSVlcFkMgEAli9fDovFgmuuuQbV1dXyT25uLk455RSH8HVcXJz8/6amJlRXV2PcuHGQJAmbN2/267X7e97y8vIc7tKSk5Nx4403YvPmzaioqPD63MTERNxwww3y7zqdDqNGjXLYt0ajkf0CFosFNTU1MJlMGDlypNf0kDiWiy++GMuWLXOIaLz33nsYM2YMevXqJS9TnrO6ujpUV1fj3HPPxYEDB1BXV+fjLPimpKQEe/fuxfXXX48TJ07If7+mpiZMmDAB3333HSwWCwCrv2PdunU4evRoh/d77NgxlJSUYObMmUhPT5eXDx06FBdccAH+97//uTzH+f0QTH755RdUVlbizjvvdPCJTZ48Gf3798fnn3/u83ic359ffvklYmJicNttt8nL1Gp10CJ4vhDncN68eQ7LnaM6kiThww8/xGWXXQZJkhw+x5MmTUJdXZ3P97Q/BLKfjny+2stVV10lp9cAoKamBqtWrcI111yDhoYG+VhPnDiBSZMmYe/evV2yypjpsC7ISy+9hFNPPRV1dXV444038N1330Gv13dom8oLFWBPrZ08edJheWxsrMMHS6zrvF579pOcnIzDhw9DrVajsLDQYb1+/fr5tX1P+0lJSQEAFBQUuCy3WCyoq6tDRkYG9u7dC0mScMopp7jdrlJYlZaW4qGHHsInn3zi8vqdL+gdPW/9+vVz8UeceuqpAKzp0NzcXI/P7dmzp8tz09LSsHXrVodlS5YswdNPP41du3bBaDTKy53/Fu649tpr8fHHH2Pt2rUYN24c9u/fj40bN+LZZ591WO/HH3/Eww8/jLVr17p4werq6uS/U3vZu3cvAHhN0dTV1SEtLQ2LFi3CjBkzUFBQgBEjRuCSSy7BjTfe6NHf5I3Dhw8DAE477TSXxwYMGICvvvrKxfzsz3ltL96Op3///vjhhx8clvnz/jx8+DB69OiB+Ph4h/UC/Wy2F/Hd4JxGc36NVVVVqK2txWuvvYbXXnvN7baEub0jBLqfjny+2oPzdvft2wdJkvCXv/wFf/nLXzweb35+fkiOJ1JQBHVBRo0aJZtNp0yZgrPOOgvXX389du/eLeeqVSqVW5+BJyOuRqNxu9x5G57W8xd/99NRPO3H1/4tFgtUKhW++OILt+uK82s2m3HBBRegpqYG9913H/r374+EhASUl5dj5syZcrTB137DgT/nfOnSpZg5cyamTJmCP/zhD8jOzoZGo8HChQsdDPeeuOyyyxAfH49ly5Zh3LhxWLZsGdRqNaZOnSqvs3//fkyYMAH9+/fHM888g4KCAuh0Ovzvf//DP/7xD5dzpsRT4z3n97PYxlNPPeXRxyT+htdccw3OPvtsfPTRR/j666/x1FNP4cknn8Ty5ctlr1QoUUbFIk0k35/+/m39RbwHbrjhBo9iuL0+m/bup6OfLyDw73Tn95c43nvvvReTJk1y+5xwCdpwQhHUxREfpPPOOw8vvvii3JsjLS3NbapF3CFGK71794bFYsHBgwcdojH79u0Ly/779u0LSZJQWFgoR1rc8euvv2LPnj1YsmQJbrzxRnn5ihUrQnJc4i5OecHYs2cPALitrAmUDz74AEVFRVi+fLnDPh5++GG/np+QkIBLL70U77//Pp555hm89957OPvss5GXlyev8+mnn6KtrQ2ffPKJQ6TOnwo0ETGsra11WO78fhZRguTkZEycONHndnv06IE777wTd955JyorKzF8+HAsWLAgYBEkqod2797t8tiuXbuQmZkZ1hJ45fGcf/75Do/t3r27Xc0de/fujdWrV6O5udkhGuTvZ9OT2ElLS3P5uwKuf1vx3bB//36H6I/zOReVY2az2a/3QHsJZD/+fr68ddnu6He6iHDGxMSE9LxEG/QEdQPGjx+PUaNG4dlnn0VraysA68Vg165dcn8LANiyZYtLmXC0Ie5QXn75ZYflL7zwQlj2f+WVV0Kj0eDRRx91ueuSJAknTpwAYL9zVq4jSRKee+65kBzX0aNH8dFHH8m/19fX4//+7/8wbNgwr6kwf3H3etatW4e1a9f6vY1rr70WR48exeuvv44tW7bg2muv9bmPuro6vPnmmz63LcTNd999Jy8zm80uaYgRI0agb9+++Pvf/47GxkaX7YjPg9lsdklZZmdnIy8vz23LBF/06NEDw4YNw5IlSxwu6Nu2bcPXX3+NSy65JOBtdoSRI0ciOzsbr776qsPr+eKLL7Bz586Aqy0B62fTaDTiX//6l7zMYrHgpZde8uv5CQkJbsVO3759UVdX55CePXbsmMP7HYAsTJ9//nmH5c4pV41Gg6uuugoffvghtm3b5rI/5XdiRwhkP/5+voS49HSeOvKdnp2djfHjx+Of//wnjh075vV4uxKMBHUT/vCHP2Dq1KlYvHgxZs+ejZtvvhnPPPMMJk2ahFtuuQWVlZV49dVXMWjQINmEHI2MGDECV111FZ599lmcOHFCLpEXUY9QzyPq27cv/va3v+GBBx7AoUOHMGXKFCQlJeHgwYP46KOPMGvWLNx7773o378/+vbti3vvvRfl5eVITk7Ghx9+6LfHJ1BOPfVU3HLLLdiwYQNycnLwxhtv4Pjx434JCH+49NJLsXz5clxxxRWYPHkyDh48iFdffRUDBw50KybccckllyApKQn33nuvfIFQcuGFF0Kn0+Gyyy7D7bffjsbGRvzrX/9Cdna22y9lJYMGDcKYMWPwwAMPoKamBunp6Xj33XdlQ7tArVbj9ddfx8UXX4xBgwbhpptuQn5+PsrLy7F69WokJyfj008/RUNDA3r27Imrr74ap59+OhITE/HNN99gw4YNePrppwM7eTaeeuopXHzxxRg7dixuueUWuUQ+JSUlJHOtjEYj/va3v7ksT09Px5133oknn3wSN910E84991xcd911col8nz59MH/+/ID3N2XKFIwaNQr33HMP9u3bh/79++OTTz5BTU0NAN+fzREjRuCbb77BM888g7y8PBQWFmL06NGYNm0a7rvvPlxxxRWYN28empub8corr+DUU091MA0PGzYM1113HV5++WXU1dVh3LhxWLlypdtI1BNPPIHVq1dj9OjRuO222zBw4EDU1NRg06ZN+Oabb+Rj7ij+7sffz1dcXBwGDhyI9957D6eeeirS09MxePBgDB48OCjf6S+99BLOOussDBkyBLfddhuKiopw/PhxrF27FkeOHMGWLVuCcl6iivAVopFQI8qEN2zY4PKY2WyW+vbtK/Xt21cymUySJEnS0qVLpaKiIkmn00nDhg2TvvrqK4/llM6ln5LkWubuqSzXXTms83PFOlVVVW5fk7L0uampSZozZ46Unp4uJSYmSlOmTJF2794tAZCeeOIJb6fIY/myp3Pn6bg+/PBD6ayzzpISEhKkhIQEqX///tKcOXOk3bt3y+vs2LFDmjhxopSYmChlZmZKt912m7RlyxaXcuFAzps7evfuLU2ePFn66quvpKFDh0p6vV7q37+/y2v0VCI/aNAgl206vw8sFov0+OOPS71795b0er1UXFwsffbZZx7Llz0xffp0CYA0ceJEt49/8skn0tChQ6XY2FipT58+0pNPPim98cYbLu8Bd+XA+/fvlyZOnCjp9XopJydH+tOf/iStWLHC5TVLkiRt3rxZuvLKK6WMjAxJr9dLvXv3lq655hpp5cqVkiRZ2wT84Q9/kE4//XQpKSlJSkhIkE4//XTp5Zdf9vkaPb3HJEmSvvnmG+nMM8+U4uLipOTkZOmyyy6TduzY4bCOp/dcIIiydnc/ffv2ldd77733pOLiYkmv10vp6enS9OnTpSNHjrhsy9/3Z1VVlXT99ddLSUlJUkpKijRz5kzpxx9/lABI7777rtfn7tq1SzrnnHOkuLg4CYBDufzXX38tDR48WNLpdNJpp50mLV261O02WlpapHnz5kkZGRlSQkKCdNlll0llZWVuy8qPHz8uzZkzRyooKJBiYmKk3NxcacKECdJrr73m1zlW4u170p/9BPL5+umnn6QRI0ZIOp3O5XV19DtdkqyfoxtvvFHKzc2VYmJipPz8fOnSSy+VPvjgg4DPS2dAJUkh6MJFSJgpKSlBcXExli5diunTp0f6cAghNj7++GNcccUV+OGHH3DmmWdG+nAIcYCeINLpcB47AVjz/mq1Guecc04EjogQArh+Ns1mM1544QUkJyd77LJNSCShJ4h0OhYtWoSNGzfivPPOg1arxRdffIEvvvgCs2bNcunzQwgJH7/73e/Q0tKCsWPHoq2tDcuXL8dPP/2Exx9/PKpK/gkRMB1GOh0rVqzAo48+ih07dqCxsRG9evXCb3/7Wzz44INdajI1IZ2Nd955B08//TT27duH1tZW9OvXD3fccQfmzp0b6UMjxC0UQYQQQgjpltATRAghhJBuCUUQIYQQQrolNFB4wGKx4OjRo0hKSgp5Az5CCCGEBAdJktDQ0IC8vDyo1d5jPRRBHjh69CgrjQghhJBOSllZGXr27Ol1HYogDyQlJQGwnsTk5OQIHw0hhBBC/KG+vh4FBQXyddwbFEEeECmw5ORkiiBCCCGkk+GPlYXGaEIIIYR0SyiCCCGEENItoQgihBBCSLeEIogQQggh3RKKIEIIIYR0SyiCCCGEENItoQgihBBCSLeEIogQQggh3RKKIEIIIYR0SyiCCCGEENItoQgihBBCSLeEIogQQggh3RKKoCjl7XWHsf5gTaQPgxBCCOmyUARFIfsqG/HgR9tw/4dbI30ohBBCSJeFIigKqWsxAgDqW40RPhJCCCGk60IRFIUYzRYAgMFkifCREEIIIV0XiqAoRIggo1mK8JEQQgghXReKoCjELoIYCSKEEEJCBUVQFGIwWSNAJosEi4XRIEIIISQUUARFIcoIkNHCaBAhhBASCiiCohAHEURfECGEEBISKIKiEKUIYoUYIYQQEhoogqIQgyL6Q3M0IYQQEhoogqIQo4mRIEIIISTUUARFIY6eIIogQgghJBRQBEUhNEYTQgghoYciKAqhJ4gQQggJPRRBUYhDdRhFECGEEBISKIKiEKUx2khjNCGEEBISKIKiEHqCCCGEkNBDERSF0BNECCGEhB6KoCiEniBCCCEk9FAERSHsE0QIIYSEHoqgKISzwwghhJDQQxEUhRhM9AQRQgghoYYiKApx9ASxOowQQggJBRRBUYjJwj5BhBBCSKihCIpCjEyHEUIIISGHIigKMbA6jBBCCAk5IRNBCxYswLhx4xAfH4/U1FS368ybNw8jRoyAXq/HsGHD3K7z1VdfYcyYMUhKSkJWVhauuuoqHDp0yOu+a2pqMH36dCQnJyM1NRW33HILGhsbO/aCwgg9QYQQQkjoCZkIMhgMmDp1Ku644w6v691888249tpr3T528OBBXH755Tj//PNRUlKCr776CtXV1bjyyiu9bnP69OnYvn07VqxYgc8++wzfffcdZs2a1e7XEm7YJ4gQQggJPdpQbfjRRx8FACxevNjjOs8//zwAoKqqClu3bnV5fOPGjTCbzfjb3/4Gtdqq1+69915cfvnlMBqNiImJcXnOzp078eWXX2LDhg0YOXIkAOCFF17AJZdcgr///e/Iy8vr6EsLOcp5YTRGE0IIIaEhqj1BI0aMgFqtxptvvgmz2Yy6ujq89dZbmDhxolsBBABr165FamqqLIAAYOLEiVCr1Vi3bp3HfbW1taG+vt7hJ1IoGyQyEkQIIYSEhqgWQYWFhfj666/xpz/9CXq9HqmpqThy5AiWLVvm8TkVFRXIzs52WKbVapGeno6KigqPz1u4cCFSUlLkn4KCgqC9jkChJ4gQQggJPQGJoPvvvx8qlcrrz65du4J2cBUVFbjtttswY8YMbNiwAd9++y10Oh2uvvpqSFJwxcEDDzyAuro6+aesrCyo2w8Ejs0ghBBCQk9AnqB77rkHM2fO9LpOUVFRR47HgZdeegkpKSlYtGiRvGzp0qUoKCjAunXrMGbMGJfn5ObmorKy0mGZyWRCTU0NcnNzPe5Lr9dDr9cH7dg7goMniOkwQgghJCQEJIKysrKQlZUVqmNxobm5WTZECzQaDQDAYnEvDsaOHYva2lps3LgRI0aMAACsWrUKFosFo0ePDu0BBwn2CSKEEEJCT8g8QaWlpSgpKUFpaSnMZjNKSkpQUlLi0K9n3759KCkpQUVFBVpaWuR1DAYDAGDy5MnYsGEDHnvsMezduxebNm3CTTfdhN69e6O4uBgAsH79evTv3x/l5eUAgAEDBuCiiy7CbbfdhvXr1+PHH3/E3LlzMW3atE5RGSZJEkvkCSGEkDAQshL5hx56CEuWLJF/F6Jl9erVGD9+PADg1ltvxbfffuuyzsGDB9GnTx+cf/75eOedd7Bo0SIsWrQI8fHxGDt2LL788kvExcUBsEaLdu/eDaPRKG/n7bffxty5czFhwgSo1WpcddVVcjl+tGO2SFDanWiMJoQQQkKDSgq2w7iLUF9fj5SUFNTV1SE5OTls+20xmDHgoS/l38cWZeA/s1y9T4QQQghxJZDrd1SXyHdHDE7pL6bDCCGEkNBAERRlOIseiiBCCCEkNFAERRnOooeeIEIIISQ0UARFGUaTo+hhJIgQQggJDRRBUQY9QYQQQkh4oAiKMlzSYRybQQghhIQEiqAog8ZoQgghJDxQBEUZjAQRQggh4YEiKMow2IzRsTHWP42R1WGEEEJISKAIijJEJChBp3X4nRBCCCHBhSIoyhCiJ16vAQCYLBIsFkaDCCGEkGBDERRlOEeCAMBoYTSIEEIICTYUQVGG6BAdr9PIy+gLIoQQQoIPRVCUYbRVg8UrI0GsECOEEEKCDkVQlCHSYbExamjUKodlhBBCCAkeFEFRhhA8MRo1YjRWEeQ8SoMQQgghHYciKMoQniCrCGKvIEIIISRUUARFGcpIkM4mgtg1mhBCCAk+FEFRhjBB67QqRSSIIogQQggJNhRBUYaDJ0hLTxAhhBASKiiCogylJ0ikw1giTwghhAQfiqAow7E6jMZoQgghJFRQBEUZQgTpNCrotPQEEUIIIaGCIijKcBcJoieIEEIICT4UQVGGwWTzBGntzRIZCSKEEEKCD0VQlOHeE0QRRAghhAQbiqAow8ETJFeH0RhNCCGEBBuKoCiDniBCCCEkPFAERRkOs8O0HJtBCCGEhAqKoChDNEakMZoQQggJLRRBUYZbTxBFECGEEBJ0KIKiDPeeIBqjCSGEkGBDERRlOMwOY8doQgghJGRQBEUZbvsE0RhNCCGEBB2KoChD9gRpVdDRGE0IIYSEDIqgKEOuDqMniBBCCAkpFEFRhrs+QYwEEUIIIcGHIijK4OwwQgghJDxQBEUZ9j5BanqCCCGEkBBCERRlyJEgrcruCeIAVUIIISToUARFEZIkwaj0BHGAKiGEEBIyKIKiCKOiCszBGM0+QYQQQkjQoQiKIpTeH3qCCCGEkNBCERRFKMVOjEbFsRmEEEJICKEIiiKE90elAjRqFZslEkIIISGEIiiKUJqiVSoV+wQRQgghIYQiKIoQBmidTfxQBBFCCCGhgyIoirB3i7YaonWcIk8IIYSEDIqgKMKgGJkBWBsmWpfTE0QIIYQEG4qgKELpCVL+y3QYIYQQEnwogqIIeW6YrTReRxFECCGEhAyKoChCeH+EJ8g+O4wiiBBCCAk2IRNBCxYswLhx4xAfH4/U1FS368ybNw8jRoyAXq/HsGHD3K7z1VdfYcyYMUhKSkJWVhauuuoqHDp0yOu++/TpA5VK5fDzxBNPdOwFhQEXT5BNDJksEiwW+oIIIYSQYBIyEWQwGDB16lTccccdXte7+eabce2117p97ODBg7j88stx/vnno6SkBF999RWqq6tx5ZVX+tz/Y489hmPHjsk/v/vd79r1OsKJiydIa//zGC2MBhFCCCHBRBuqDT/66KMAgMWLF3tc5/nnnwcAVFVVYevWrS6Pb9y4EWazGX/729+gVlsFwb333ovLL78cRqMRMTExHredlJSE3NzcDryC8CN7gjSOniDrYxL0IftrEUIIId2PqPYEjRgxAmq1Gm+++SbMZjPq6urw1ltvYeLEiV4FEAA88cQTyMjIQHFxMZ566imYTCav67e1taG+vt7hJ9zIfYK0jp4ggL2CCCGEkGAT1SKosLAQX3/9Nf70pz9Br9cjNTUVR44cwbJly7w+b968eXj33XexevVq3H777Xj88cfxxz/+0etzFi5ciJSUFPmnoKAgmC/FLwwmR0+QRq2CRs1J8oQQQkgoCEgE3X///S6GY+efXbt2Be3gKioqcNttt2HGjBnYsGEDvv32W+h0Olx99dWQJM9G4bvvvhvjx4/H0KFDMXv2bDz99NN44YUX0NbW5vE5DzzwAOrq6uSfsrKyoL0Of3H2BFn/LxomUgQRQgghwSQgl8k999yDmTNnel2nqKioI8fjwEsvvYSUlBQsWrRIXrZ06VIUFBRg3bp1GDNmjF/bGT16NEwmEw4dOoTTTjvN7Tp6vR56vT4ox91enD1BgFUQtRotskAihBBCSHAISARlZWUhKysrVMfiQnNzs2yIFmg0GgCAJYBqqZKSEqjVamRnZwf1+IKN8+wwgA0TCSGEkFARMk9QaWkpSkpKUFpaCrPZjJKSEpSUlKCxsVFeZ9++fSgpKUFFRQVaWlrkdQwGAwBg8uTJ2LBhAx577DHs3bsXmzZtwk033YTevXujuLgYALB+/Xr0798f5eXlAIC1a9fi2WefxZYtW3DgwAG8/fbbmD9/Pm644QakpaWF6uUGBec+Qcr/s2EiIYQQElxCVnT90EMPYcmSJfLvQrSsXr0a48ePBwDceuut+Pbbb13WOXjwIPr06YPzzz8f77zzDhYtWoRFixYhPj4eY8eOxZdffom4uDgA1mjR7t27YTQaAVjTWu+++y4eeeQRtLW1obCwEPPnz8fdd98dqpcaNIwmmydI0R9IVIoxEkQIIYQEl5CJoMWLF3vtEQQAa9as8bmdadOmYdq0aR4fHz9+vINJevjw4fj555/9PcyowpMnyPoYPUGEEEJIMInqEvnuhjdPENNhhBBCSHChCIoivHmCmA4jhBBCggtFUBRhdCuC2CeIEEIICQUUQVGEyeb70SmM0eL/jAQRQgghwYUiKIowuPEEMR1GCCGEhAaKoCjC3dgMuVmiidVhhBBCSDChCIoijCYvzRIZCSKEEEKCCkVQFOG2TxA9QYQQQkhIoAiKImRPkFbpCWLHaEIIISQUUARFEe5K5HXsGE0IIYSEBIqgKMKdMZoDVAkhhJDQQBEURXibHUZjNCGEEBJcKIKiCIO76jAxRZ6RIEIIISSoUARFEd4GqNIYTQghhAQXiqAoQvYEad2lw2iMJoQQQoIJRVAU4c4TxNlhhBBCSGigCIoi3E+RpwgihBBCQgFFUBRhN0YrPUFslkgIIYSEAoqgKMJ7nyB6ggghhJBgQhEURcieIDfGaEaCCCGEkOBCERQlWCwSTBY3kSAaowkhhJCQQBEUJRgtdpFDTxAhhBASeiiCogTlgFTODiOEEEJCD0VQlKAci+FWBLFZIiGEEBJUKIKiBJHu0qhV0Kjt6TAaowkhhJDQQBEUJRjczA0DAJ2WniBCCCEkFFAERQnuegQpf+cUeUIIISS4UARFCe5GZgD2nkH0BBFCCCHBhSIoSnA3MsP6Oz1BhBBCSCigCIoSPEaCKIIIIYSQkEARFCUIT5DOkyeIIogQQggJKhRBUYKnSFCM3DFagiTRF0QIIYQEC4qgKEEukdc6eYIUw1SNNEcTQgghQYMiKEowmrx7ggCmxAghhJBgQhEUJfjqEwRwfhghhBASTCiCogQR5XE2RmvUKogpGowEEUIIIcGDIihK8DQ2w7pM7bAOIYQQQjoORVCU4Kk6DFD2CqIxmhBCCAkWFEFRgmyM1roRQVr2CiKEEEKCDUVQlOCpWSKgSIfRGE0IIYQEDYqgKMGrJ0grGiZSBBFCCCHBgiIoSvDmCYqhJ4gQQggJOhRBUYJ/xmhGggghhJBgQREUJcieIDfGaJbIE0IIIcGHIihKMJi89QmyeYJojCaEEEKCBkVQlOCPJ4iRIEIIISR4UARFCV49QewTRAghhAQdiqAowZ8+QUYTq8MIIYSQYEERFCV4nx2mcliHEEIIIR2HIihK8DY2I4Yl8oQQQkjQoQiKEugJIr4wWyQ0tZkifRiEENJlCJkIWrBgAcaNG4f4+Hikpqa6XWfevHkYMWIE9Ho9hg0b5nadZcuWYdiwYYiPj0fv3r3x1FNP+dx3TU0Npk+fjuTkZKSmpuKWW25BY2NjB15N6PHmCeIUeQIAj366HcV/XYE9xxsifSiEENIlCJkIMhgMmDp1Ku644w6v691888249tpr3T72xRdfYPr06Zg9eza2bduGl19+Gf/4xz/w4osvet3m9OnTsX37dqxYsQKfffYZvvvuO8yaNavdryUcGPwpkWefoG7Nyp2VMJgsWLmzMtKHQgghXQJtqDb86KOPAgAWL17scZ3nn38eAFBVVYWtW7e6PP7WW29hypQpmD17NgCgqKgIDzzwAJ588knMmTMHKpWriXjnzp348ssvsWHDBowcORIA8MILL+CSSy7B3//+d+Tl5XX0pYUEo1djNNNh3Z1mgwnltS0AgG1H6yJ8NIQQ0jWIak9QW1sbYmNjHZbFxcXhyJEjOHz4sNvnrF27FqmpqbIAAoCJEydCrVZj3bp1XvdVX1/v8BNOZBHkzhjNKfLdngNVTfL/t5VTBBFCSDCIahE0adIkLF++HCtXroTFYsGePXvw9NNPAwCOHTvm9jkVFRXIzs52WKbVapGeno6KigqP+1q4cCFSUlLkn4KCguC9ED8QPYDoCSLu2Fdp97QdPtGMuhZjBI+GEEK6BgGJoPvvvx8qlcrrz65du4J2cLfddhvmzp2LSy+9FDqdDmPGjMG0adOsB64Orn574IEHUFdXJ/+UlZUFdfu+CMbYjLpmIx75ZDu2lNUG/fhIZNlf5Wjs386UGCGEdJiAPEH33HMPZs6c6XWdoqKijhyPAyqVCk8++SQef/xxVFRUICsrCytXrvS6n9zcXFRWOhpHTSYTampqkJub63Ffer0eer0+aMceKN6bJfpnjP5qRwUW/3QIpTXNeGPmGcE/SBIxlJEgANheXo9xfTMjdDSEENI1CEgEZWVlISsrK1TH4hGNRoP8/HwAwH/+8x+MHTvW43GMHTsWtbW12LhxI0aMGAEAWLVqFSwWC0aPHh22Yw4U75Eg/zxBoodMaU1zkI+ORBoRCRrVJx3rD9XgV/qCCCFRgiRJbguVOgMh8wSVlpaipKQEpaWlMJvNKCkpQUlJiUO/nn379qGkpAQVFRVoaWmR1zEYDACA6upqvPrqq9i1axdKSkrw+9//Hu+//z6effZZeRvr169H//79UV5eDgAYMGAALrroItx2221Yv349fvzxR8ydOxfTpk2L2sowQNEnyI0x2t9miSJSVH6yBZJE/1BXwWS24GC11Rh9ebH1PcwKMUJINPD19gqcseAbLPoyeFaYcBIyEfTQQw+huLgYDz/8MBobG1FcXIzi4mL88ssv8jq33noriouL8c9//hN79uyR1zl69Ki8zpIlSzBy5EiceeaZ2L59O9asWYNRo0bJjzc3N2P37t0wGu1G0bfffhv9+/fHhAkTcMkll+Css87Ca6+9FqqXGhTksRle+wR5FzZttm20GM042UzjbFehtKYZRrOEuBgNLhxoTekerG5CI7tHE0IihCRJeP37A7h96UZUNxqwendVpA+pXYSsT9DixYu99ggCgDVr1nh9PDMzE2vXrvW6zvjx412iHunp6XjnnXf8OcyowR9PkL+RIMAaDUpP0AXxCEmk2G8rjy/KSkBWkh49UmJxrK4VO47WY1RheoSPjhDS3TCZLXj4k+14e12pvKzZ0DlvyqK6RL47IQSO2xJ5f9NhisfLa+kL6ioIU3TfrEQAwOD8FACgL4gQEnYaWo24eckveHtdKVQqYNoZ1nYyTW3mCB9Z+6AIigLMFgkWWzDL7QBVP43RykjQkZMtwTtAElGECOqXbRNBeVYRtJ0iiBASZua/V4Lv9lQhLkaDV28YgTvH9wPASBDpAEpx47ZjtNwnyJcnyK7ExYgF0jnYe7wBm0tPun1MVIYJETSkZzIARoIIIeFn/cEaAMBrN47ApEG5iNdrAADNBjMsls5XkEMRFAUo01hePUE++gS1OXmCSOegqc2Eq175Cdf8cy3KnNobSJKE/c7pMFskaH9VY6e9+yKEdD4kSZILMk7LSQIAJOjs1uIWY+dLiVEERQFKcRPjphN2u4zRjAR1Gj7behT1rSYYzRK+2Xnc4bGqhjY0tJmgVgF9MuMBANnJschO0sMiATuPhXfGHSGk+9JsMMvWjcRYq/iJjVFDtAhq6oQ3ZRRBUYDoEaRVq6BWu0aCdH4OUKUI6py8u8E+omXlTsdu58IP1Cs9HnqtRl4um6OPMCVGCAkPDa1WkaNRqxAXY/0+UqlUcjSoxcBIEGkH3rpFK5f7GpuhTIfVNhvlDtIketld0YDNpbUQ2nfdwRNoaLX3eNrn5AcSCBG07SgjQYSQ8NDYZv1uSorVOnSIjtdZBVFnrBCjCIoCvPUIsi73zxjtLJIYDYp+3rNFgS4YmIOizAQYzRK+21MtPy77gZxFUJ7VHL2N5mhCSJiot0WCEvWOLQYTbL93Ro8iRVAUIPcIclMZBgTgCXJ6nObo6KbNZMbyzUcAANPO6IUJA7IBACsVviARCRKmaMGQntZI0N7KRrR2QjMiIaTzIdJhSbExDsvlSBDTYaQ9GG3jMDylw3QBGqPF+owERTdfbz+O2mYjcpNjcc6pWZgwIAcAsHp3Jcw29+H+Smu3aOd0WG5yLDISdDBbJJqjCSFhoVEWQY6RICGCmjuhBYMiKAow+PIEBWiM7p1hrSKiCIpuRCrsmpE9oVGrMLJ3GlLiYnCy2YhNpSfR0GpERX0rANdIkEqloi+IEBJWhF8xSe8sgqy/MxJE2oXRhyfIHgmSvE6HF80SCzMTADAdFs2U1TTjh33VUKmAqSOtbee1GjXOOy0LAPDNzuPyzLCsJD1S4mJctjE43+YLYoUYISQMNHiIBCXIDRMZCSLtwGd1mMIrZPRijhaRoMIsmwhiJChqWfaLNQp0Vr9MFKTHy8tFSmzlzkrZFN3PKQokGGKLBG0qPdkpO7USQjoXDbZ0V6JLOswWCWJ1GGkPvozRyqGq3lJiIq3WN9N60WQkKDoxmS2yCJp2Ri+Hx849LQtatQr7Khuxape1Z1Df7AS32xneOw06jRp7KxvxxJe7QnvQhJBuj5wOczJGJ+gYCSIdwODDGB3jpwhqc4oEHW9o9dlbiISfNburcLy+DekJOkwcmO3wWHJsDEYVpgMAvth2DIDnSFB2UiyeuGoIAOC17w5gyU+HQnfQhJBuj6d0WLyekSDSAXx5gjRqldxMz7kMXokQQbnJsdBr1ZAkoKKuNbgHSzpEXbMRj362HQBwZXG+QxdogUiJiQxXv+wkj9u7cnhP3HvhqQCARz7djq+3VwT5iAkhxIpcHebcJ4iRINIRfHmClI958gRJkiRHfWJjNMhPjQMAHKltdrs+CT8Wi4S7l5WgrKYFPdPiMPf8fm7XmzjAMTrkKR0mmHNeP1w3qgCSBMx7d7PHafSEENIRGtrcp8NYHUY6hOwJ8iKCdD4mySvFkU6rRn6aVQTRFxQ9vLxmH1buqoROq8arN4xAarzO7Xq9MxLkvkAJOg1yk2O9blelUuGvlw/GeadlodVowS1LfpFnjhFCSLDwWR3GPkGkPYhxGF4jQVoxOsO9CFIu12vVciSIFWLRwfd7q/D0ij0AgL9dPlju8eMJ0T26b3aiw4weT2g1arx4/XAMzk9GTZMBV73yE37aX+3zeYQQ4i+NHsZmiEhQMyNBpD2I6E6Mh+owwO4X8mR0Vi7XaRQiiJGgiFNe24J5/9kMSQKmnVGAa84o8PmcG0b3xuD8ZPx2TG+/95Og12LxTaNQ3CsVdS1G3Pjv9Xh3fWlHDp0QQmTqPYzNYJ8g0iFMFu/GaOtj3kdniEaJWrUKarXKng5jJCiiGEwW3Pn2JpxsNmJIfgoe+c0gv55XkB6Pz353ttxI0V8yE/X4z21j8JvT82CySLh/+a94/H875TEchBDSXuwl8l2nY7TW9yok1Ag/j1+eIA/GaBEJ0tuiSUyHRQcvr9mHLWW1SImLwcvThyM2xrUaLNjExmjw3LRh6JuViH98swevfXcAvxyqwdCeqchPjUN+WhwK0uIxKC8ZarXvVBshhBhMFrkCuSvNDqMIigKEgPGvOsx7Okw0XBSRoGO1rbBYJF7sIsDOY/V4cdU+AMBfpwx26AwdalQqFX4/8RQUZiXg3ve3YFNpLTaV1jqs89sxvfHXKYPDdkyEkM5Lo0LgePIEMRJE2oU/JfI6H8boNicRlJscC41aBYPZgqrGNuT4qDDqykRCBJrMFvzhgy0wWSRcODAHlw3tEdb9C35zeh4G5yXjx/0nUH6yBeW1Ldhf2Ygdx+rx7Z6qiBwTIaTzIVJh8ToNtE7Xqs7sCaIIigJkEaT15gmyTZL3YIx2FkFajRq5ybEor23BkZMt3VYE7TxWj2v+uRZ3jO+LO8e778sTCv753QFsK69HSlwM/jZlsF8VXqGiKCsRRYqu03XNRpz+2NcorWlGXYvR7XBWQghR0uChMgywR4KMZmu/Ok8joKKRznOkXRh/PEG+miXK6TDFNugLAtYdOIGGVhM+KTkatn3uq2zAc9/sBQA8dOlAZEeZAE2Jj0FPW7p0x9F6j+vtPFaPY3Xd971DCLHjqUcQYPcEAZ0vGkQRFAUYAkiHefQEmYUx2v5mZMNEoKbJAADYX9UYljlqZouEP3ywFQazBeedloUrh+eHfJ/tYXCetU/R9qN1bh8/crIZv3nxB1z6/A+obmwL56ERQqIQ4QlyLo8HrNcucY3qbL4giqAowBiAMdpjs0SndBigjAR139EZ1TYRZDRLYemi/MYPB7G5tBZJei0ev3JIRNNg3hicnwwA2FbuXgSt3X8CRrOEE00G/OXjbZAkltgT0p3xVB4vSOikFWIUQVGArwGqysd89QlyEEGMBKGm0SD/f+cxz6mfYLD3eAOe+no3AODByQPQIyUupPvrCINsHau3eUiH/XLIPn/si20V+GzrsbAcFyEkOvGWDgM6b4UYRVAUIHuCvHaMtkWCfHSM1ruNBHVfEXSiyZ7K8SaCVu+u9OqP8YXRbMHdy7bAYLLg3FOzcK0fXaEjyaA8ayToQFWj2xz+L4drAAAjeqcBAB767zZUNTAtRkh3RU6H6d0XUnTW+WEUQVGAX54gP/sE6T1EgrprOuOEMhJU4V7k7Kqox01vbsCtSza0+zy9tHoffi2vQ0pcDBZdPTRq02CC7KRYZCfpYZGAnccaHB6raTJgf1UTAOCV6cMxoEcyTjYb25UWM1skvL3uMO7/cCtO0FtESKel3pYOS2QkiAQbf/oE+YwEmT17gpoMZtS1GINyrJ2NE03KdFiD24v4T/tOAACO1rWi0ku047FPd+CmN9e7RJS2HqnFC7amiI9dPqjTtCMQQ1ydzdEbD1tTYX2zEpCdHIu/Tx0KrVqFL7dX4NMA0mK7Kxpw9as/4cGPtuHdDWW4670SWDi+g5BOia90WGftFUQRFAX44wnSx/iXDlOWyMfGaJCZqAMAHOmGviCDyeIg/mqaDG5Fjkj9AJ5LxuuajXjjx4NYvbsKl77wAx7/3040G0xoNZpx97ItMFskTB7SA785PS/4LyREDM5zb44W5+OMPukAgEF5KZh7vrXH0sN+pMVajWb8/avdmPz899hcWotEvRaxMWp8v7caL63eF+yXQQgJA54myAs66yR5iqAowGjy3SdIzJxq9bNZoiDPFg062g19QSebrVEgjVqFvlkJAFxFjiRJ2KAwAe/w4BvaZouWaNQqmC0SXvvuAC545jvM+89m7KtsRFaSHn+NcFPEQBkol8k7vmZhihZ+IAC4c3w/OS32ypr9HrdZ2dCKS57/Hi+u3geTRcIFA3Ow4u5z8NfLreM5/vHNHqzdfyLYL4V0Y47VtXhs9UCCh6gOS3ZTIg/YewU10RNEAsUfT1CsTdy0Gt2rbE8iSKRmvKV5uirCD5QWr5Mv+M4i5/CJZofIhicR9KstWjJpUA7emDkSPdPiUF7bgq93HAcAPHnVEKQn6IL+GkKJKJPfc7xBri5sNZrx6xHrax1piwQB1vfVHyadCgD4ZEu5R2/av384iANVTchK0uPVG4bjXzeORI+UOEwdWYCrR/SERQLmvbuZJmsSNGa+sQGXv/gjKhtaI30oXRp/q8MYCSIB02J704iUlztibSq7xcMbzG6MdpxSnpOsBwBU1ne/LwhRGZaZqMOAHkkAXCvENhyypn6EoXynh3SYEEGD81Nwfv8crJh/Lu4Y3xd6rRq3nlWI8/vnhOQ1hJL81DikxsfAaJaw97i1h9K28joYzBZkJurQJ8Nx4OvZp2QhM1GH6kYDvt/rOnfMaLbgw43lAIC/Xj4YFw12nJf22OWDcEp2Iqoa2jD/vRKY6Q8iQeDgiSaYLBKO1na/77hwIqrDPBmjRZ+gJnqCSKCItE1avOdIQqzWezrMXbNEAMhJskaCjtd3vztv0S06PUGHgT2sUQ9PIkh4eQ6eaHJr7BO+mSE2M3GcToP7LuqPHY9dhD9fOjA0LyDEqFQquXO0eH0bFKkw59RejEaNy2zn6cNN5S7bW7O7CtWNbchM1GHCgGyXx+N1Wrw8fTjiYjT4YR/9QaTjtJnM8nefSNeQ0GCPBHlIh9m8Qs1tjASRAJAkCbXN1g9varznQZayJ8hjOszWLFHjPh12vBuGiqtt6bCMRL0sgg5WNzmcQ+F/uXhILrKS9JAkYFeFY8l4XYsRh09Yu24L0SDQhHk6fbAR/YKEL2ijzRQ9sne62/WvGt4TALBix3GXisP3NpQBAK4c3tNjaveUnCT8dYrVH/Tqt/th8pBWI8QfxIXZ+f8k+NT72TGakSASEM0Gs+wJ8uYpiY3x7gnyFAnKtqXDKuq6nwgSfWkyEnTIStIjI0EHi2Qt3QaAqoY2HKhugkoFjOiVLgslZ/P0dluUpGdaHNI6me/HF/bO0XWwWCS5PH5knzT36+cl49ScRBhMFnzxq71cvrK+Fat3VwIArhnZ0+s+ryzOR1KsFs0Gs4vgJCQQlMKnkSIoZEiSpGiW6METxEgQaQ8iZaPTqhEXo/G4XpyPSJB9gCqN0QJxbjMSdFCpVBjglBITUY/TcpKQEh+DgXnuU2a/OqXCuhKDFa95b2UjTjYbodeqMSjP/WtVqVS4otgqcpYrUmLLN5fDbJEwvFcq+mUned2nWq3CsIJUAMDm0pNe1yXEG8oUWD3TYSGjyWCGaLHmKR3GSBBpFyIVlhYf47W82p4O879jNADk2kRQTZNBTpl1F5TpMAAu5uj1Bx2jHkIkOVeIKU3RXY0+GQlI0GnQarRg2S/WdNbpBaleR7hMKc6DSgWsP1SDsppmSJKEZbZUmL/jQob3sp7zzaW1HXsBpFvjEAnqZKXZnQkhNrVqlZyVcIbVYaRd+GOKBuyVYy0BpsNS42Nkn1B3K0uusVWHZdgaRtojQdYUjHNTQJEO23WswaFyydkU3ZVQq1Vy1EeIoDM8pMIEPVLicGbfTADAR5vL8cvhkzhQ3YR4nQaTh/rXLLK4VyoAYBMjQaQDKCNB9ASFDrlRYqzW48266BjNPkEkIPwVQb7SYZ76BKlUKtkX1N0qxE4o0mEAHNJhjW0m2QwsRFBhZgJiY9RoMZpx+IR1dlZ9qxGHbKboriiCAMhpQHER8WSKVnJFcT4AYPmmI7IhevKQHh67yTpTXGAVWodONHOmGGk39Q7GaKbDQkW9jx5BACNBpJ3I6bAEz5VhgP/pMJ3G1Vck+4K6Wa+gE07psL5ZiYjRqNDQZsKnW47CbJGQnxond9XWqFU4LdcxJSaiQPmpXc8ULVCm+VQqe6rKGxcNzkVcjAaHTjTjo81Wb5C/qTAASImPQb/sRABASVltYAdMiA2mw8KDEJieJsgDnB1G2okw76b6iAT5LJH3YIwG7A0Tj3cjEdRqNMtfiqLqTqdVy6bd/1t7GIBr6se5Qqwrp8IEonM0AJyabTWJ+yJBr8XFg3MBWCfFF2UlOIzZ8IdimzmaKTHSXpgOCw++GiUCQIKYIs/qMBIItXI6zPuFx2d1mId0GABki4aJ3cgTJMRljEaFZMUH19kcfUahY+rHuULs13Lrv0N6dl0R1C8rURbPI3z4gZRcMTxf/v+1IwsCnps23CaaNh2uDeh5hAjYJyg8iHOb7EUExYmpBkYzLJ2oGzxFUIQ5KVeH+YoEWf9UJovkdm6T3CzRbSRIdI3uPpEgORWWoHe4OItIj+CMPk4iyCaSnNNhXbEyTKDVqOXXN7rQtx9IMK5vJk7NSURafAyuHO69N5A7RNpty5FaNk0k7aKRnqCw0OijWzRgjwQBngt4ohH/XIwkZPhrjI5V9BBqNZpdOvJ6iwTZ54d1n0iQmBvm3IBSKYJS4mLQLyvR4fHTcpOhUllN5Ieqm3Cw2mqQ7srpMABYcMVg/LjvBC71s7oLsHqolt95Jkxmi890rjtOyU5Ekl6LhjYT9hxvlKNwhPhLQxvTYeFACExvhQ+xMWqoVIAkWXsFJfhZJBFpGAmKMLII8mGMVnp93JmjPfUJArp5JCjR8eI8QCGCzuiTBrXT2ItEvRZ9MhIAAB9sPALAaorubBPiA6V/bjJuOasw4DEgiXptuwQQYC3PP52+INIBaIwOD/5Uh6lUKjka1Jm6RlMERZiTTWJumPcLiUql8jo6w1PHaKB7GqNrnMrjBWkJOrmB5Mg+7lM/wjckRJDSOEyCy3D2CyIdQFki32wwM60aIuSRGV7SYQAQ3wm7RlMERRhhjE73427aW4WYtxL5bNtFv77VhJZO1sOhvVTLjRL1Lo9dNSIfmYl6TB7Sw+1zRcqswiYau3oqLJIU28zRJewcTdqBsw+os1UmdRbkdJiXSBAAOQXWmXoFUQRFEIPJgibbm8WXJwhQVoi5M0Z79gQl6bXycyu7yTR5T+kwAPjDpP745c8TUZAe7/a5zt6UrmyKjjSiTP5AdRNO2qJ3hPiLsw+I88NCgz/VYYAiEtSJUpMhE0ELFizAuHHjEB8fj9TUVLfrzJs3DyNGjIBer8ewYcPcrrNs2TIMGzYM8fHx6N27N5566imf++7Tpw9UKpXDzxNPPNGBVxMaRBRIrfKeaxXIkSCnGWBmiySPeXAnglQqlZwS6y7T5D2lw/xhgFMFGSNBoSM1XoeiLKsHa3MZU2IkMJwjQTRHhwa5T5APs3NCJ+waHTIRZDAYMHXqVNxxxx1e17v55ptx7bXXun3siy++wPTp0zF79mxs27YNL7/8Mv7xj3/gxRdf9Ln/xx57DMeOHZN/fve737XrdYSSmmZ7o0Rng647hN/HOR0mUmHKdZwRKbHu0itIjGLISHBNh/kiNzlW7tuUnxrnNqVGggeHqZL2YDRb5Ki4KFygOTo0NPhRIg8A8Z1wfljIatgeffRRAMDixYs9rvP8888DAKqqqrB161aXx9966y1MmTIFs2fPBgAUFRXhgQcewJNPPok5c+Z4bc6WlJSE3NzcDryC0GM3Rfvu0AsomlEZPIsgT9O/c7vZ6AwxNyzdTTrMFyqVCgPzkvHjvhM0RYeB4l6p+GDjEZqjSUAooz49UmJR02Rgr6AQIY/N8OUJYiQouLS1tSE2NtZhWVxcHI4cOYLDhw97fe4TTzyBjIwMFBcX46mnnoLJ5F2ZtrW1ob6+3uEn1ARiigaAWK1Ihzl6gkSjRJUK0HqIKHW3CjHhCcpsRyQIAM7qlwUAOOfUrKAdE3GPiASVlNbKaV1CfCEuzPE6jXwjGepIUHVjG37z4g94a+2hkO4nHByoasS/vjvgcQqBEiE4faXDWB0WZCZNmoTly5dj5cqVsFgs2LNnD55++mkAwLFjxzw+b968eXj33XexevVq3H777Xj88cfxxz/+0eu+Fi5ciJSUFPmnoMD/YZDtRXSL9rfPiqcSedkUrVF7jI7ZewV1/XRYs8Ekdyx1Z4z2h9vOLsSnc8/CdWf0CuahETecmpOEBJ0GTQYz9hxviPThRA0lZbWY+upP2MwImVsaFL1rxGDP+hB7gtYfrMHWI3Vy+4zOzNNf78GC/+3EV9srvK5nMFnka0yyj3SYXB3Wiar0AhJB999/v4vh2Pln165dQTu42267DXPnzsWll14KnU6HMWPGYNq0adYDV3s+9Lvvvhvjx4/H0KFDMXv2bDz99NN44YUX0NbmWQA88MADqKurk3/KysqC9jo8cdLPuWECkQ5z8QR56REkyO5GDRNFFEivVct3JoGi1agxpGeKX14t0jE0iqaJX/x6DJLEaBAA/N/aQ9hw6CSWbyqP9KFEJfVyiiZGLt0OdTpMeF06U7rHE1U2f6ivSQLKc+qrRD6uE0aCAvIE3XPPPZg5c6bXdYqKijpyPA6oVCo8+eSTePzxx1FRUYGsrCysXLky4P2MHj0aJpMJhw4dwmmnneZ2Hb1eD70+vAZYURKc5mcFk5wO82CM1mk9X/BzkmyjM7qBMVr4gTIT9QEP9SSR4fz+2fhp/wk8v2of9hxvxONXDunyXbp9IXondZe2FoHiEAmyXZwbQxwJEhHmriCCGmyCzpdwFCnGeJ3GZ0f5BJsI6kyRoIBEUFZWFrKywu+R0Gg0yM+3Tqz+z3/+g7FjxwZ0HCUlJVCr1cjOzg7VIbYLf4enCvQe+gS1eRmZIVCOzpAkqUuLgxoPc8NI9HLTmYVoM1nwjxV78OX2Cvxy+CSevGoIJgzIifShRYSTTQYcsM2t6w43Lu1BWbGUpNc6LAsVQvz446OJdhptc9d8pRAb/BiZIYgXxuhOdH5CVh1WWlqKmpoalJaWwmw2o6SkBADQr18/JCZah1bu27cPjY2NqKioQEtLi7zOwIEDodPpUF1djQ8++ADjx49Ha2sr3nzzTbz//vv49ttv5f2sX78eN954I1auXIn8/HysXbsW69atw3nnnYekpCSsXbsW8+fPxw033IC0tLRQvdx2URtoOswmgpwn9HobnirIthmjmw1mNLaZfJY6hhNJkrD9aD36ZiXK4dSOUO2lUSKJTjRqFeac1w/nnpqF+e+VYG9lI25Z8gtmjuuDR34zKNKHF3ZKjtTK/+9Og48DQVmxJL7PQm2MFiKoK0SCRNSsvsV7JEiZdvRFgl5EgrpoOiwQHnroISxZskT+vbi4GACwevVqjB8/HgBw6623Oggasc7BgwfRp08fAMCSJUtw7733QpIkjB07FmvWrMGoUaPk5zQ3N2P37t0wGq1/KL1ej3fffRePPPII2traUFhYiPnz5+Puu+8O1UttN8o+Qf7gyRhtUBijPRGvs4aMG1pNOF7fFlUiaM3uKty0eAOuGdkTi64+vcPbk7tFt7MyjESOwfkp+PR3Z+HvX+3Gv388iMU/HcK0UQXon9u9WhUoeyZVNbR1+ehte1B2MU4KkyeoxeZ1aTGaYbFIndYzKEmSLBh9ddlu9LMyDLBHgtx5giRJgiQh6s5ZyETQ4sWLvfYIAoA1a9Z4fTwzMxNr1671us748eMdjJTDhw/Hzz//7O9hRpRaOR3mnyCJ9ZAOM5itokgf493nnpMci4bWRlTWt6JfdmKghxsyfi2vAwB8v7c6KNurkeeGMRLUGYmN0eDPlw7EjmP1+Gn/CWwure2GIsheEWYwW1DbbPTbO9hdaHBjjA51dZgyAtRqMssX/c5Gm8kCo9l63QxmOkyOBBmcK5jNuPi575ESF4Pld4yLKkEf1SXyXR1RHeavdyXOwwBVfyJBgKJXUJQZLctPtgAAjtW1BmWshz0SxItGZ2aYrWKsuw1XtVgklJTVOiyjL8gV+eKsV6TDQm2MVlzcO/MwamXa0Fc6TIhNX+XxgCIS5JQO21/ZhANVTdhcWov9VY2BHm5IoQiKEGaLhLqWIPcJ8uIJAoCcpOjsFXSktln+f0kQ5kdVNwUmLkl0IosgJ0HQ1TlQ3YiGVhNiY9TyXDVWiLkiN/CL1cqpmoa20KbDlBGOzuwLUopFX2Zyf+eGAZ47RpfW2L/j1x+Mrr5XFEERor7FCJHF83dsht5DJMhfERStvYJEJAgIzvwokQ7L5MyvTs2wXqkAgD2VDd1qJtQm22dgaM9U5KXEAaA52h1Kw25ymErklVVPzgUqnYnAIkEBVId5mB1WWtMk/3/DoRq/jzMcUARFCGGKTtJrEeMjjSXwVR3mrUQeiM7RGRaLhKO19uMJhgg6weqwLkF2UizyU+MgScDWbhQNEp+B4l6pyO5G/b0CxbFPUIy8LJTNNlsUht/OnA5TRn8aDSZYvIyrEf2E/KoOU0SClH+HwyeUkSCKIAJFeXwAKRuPxmg/miUC0Tk6o6qxTe54DQBby2thNFu8PMM7kiTZh6cyHdbpESmxzd1KBFnTBcUFachKFiIoem5cogVlibwwRpssksv3YzDpMukwRaRGkuxCxx3KtKMvRCTIZJEcvteV6bDy2haU17a4PDdSUARFCDFB3t/KMMC/2WHeyInCdNiRk9YPR15KLJJjtWg1WrC7wr/5UT/tq8bUV3/CNlt1GWD9cAtRyBL5zk938wU1tpnk+WnWSJD1M9sVI0HrD9agrrn9Hh57iXwMEnQaiIKjUPqCHIzRxs6bonVuJeAtJebvBHkAiI+x34gru0YLESQsGxuiKBpEERQhTgbYIwiwR4LaTJ4iQf6lwyrr2wIOGTcbTHhx1V7sqwzugMsjNj9Qz/R4eX6UP3f9Da1G3PVeCTYcOonH/7dTXi5SYQk6TVAaL5LIInxBJWW13WKm2NYjtbBIQH5qHHKSY+V0WFUURW+DwboDJ3DNP9fipsXr2/13VabDVCqV3RwdQl9QNESCWo1mmDoQLQdcm0p66xXUqOjH5AutRi1fh0SvIJPZIvs+Lxxo7QC/jiKIBDo8FVB4gpw+fHKfIB8iKMv2hSr6jgTCZ1uO4e9f78ETXwRvQC6gEEFpcSjuZe3o7c/U7Ke/3iPfHf+0/wS2H7VGg+RUGP1AXYLBeSnQqFWoamjD0SC0T4h2hB9IiD+7J6hrvfYtto7Ym0pr29UfzGi2yN5I4VVJDkOZfHOEPUEtBjPO+/sanLNoNX7pgMHYWSh6E45yOkzv37VKnh9mOz9Ha1thskjQadW4dGgegOgyR1MERQh5blhAniBbOszUPmO0XquRfTKB9go6eMLq7t99PESRoNQ4FCvu+r2xrbwO/7f2EADg1Bxr08d/f38QAHCi0dYokamwLkGcToP+uUkAuke/INkUbYuKiorOrpYOO1htrxZ6cdW+gJ+vFDoiTWPvGh06EdQS4eqwX8vrcKyuFUfrWnHtaz/j1W/3ezU1e8IlEhSkdBjg2itIpMJ6pcdjdGE6AGBfZaP8XR1pKIIihH1umP8iSO9zirzvP6e4swzUHC3EypGTLUEdHigMcj3T4jGsZyoA4EBVk3x+nDFbJDz40a+wSMClQ3vgKduYjU+2HEVFXStqmtgosath9wVFV3+RYCNJkvwaRVRUfF7FzL+uglIErT9Ug3UHTgT0fCF0YmPUcnWtPR0WGk+Q0WzvsgxEJh0muuvH6zQwWyQ88cUu3Pp/v+Bkk/vvS084R8u8dY0Wpml/jNGAa9fow7by+F7p8UhL0Mk3rhsORcfnmSIoQrTHGC08Lq1Gi0Me3V9jNNB+c3SZTc1LkuMXWEcRxuj8tDikJehQmGltDucpGvTO+lJsOVKHJL0Wf7l0IE4vSMWoPukwWSQsWXtIToexPL7r0F3M0UdOtqC60YAYjQqD8qxjQhL0Wjm9UBlFBQ0d5VC19XN/es8UAMCLqwOLBrkb6ilHgkIkFp1FTyTSYdttImjWOUV4/Ioh0GnVWLWrEpOf/x77Kv3vxOxvJMhisc8YCzQSJM5X6Ql7JAgAzuhjjQZFS0qMIihCdMQYDTiaowOJBNnN0YF9oR5RNDQ8UBUcESRJkmyY65lmbQon0gDu+gVVNbRh0ZdWT9I9F54qC7pbzi4EALz982FZrGWwUWKXQaRJfy2v61D7hGhnk80LN7BHssNnvaulxJoNJlTYvn8WXDEEWrUK3++t9ssLKHDXwC9R0SsoFDiLnkikw7bZvI9D8lNw/ehe+PjOM1GYmYCjda14eY3/QlKcI61tmKmnc9ZkMMlNff0ZmwEoI0HWbYoeQb0zrCJolC0lFi39giiCIkSgc8MAIFYhcpQpqTazf54goH29glqNZlQr8rfBmv1S3WhAm8kClQroYeuMO8yLL2jB5zvQ0GrC4Pxk/HZsH3n5xAE56JMRj/pWEz7aXA6A6bCuRFFmIpICbJ/QGbE3SUxzWJ7VxRomiihQWnwMBuen4IrifADASwFEgxq8RIJCZYxudpqM7vx7qGk2mORoz5B8awRtYF4yHr5sIIDAGs022toI5KZYrweeqsNEFEirVvl1fQGUniBbJKjGUQSJSND2o3VRkeKlCIoQwhjt78gMwFp+GKOxKnflXYi/zRKB9o3OUEaBgOCJIOEHykmKlaNYxQXWC0BJWa2D4e/r7RX4uOQoVCpgwZQh0KjtU4g1ahVuPssaDRIRMqbDug5qtQqn2/xiXTklJl6biHwJ5AqxLpIOE+n0PrbU9x3j+0KtAr7ZWSlXefqiwU3Ztt0YHRpPkGs6LLxRyZ3HGmCRrKJYfI8D9nTxweomv71BQnzkpVpvPj2lw5zbEPiDvTrM2r1baYwW++yZFgeLBGw6HHlfEEVQBJAkqV3GaACI1bp2jfZ3dhgA5AhjdAB3lcK3IwhWOkxsV6TCAKB/jyTotWrUtRjlirTy2hb84YOtAIBbziyU+wkpuXpET6TE2QVlOqvDuhRd3RfUZjJjx9F6APYbAYGI3lZ1lUiQ7XMt/H9FWYmYbCudfnn1fr+24a5iKSnEfYKc01/hbpYomsKKKJAgNV4nD9r19/MhomV5PiJB7iJuvojX2yNBNU0GNLaZoFJZi18Eo/pET0qMIigCNBnMcpVBoCLI3RBVg61k3j9PkPVNX1Hnf9vyMlskqMj2pbW/qjEojeuEHyhfIYJiNGoMtZklS0qtIzTm/Wcz6lqMOL1nCv54UX+324rXaTF9dC/5d6bDuhZdXQQdqGqCwWxBSlwMCtLjHB7ravPDRCSoMCNBXjbnvL4AgP9tO+ZXpFk26+qV6bAYh8eCjXMkKNzVYUIEDbaZ5pUI4eyvr8o5EuRJONp7BPlnigYcI0GHbVGg3ORYB5+b7AuKAnM0RVAEECHL2Bh1wF2N43TWP5nbdJgf1WFCcFQ2tMnP84WI2IzrlwGtWoVmg1k2NnaEI06maIF9XtRJPLNiDzYePokkvRYvXDfcq9CbMa4PdBo1NGoVeqTEelyPdD6EV2x/VaPX7radFXFDUJAe55J2yO5i88Oc02EA0D83GaML0yFJ8KsJoFtjtO1CHar3R4uLJyjMIsgWKRzkFAkC7ClUf2fsifMnp8M8RoICqwwDFJ4gg0kuVBGpMMEZNhFUUlaLNlNkOm8LKIIiwMl2psIAZTpMIYKEMTrG958zI0GH2Bg1JAk45mc0SIiVwsxE9LKZ2/ZXdjwlZk+HOX5AhDH0863H8Moaa3j8iauGyvv2RE5yLP7vllH45w0jWB3WxchM1KNnmpgo759vpDMh/HH5qXEuj4n5YdE0+LgjHKp2TIcJZJNui+9ITr18cXZjjA5TJCiY/dJ80Wo0Y6+tUa1zOgywi6CS0lqfzRMNJotsociXPUHeI0GBpMPk6rA2s1wZ5iyCijITkJmog8FkwdYjkf08UwRFALspuh0iSMwPU3qCbP/X+xEJUqlU8hu//GRgIqhnWhyKMq2Nrg5Ud9wc7emLX3ygxXm6YUwvTB7aw69tjinKwETbfBrStejKTRM93RAAoTFGl55oxoMf/RrUnl/+UNdilHt59XESQclyibvvSI5bT1CIS+SFCBJm7HBGgnZXNMBkkZCeoHMb5T4tJwlxMRo0tJl8phOVIrFHqq/qsMC6RQNAnCIS5FweL1CpVHKVWKR9QRRBEaC2HXPDBPL8MDeRIH88QQCQb/uida768kS5wsDcN9vmCwqgMZc7JEnymA7rkRIn9zPqn5uEP08e2KF9ka6BEEGrd1dF9TDVOW9vwsRnvg1oQro/kaD6VlPQog9v/XwIb68rxZKfDgVle/4iokBZSXoXn0lynEhn+RYx7tI0oa4OE32CMm1R5nA2SxSdogfnp7it0tIqvJS+SuWFKTpep5GzEQ2tJrefKTFj0p/hqQLl7LBS0S06I8FlPSGCAintDwUUQRFAeILakw4TKa9WtyXy/v05hehwrvpyR7PBhGrbZPaeafHom2WNBO3vYIVYbbNRvpPKc/PFf9vZRRicn4yXpg93MNSR7sukQbnQa9XYePgk3v/lSKQPxy2NbSZ8/usx7KtsxH82lPr9PHdFAoLkOK382Q5WhZgoWxbiK1w4V4YpEZEgb3OsBJHpE2T9vhLtN8LZLFG0DnBnihbIA6h9REobbNGdRL1WPudmi+Q2siX6w4leVf6gnB0m9whKd41wXjq0Bz6ecyZeuWG439sOBRRBEaBGHp4aeCQoNsa1RN4+QNU/sSDuNo/48QUovpyTYrVIiYtBX1sp5oEO9goSUaCsJL1bkXPr2UX47Hdny6KLkIL0eNx9wakAgL99viMq++bsUQwYXvLTIb87XHuLBKlUqqBPkxf7q6gL7zl0VxkmSLa1uPDH2OyuT5CILDUZzDC3Y6ioL5ptJfGiwW04myUqI0GekM3RfkaCEmO1ttlr1siSu/MuRLeIRvqD8ATVNBlkH5uzJwiw9qwbVpAqz36LFBRBEaC9PYIAD+mwdkeCfIsgsU6BLYUmPEFH61rlKcHtobzWNjPMzZc+IZ645axCDMlPQX2rCY98ut3l8WaDCU9/vRv/LSmPwNHBoaP1sbpWfLGtwudzWgxmOdpa4MYTBCh9QcGJBImbG3+LI4KFu8owgT0SFEg6zLVEHgiNObpFjgRZ/xatRku7JrgHisFk75TuzhQtECJo9/EGrylBe3sBawPEJC/nvaoDkSBRHp8Uqw2oKXC4oQiKAB0zRrumwwJplgjYRZA/xmjnhoZpCTq5B09HTJWe/ECEeEOrUeOJq6wdw//3awW+2m4XGQeqGnHFSz/hhVX7cP+Hv4YkGuALcbESUYl//3DQp39JRGUS9VrZF+OMuBMPRq+gpjaT/B1kHV0TvrSOp8owwJ7O8i8S5GrY1WnV8miHUPiCRLooU9GDLBwpsT3HG2A0S0iJi/H6fZmdFGuvoPRScdXoNBU+2YuXSojuQESQiASJt33vjHi/u01HAoqgCNARY7S9Osz64ZMkyW6M9jOsKCpQKupbYfIRrreLFfsdqt0X1P6UmLvtEuIPg/JScPs5RQCAv3y8DXUtRny5rQKXv/gjdtvSUS1GMw6fCG/lE2AXQXPP7wedVo0tZbXyYFRPKFNhni4WwewV5OwDOl4XntJ7SZJwwIsIktNhPjxBZouEJpsgca5aspujQxcJSguzCJKbJOYn+xQTsi/Iy3vOuQGipzSk2SLJlXwBiSCd49+kd7rr3zqaoAiKADXCGN2OrsYiHdZqi/4YFCLG30hQVqIeOo0aZovks+lhmZvRFqJFe0fM0Ue8GEEJ8cW8CaegKDMBlQ1tuPqVnzB76UY0tJlwRp802bcW7mGrkiTJIuzMvpmYMsw6CuLfPxz0+jxvpmhBIOmwb3Ycx7z/bPYYUXGOAB8NU0qspsmAhlbrCAXnkmnAHpHwVR2mND47968JZddo4QFK0GnttoQwVIhtk03RnlNhgmLRaNaLL0iOBNm6bXtKQ55sNsBskaBSBTboO96pAXCBGz9QNEERFAFE2WH7qsMcP3zKrs/+TvlVq1XIs/WH8OULcpe2Ck4kyFVcEeIvsTEaLLxyCABgr61dw61nFeKd28ZguO1ueFeYRVBVYxtqmgxQqYB+2YnyUN8vt1V4rcT057PgbzrsQFUj5v5nEz7ZchSfbz3mfn9OkaBw+YJEZVheSpzbYggRkWhsM3lNZQpxp9eqXW78EvWhK5MX6bA4nUbu9B+OXkG/lnvuFO2MsnO0pzRso1N7AU9pSGGKTo/XBWReTnBqfeBO8EYTFEER4GSH0mGOnqA2hQjyNx0G2NNQvnxBsjFaoeaD0StIhOR70hhN2snoogzMn3gqeqbF4eXpw/HnSwciRqPGablJABwrtcLBngrr56FPRgLidBr0z03GWf0yYZHgtR+Pt8owQZacDvMsgkxmC+5etkWuHPXk2XMWZMfCVCEmBi/3yXR/UVSmtryVuXvrYhzSdJjtOzdep3FboBIKjGYLdh6ziiBvpmjBwLxk6DRq1DQZ5PJ0Z+yRIOEJct9kUoigQFJhgFWcqhVZO3fl8dEERVCYaTWa5buH9hijXdJhtn9jNCqo1f6bz+QyeS8iqKnNJKfulKF6USF2sLqpXdURdS1G+QPHdBjpCL+feAp+uO98XDLE3lFciKBwp8N2VVgvVqflJMnLbrFFg95dX+YxRRNIOqzKiyfolTX7HQbMempjIfYnLoLHasMjgrz1CAKsLT7ETZ43c7SI8rhr4BdKEaSMBMUrhoSGkn2VjTCYLEjUa/0SE3qtBoPyrb2EPKXEnBtNyk0qW9xHggIVQSqVysEX5GvcUaShCAozIhWmUasC6sIpiPWQDgskCgT41zBR3KGmxMXIdwviuTqNGm0mS7uarYkv4fQEnVxOSUiwECLo0ImmsM53EqJL7B8Azj01C0VZCWhoM+H9X8rcPs+fSJBIh51oMrgtZthWXofnVu4FAFw9oicAyCZkT/sTqZOwpcOqrd81fdz0CBKI75k6L+bohlbH6iYlwucSSmN0vE4rp8NC7QkSpuhBecl+3+T6migvRmHYq8PcG6Pl8vh2zGGMt1WIxWhU6JES3Te6FEFhRpkKa0/ZoLhTEmWt9uGpgXVVFned3kSMmADs7FXQatRySLs9viD6gUgoyUrUIy0+BhYJ2Hu84zPu/EWk35QiSK1W4aZxfQAAn2w56vIcg8mC47biBG+RoIwEHTRqFSQJck8hQavRjPnvlcBkkXDJkFzMtzWULD3R7FYwieivGFsQtnSYl8owgT8NExu8zLOyD1ENhSfIPm5CRORD7Qnabpsc761JojO+Jso7V4fJnqCW4KTDAHuvoJ5p8dAEkKGIBBRBYUaIoPakwgBFOszY0UiQ7/lh3nr5yINU21EhJvuBKIJICFCpVPaUWJh8QRaLhD02waUUQQBw9ilZAIAdR+tdOkhX1LXCIll9FN7uuNVqFTJt4xqcy+Sf/no39lY2IjNRj79NGYIeybGIjVHDZJFQ5vT5bjWa5YvbyD7WiEE4RJAkSV57BAk8XZCVyOkcfXg9QXI6LMaeDgu1J2hvpfX929/pPeUNIYJ2HK13GwmVmyXK6TAPkaAOiSDr+XHXKTraoAgKM/bKsPZ10NQ7GfJERMjf8niBuOs8VtfisRJDRGzcdbGVzdHtigT5Dv8T0hH651p9EbttPp1QU1rTjBajGXqt2iXd0zsjHkmxWrSZLC5m7SOKzum+IsNyhZiiTH7j4ZN43VaCv+jqIUhP0EGtVqFQ9u05fj6F4ImL0WBQD2t0oabJEPK04fH6NrQYzdCoVV5Lpj2lZpS4G54qCNX8MLNFkotQ4nUaOdIR6nSYGGsSyHdlfmocMhJ0MFkkt5FQeWyGc4l8kIzRgL1XULRXhgEUQWHnZAdGZgBArNZxdlig3aIFOUl6aNUqGM2SxwZs3iJBHSmTL2ejRBJiTrWZk8NVJi/2c0pOokv4X6VSyRO+nTv5+mOKFtjnh1kvTpIk4a+f7YAkWX1A5/fPkdctyhQz/hwjtcr9Jcdp5Tv2UM8QE5VqPdPivJZbi6iEt0hOvZvhqYIkDxf0jqKM+MTrtLI3M9TpMDF7KzvZ/9ldKpVKfj+56wPX4FwdJs65szG6A54g0TWakSDiQv/cZNx+bhEuHJTbruc7l8i3Nx2m1aiRm2L9YHkqk/fW1bkoq/3psCOcG0ZCTLjL5MV+Ts1xn7YY2jMVgBsR5IcpWuDcNfrTrcdQUlaLeJ0Gf7zoNId1RUNTZ3O00o+nUqnk74BQN0z0VRkmkBsm+mGMdhcJClWfIOEHUqms38HhSIc1tpnk1JX4O/lLjk00HXcjggLtE9SeSNC0Ub0wujDdoWozWmFpTpgZ0TsNI3qntfv5oirBWQTpYwLXsz3T4nDkZAuOnGzByD6uj8vdotPdeIJsX7KVDW2obzU6VI/5QhZXbrZLSDAQIuh4fRtqmw3t9uD5i6gM8+TdGGoztv5aXuuwPJAZeiIddry+Da1GM578YhcAYPa5fV2mfAuxcdA5EuQkuvJS4nCgqilskSBvlWGAn8Zof9JhQe4Y3aLwA6lUKrsICmGJvPibJOq1srjzlxwhmJ1EkMlskYWb69gM+2tpM5nlCr1AJsgLJg3KxaR23uiHG0aCOhke02EBRoIARcNENxViDa1G2b/kLhKUHBsjh+cDiQbVNdu362liNiEdJVGvlYVFOFJico8gmxfJmSG2dNiuYw0O/puA0mHJ9l5Bi386hPLaFuQmx+K2s4tc1hUi6ICTJ8h5fyLC4K85+kBVIxb/eBBNAYoMIYLEzZMn/Jkkb+8TFD5jdLNcHm/9/g1HOkxEcYSgCYQcm3BxToc1tdmPN0Fulmj912CyyO9NUYGo06g9DvXtKlAEdTJiPVWHBegJApQNE117BQlhlBYf4/EuROR7jwbQK2i/7Us5NznWpb06IcFENC0MddPEVqMZh05YP0OeIkFKs6pSlNkjM75vCMQd+d7KRry0ah8A4N5Jp8nRYSWievN4fZuDYHFOceeJdJifn+GH/rsdj3y6A1NfXRtQf6FDfkeCfE+S9x4JCs3sMGWjRABhSYcJERRoKgwAclLsUUMlor2AcuRIgk4rd3gW511EkLKS9FE9AT4YUAR1MkSJvMkiwWi22PsEtUME2Rsmun6ZHanxbV4WA2CF2dsfxKgNUV1GSKgIV5n8/qpGmC0SUuLs0VFnVCqVHA3aeqQWgLXiSAiJQIzRh080o6HNhEF5ybiyON/tuinxMciwfT6V4zOc02E9bP/6kw4zWyRssjXg23GsHr958Uds8dCLRonBZJE9Qf5Hgnx3jPY+NsPocXaW4JMtR/H93iqv6wjkRokx1u3Hh6FZooji5LQjHeXJE+RcHg9Y2y+IG10RgRN+oMx2+IE6GxRBnQyl96fVaLZ7grSBNUsEFA0T3YkgPxoaptt8Fieb/BdBwqgp7lQJCRXhGp+h7BTt7a5Z+IKEObqyoRVGswStWoUcPy422U5pkQcnD/DaRdieErN+5kxmi3xh7emUDjvqhwjaW9mAZoMZCToNTstJQlVDG67551p8ttW1CaSSPccbYDRbRaIvA3iSH5Pk/TFGG82Sw1xFZ47VtWDefzbjzrc3+RRLgN0YLSJBcbYS8JCmw2x/k5z2RIJs7xUXEeRh7pq9Ks8qMDtSGdbZoAjqZOi1aojv2Vajpd19ggC7J+dIbYvLDLAyPwybqQnWD05Nk/+VGHIkyMcdISEdRa4Qq2hwudA98/Vu3Pv+Fpfmhe3BlylaICrEfrWJIHHzkZsSC60fnr7MRL382Z84IBvj+mZ6XV9EXYQ5+lhdK8wWCTqNvTFjXoq9X5gvSmyzqIb2TMUHd4zF+f2z0WayYO47m/H69wc8Pm+HrevxwB7JPlMrzhdjd3gTQQk6rXyOvPmChI+xodXkVSwJlMNTAXtEPhyRoNwAyuMF4jknm43yNQJwLY8XOPcK6khlWGeDIqiToVKpFOZoc7tL5AHrl69aZQ1XVzc55o7tkSDP6TARCaoNIB0mR4KyGAkioaUoMxFatQoNbSYH8//PB07g+VX78MHGI/hhb3WH97PbzbgMd4heQdaIiimg8ngAiNGoMSgvGfE6De6/uL/P9UXDRGGOFvvLS42VI0g9Uq0Xy9pmo88L+hZbGu/0glQkxcbgXzeOlAfELvpyt8eGiztsU9AH5bk3jSvxlQ4zWyRFSsc1HaZWq5Co810mr5yw7m1OmcDZGB0OT1CFzc+T0w4RlBIXI98YK5trNrZ6EEFOQ1QpgkhUo+wV1BFjdIxGLd8xOPuCxO8FXsrYhSeoxk8RZDJbcNjmDeibTRFEQotOq5abeoo+PpIkYaGttBwAPtt6rMP7kdNhHnoECbKTY5GbHAuLZJ0JdSSAyjDBf24bgzX3jke/bN9jFOQyeduNh7tKtCS9Fgm2C7qvaJCYSj6sIBWAdQj0nycPQGaiHgazBduP1rl9nlg+0B8RZLsYN7SZXKLTgKPh2V0kSLncmzn68Am7CPLmPxLYjdFa27+hnyJf2QFjtEqlcpsS8zR8NsmpUzdFEIlq7BViFrSZ2y+CAPe+ILNFku+U/IkE+esJKjvZAqNZQmyMGj3acXdDSKCI6IyoyPpiWwW2lNXK1TBf76iQbySc+WlfNf788a9ey8Hrmo1yefmpfsx3EuboLWW1XpuReiIpNsbv7sF9FekwSZLsM/sUlWgqlcovc3RTm0kWkkIEiefLAzttIkmJxSLJ6bBBeb6HgIpIkCQBjW4Ehoju6DRq+XvQGXFB95YOK1NEgrxVoglEP6D4mPCkw8wWSe4M3p4SecBuqFZWiInBskke0mHinNETRKIaeYiqSWmMbt+f0t0g1f+WlKOh1YSUuBivbc/TbJ6gk83+eYKEH6goM9GroZOQYKE0RxvNFjz11W4AwJzz+iE7SY+GVhN+2OdaIWQ0WzB/WQmW/lyK/6wv9bj9PbYBl/mpcX41DLU3TaxTiJLQNA3tlREPlcoaValqbJNT3M6Rpx5+mKO3ldfBIlm9Js6RCW8iqLSmGU0Gsy0q59sHGBujkW/o3EVovPmBBImxvtNhh2vsFXOBpMPCVSJ/orENZosEtar9QsReJm//uzZ6iAQxHUY6FXrFXUh7Z4cJhPG53DbKwmi24Nlv9gIAbj+3yOPdFmCff+ZvJEh4E3yVyRISLJS9gt7dUIaD1U3ITNTh9nP74uLB1o62n2+tcHnel9sq5Dvob3Ye97j9XRX++YEEQ21RlK1H6lDuQZQEC71WI3++D1Y1efQgCRF0zEuvoBJbKbwyCiQoLrB2wN9sK59Xst0WBeqfm+SX+Rvw3jDRHxHkT8PEUod0mO+UlrMnyJ4OC40IEqbozES93+fNGXskSJEO82mMtrYWECLIU8uHrgRFUCckWJ4gQNkw0foFuOyXMpTWNCMzUY+Z4/p4fW66zRPU0GbymFJQsr/S5geiKZqECSFO9lc14jmbuJ834RQk6rWYPDQPgDUlpqygAYA3fzwo/3/DoZMezf8/HzgBABjQwz8RNMQWCTpY3YSymsCM0e2hSDZHNykGFzuLIFuFmJs5UwKlKdqZoT1ToFZZI0nOKTXhB/LHFC3w1jDRW48ggX1+mHtxU9tscCjB9y8d5myMtu6jzWSB2Y13qaOI89geP5DAnSfIUyRIbk3QYkJDm71iLpPpMBKN2NNhlg5VhwGK0RknW9BqNOOFldZOtHPO6yt/0D2RHBsjeytqW3xHgxgJIuEmPzUOCToNjGYJ1Y1t6JMRj+tG9QIAjOydJqfEftxnrxLbUlaLTaW1iNGo0DMtDmaLhNW7K1223dBqxDc7rFGiiwf7NygyPUEnixCD2QKVyl6hFQqEOXp/ZSOO1lovhp7SYV4jQU6maCUJeq08LqSkzDEaJCrDBvrhBxJ4qxBz1+zPGV9do5WVYYDV1+WLZqOTMVoRIQ9FSux4Q/srwwS5brpGy+fPpTrM3ppARIGS9Fq33ci7GhRBnRDZGG1QeIK8pK28ka/oGv32ulJU1LciLyUW14/u5fO5arVKHkx50o9eQfurGAki4UWtVjkYlu+ddBpibDcMarVKnnKtrBJb8tMhAMClQ/MwZZi1I/OKHa4psa+2H0ebyYKirISAIh2n2/oFAdZ0Q3sanfqL8OGsP1QDg9kCjVrl0ndGGKM9zQ+rrG/F0bpWqFR2Y7cznnxB2xU9gvzFWzqrPqB0mPvvJGVlmHWbARij5dlh9n5toTBHy40S22mKBpQDdxWRoDYPniBFn6Du5AcCKII6JXI6zGSWw/j6dkaC8mx3oS1GM579Zg8A4HcTTvH7izktXjRM9B4JOtlkkNcRd6eEhIP+tijF6T1TMHmIY8RGiKAVO46jzWRGZUMrPrV1QJ45rg8mDswBAHy7u8olZfbfknIAwJRh+QHNV1IKiVCmwgB7r6Bfy61pqdxk18aMPXwMURV+oFOzkzzOESy2RYiUIqiyoRVVDW1QqfxPFwLeJ8n7kw5L8pEOc44EtccTpFKpQloh1pFGiQKvJfJ6547RdmM0RRCJepRDVA0dLJHXazWy+a2h1YTeGfG4ekRPv5+f5mfDRJEK65HCwakkvNxyViEuH5aHp6853UWsKFNiP+ytxjvrSmE0SxjeKxWnF6RiaH4KspP0aDKYsXb/Cfl5lQ2tcgrt8mF5AR2PqBADgPwAyuPbQ6EtEiQaZrszYQsRVNdidNv3xpspWlDcy2qO3lpeK3fhFqXxRZkJPlPrSrwZo0XqylslnhwJ8pQOs0WCMhOt310BVYcpIu7i/83G9vUKuu+Drbj8xR/cnnP7BPmOiCDrc5sMZjkC5CmdqDRGUwSRqCdWvgOxdNgYDTgaJedPPFVOF/iDvw0TmQojkaJfdiKem1bstsGgMiX2cclRLP3ZWg4/88xC+fEJA6zRIGWV2Odbj8EiWYVBbx+T0Z0ZrIgEeRtLEwx6JMfKkWPAfTl+UmyMHD1xFw3yZooWFGUmIDlWi1ajRW4eKafCAvADAd6N0aKMv4cXw3Cijz5BIhI02CZGAzNG28VDXAeGqO6uaMB7v5Rhy5E6rD9Y4/J4RybICxL0WvnvKozWHjtGK85ZJUVQ8FiwYAHGjRuH+Ph4pKamujy+ZcsWXHfddSgoKEBcXBwGDBiA5557zmW9NWvWYPjw4dDr9ejXrx8WL17sc99bt27F2WefjdjYWBQUFGDRokVBeEXRgTw2w9SxsRkCcTd6ak4iLjs9sLtafxsm7q+iKZpEJ0IEfbrlKKob25CTrJfL5wHgQltK7JsdlfIMso9LrCmzKQFGgQDrBafIlhIOdTpMrVahj0KkeSrHz5XN0Y4iyGKRsLXMmkrzFglSq1UY1suxVN7eJNF/PxDg3Rh9TB794aWTvS1FX+mh2k2IIFGp51ckyOg4QBXo2CT5d9Ydlv8vxKISuTqsg01lxdBdcS48RoLi7ANhRedwiqAgYDAYMHXqVNxxxx1uH9+4cSOys7OxdOlSbN++HQ8++CAeeOABvPjii/I6Bw8exOTJk3HeeeehpKQEd911F2699VZ89dVXHvdbX1+PCy+8EL1798bGjRvx1FNP4ZFHHsFrr70W9NcYCeJ09hL5NtkY3f4/5ZXF+eiblYC/Xj4YmgCbGPo7RPUAI0EkShEpMcFvx/R2iIaO7ZuBeJ0GFfWt2FZej4PVTdhSVguNWiWX2QfK9DG90TsjHuecktXh4/eF8sbDU+RJmKOPOo3O2F/ViIY2E+JiNDg1x/tn19kXJFeGBWCKBrx7go7aRJC3ijrRmXrP8QaXbt8Gk0V+jWK99pTIA4p0WIAiqMVgxvLN5fLv4jwpHxcG8PZMkFciV4g1tMKimLvmHAlS/i6+q7tDt2gACKk549FHHwUAj5Gbm2++2eH3oqIirF27FsuXL8fcuXMBAK+++ioKCwvx9NNPAwAGDBiAH374Af/4xz8wadIkt9t9++23YTAY8MYbb0Cn02HQoEEoKSnBM888g1mzZgXp1UWOYA1QFZzXPxvn9c9u13P9HaLKSBCJVkRKbPFPh6DTquUSekFsjAbnnJKFL7dXYMXO49DYfEVn9sts993yLWcVysNHQ43oFQQA+anuPUh5toulc58f4Qcakp/is2mfXCFWVovGNpM8syzwSJC9Z40Sk9kil457i6DlpsSiR0osjtW14tfyOowpypAfK69tgSRZBYyonGuPMRpQNEwMsET+061H0dBqglatgkkxVkQgTNFxMRqXUvZAEQ0TK+raHMaQOFeHaTVqJOq1aGwz4YDtu5qRoAhRV1eH9PR0+fe1a9di4sSJDutMmjQJa9eu9biNtWvX4pxzzoFOp3N4zu7du3HypGtXUwBoa2tDfX29w0+04jA7LAieoI7gjyfIaLbIZkRGgkg0Mn10L6Qn6HD7OUXIcHMHfIEtJbZix3FFVVj7okDhRlmN6TMd5hQJkk3RNoHjDZEuO1jdhJ9spvHc5Fi359Mbcs+aNscITWWDdZSEVq3y2cRvmJtqNQDyAOde6fFIibdHnNwNaxVIkiT3AnJMh1mFRGuAkaB31tl8Z7ZmtAermxx6GikbJQZSdegOMWfueH2r7AfSadRuq39FiqzJ9noogiLATz/9hPfee88hWlNRUYGcnByH9XJyclBfX4+WFvfNvTw9RzzmjoULFyIlJUX+KSgo6MhLCSmxuuBVh3UUf0ZnlNY0w2SREBej6XCOm5BQcEpOEjb95QLcc+Fpbh8/r3821Cpg57F6HKhuQmyMGhcOynW7brRRqIi+5nlII+XZukYfdfIEyaZoRW8jT6TG6+RI7zu2eWv+TI53xlMkSAi03JRYn2l7EZVybt4oBqf2yoj3OaxV0Gq0yNV17ozRgUyS33G0HiVltdCqVbj93L7y9+FORUqssqHjPYIEucIT1NDqsUeQwLnijiLIA/fffz9UKpXXn127dgV8INu2bcPll1+Ohx9+GBdeeGHAz+8oDzzwAOrq6uSfsrKysB+Dv8TaBE+LUTlANTKdPdP9GKIqcsxFWQkcnEo6JekJOozsY49QTxyQ47FnTrQxsEcy+mYlYOKAHI/fE7lu0mGtRjN2HbNNjvcjEgTY54h9u8c6lDbQVBjgWK6tpNwm0IRg88YweZ5ZrWxmB+yNEnulx/sc1ipQihz3JfL+R4LEMN5Jg3KRlaSXz48yJRYsUzRgL5OvqGtV9AjyIILi7MvVKiAjoXuIoIA/xffccw9mzpzpdZ2ioqKAtrljxw5MmDABs2bNwp///GeHx3Jzc3H8uGO31uPHjyM5ORlxcR5Cux6eIx5zh16vh17fOf7oyj5BcrPEKI4E2f1ATIWRzssFA3LkcmbRSbozEBujwTd3n+t1HREhOlrXgoZWI77bU43Pth6FySIhM1Eve4Z8UdwrFR9uOiJHTtolguLs1WGSJMkpIXtlmO9jGZKfAo1ahcqGNhyra5WryURlWO8MqzcqJS4GVQ1tqGsxomea+20JP5Beq3aIQAl/kL/psGaDCR/bDNGiI//AvGSs3FUpz1gD7J6gjpqiAWU6rM2jKVqgjASlJ+gDLpLprAQsgrKyspCVFbyKhu3bt+P888/HjBkzsGDBApfHx44di//9738Oy1asWIGxY8d63ObYsWPx4IMPwmg0IiYmRn7OaaedhrQ0D+/0TkScwhMUjD5BHUGIIDFE1d1xCKNdX5qiSSdm0qBcLPpqF9LidTjn1NBXdQUTX96SXFt0paHVhOF/XQGj2R49uXBQjt/elGKniNHAHoH1CALsF2OLZPWniIu2vTLMdyQoTqdB/9wkbLeln5xFUEF6vG1fWlQ1tHk1Rws/ULzTHK1AJ8l/uuUoGtpM6JMRj7E2s7YQicoyeblRYlLHRZCI8FU2tMrdtj2lw5Rl890lFQaE2BNUWlqKkpISlJaWwmw2o6SkBCUlJWhstF4Ut23bhvPOOw8XXngh7r77blRUVKCiogJVVVXyNmbPno0DBw7gj3/8I3bt2oWXX34Zy5Ytw/z58+V1XnzxRUyYMEH+/frrr4dOp8Mtt9yC7du347333sNzzz2Hu+++O5QvN2yISFBTmwnCz9eR6rCOkBzne4jqfjkdxkgQ6bz0yojHR3eeiQ9mj4vYTUeoSNRr5TYBRrOEoswE3H5OEd6fPRZ/u3yw39s5LSdJvklLitWiID3wPkixMWrEaKxfKso0lWiU6K1HkBK7OdrqC5IkyR4JEiLISzm+oNlNo0TAezps2S9leGbFHqzZXSlvWxiirxvVS7YFiDL9vccb5RvaYEyQF4gyd6PZ/to9VZyJcwF0LxEU0qT2Qw89hCVLlsi/FxcXAwBWr16N8ePH44MPPkBVVRWWLl2KpUuXyuv17t0bhw4dAgAUFhbi888/x/z58/Hcc8+hZ8+eeP311x3K46urq7F//37595SUFHz99deYM2cORowYgczMTDz00ENdojwesM8OU35wI/WlrLENUa1pMuBkk1Ee2qeEkSDSVRicH3hko7Pw6m9H4NcjdTjrlMx2V3FqNWoM7ZmCdQdrMLBHcruqm1QqFZJjY3CiyYD6ViPyIEzbtnSY36m5NLy9rlSucKtuNKDZYIZKZa+SS7Fd+L01TBSeIOeJ6p7SYRV1rfjjB1sVr8daFbuvshE6jdphLFHPtDgkxWrR0GrCvspGDMxLlqe+d2RkhkCnVSMjQYcTTQbsr7TejHoaPqtMh3WXHkFAiCNBixcvhiRJLj/jx48HADzyyCNuHxcCSDB+/Hhs3rwZbW1t2L9/v4sn6ZFHHnF5ztChQ/H999+jtbUVR44cwX333Re6FxpmRCRI+cGNlCcIAFK9DFGtaTLIpmkOTiUkehneKw0zxvXpcBuLcX0zAQAj+7TfemD3BdnTVMfaGQnaeqTO2qbDFgnJS4mTDeLeulML3DVKBIA4nb3LspKyk83y+r0z4iFJwL5K643gRYNzHVoGqFQquZnk9qN1sFgkuTosGJEgwC6mhDfTY3VYXPdMh3WO8gbigLJPEGB18vtqZBZK0uN1OIAmtw0TRRQoPzUuoCGKhJDOyezxRSjKSsCEAe1rwArYy+SFj6XFYJZvsvypDgOs88xElGV3RQNKa6yREGWKTjk93RPuhqcCQLyHdJgQa0PyU/De7WNR2dCKTYdP4mB1M6aOdB1OPSjPGjnbcawe5zUbZD9WdpCESE6yHjuOAfttQsx5grwgKZbpMNJJiHUakRFpf4K3honsFE1I90Kv1QQ8g9CZJKcyedEjKF6ncYhYeEOtVmFYQSq+31uNzaUn5dE+vdPt30UpsifIizHaYyRIzA5z6mckDNy2SE52UiwuGtzD4/aV5mjhB8pM1AU0yNobIhLU4GFumECZDguWAOsMdC13Xzch1umOJFKmaIG3IapyjyCmwgghfmKP0Fgv3MpUWCA+I3meWVktDtsiQb0y7KND/EmHCU+QizFaiCAPkaBcPyNWoqHkToUICoYfSOC8LX/6BDESRKIa57CsPiYyjRIFqV4aJh6oZmUYISQwnMVJuVN0xV+KbZPtS8pqkWGLWPdKt4sgv4zRbkZmAIp0mBtjNOBfPyMA6JedCJ1WjYY2E345bK1kC2Znfb9FUDdNhzES1AnpTJEgMUSRpmhCiL84l64fs3WL9jY41R2n2yJBB6qasKvC2v1aKYL8KZH3nQ5zigTVB9bxOUajxmk5SQCA1bsqAQSnUaLAefwG+wQ5QhHUCdGoVXIfDSCylWGAZ0+Q2SLJAwspgggh/uI8P0xulOhnikmQnqBDH1v6S4yN6O02HebZEyQboz2UyLukw9pxrMIXtPu4VaiFMhLkqU9QTnIsEnQa5KXEdnh6fWei+7zSLkZsjAZGs20qcKRFkIdIUPnJFhjNEnRatd9lrYQQ4hyhOVrn/8gMZ4YVpOKQbWZYUqxWToEB/vYJskWCYpw9Qa4l8kazBVWN1j4/gZS4Ow+aDcbwVPu2nNJhHiJBCXot/vf7s6HXajo8vb4zwUhQJ0WZEou0CPI0RPVAtbUyrE9GfLeZQ0MI6TgiQiOiN3KjxHbcTIl+QYA1CqS8wMsGbK/pMGGMdu8JMpgsMNta91c2tEGSrBYF4UHyB+cZa8E0Rmck6KBVfP96G/zbOyMhaP2JOgsUQZ0UZZl8xNNhHiJB9AMRQtqDUpxIkhRwo0QlwhwNOPqBALvYajaYYTRb3D7fUzpM+buoIBOpsJwUvTwawx/65yZDGXwJphBRq1UOJe/KfkCEIqjTEhdFkSDnIaoCuwhiZRghxH+SFNVhdS1GWYgEWh0GAAN6JMvfkb3SHW/IlGZgT2Xyngao6rVqWbiIdYRY65EcmFhL0GtRmGE/tmB6ggD7NHnAc5+g7gpFUCfFIR0W4eowT0NUhQhijyBCSCDIhuVWE47aKsMyEnQulbH+oNOqMcQ2860w0zESpNWo5fSQp4aJzR6qw1QqlZwSExViHRl+KnxBeq3awbcUDITHSKtWRTxzEG3wbHRSYrXREwkSQ1QB4GST/W5KNEosZLdoQkgAKMdZyJVh7TBFC/5y6UDcclYhLh+W77ovW2TEkznang5zjaCIlJhYR44EteNYxUT5nOTYoBuTRWQpMVbbrUzP/sC4WCclVnFXotdGtlkiYB2iWtNkkOf7tBrNckUHPUGEkEAQkSCTRZJH7/g7M8wdwwpSHQzSDvuKi8HRulbP6TAPxmjAtWu0GO/Rox3prLF9MwC4mqSDgUiHeTNFd1d4RjopsYroT6QjQYDrENXSmmZIkjX/HEiVBCGExOs00KhVMFsk7LY1OQxVmw1fDRM9DVAF7GXzLU6RIH9HZigZVpCKFfPPQc+0eN8rB0gORZBHIn/1JO0imkrkAdeGicqZYQy/EkICQaVSyWmqXbIICk3ptog6eUqHeeoYDbimwwIdmeHMKTlJLlVowaC4Vyq0ahVO75ka9G13digLOylxUWSMBoC0eFuvIFs6jOXxhJCOkBwXg5PNRuyrtKbDAu0W7S/yJHk3XaMlSZJnhzkPUAXs38PNBhNMZgsqG9pvjA4lfbMSsfEvF3SrTtD+wjPSSYmmPkGAPRIkGiYetDVKZHk8IaQ9iAiNwda/J3TpMM8NEw1meyNEdxEaER1qNZpR2dAGi2StwMpMiL7ZW8GuOOsqRP7qSdqFMh0WDSLIeYiqiAT1yQx+fpsQ0vVx7mcTqnSYt9EZyuGovtJhwg+UkxwbUKNEElkif/Uk7SLqPEHxjp4ge48gRoIIIYGTrOhsrFGrkJ0UWk+Qu+ow4fWJ0agQ48Z2YE+HmTvsByKRIfJXT9Iuok4EJdgjQXUtRlQ3WsUQI0GEkPYg0lSAtc9NqOYPJnuJBHmrDAMc02GiPL49lWEkctAT1ElReoKiwRitHKJ6yBYFykrSc04NIaRdKCNBoYyuyMZoNx2j7ZVh7i+VyknyHRntQSIHRVAnxXF2WDQ0S7RHglgZRgjpKMkKI2+oKsMAe8foBreRIM+NEpXLmw1mOZ0W7LlfJLREPoRA2kW0GqMb2kzYc9za14Mzwwgh7SVZYYwOVWUY4CMdZnQ/QV4gbkaV6TB6gjoXkb96knbhkA6LAhGkHKK6ubQWACNBhJD2o4wEhScdZoQkSQ6PeWuUCCirw0wd6hZNIkfkr56kXUSbMVo5RLWkrBYARRAhpP04eIJCmQ6ziSCjWUKr0eLwmLfhqYBdHDW2mVDZ0AaAnqDORuSvnqRdRJsIAqxDVAH7MMEiTo8nhLQTZZ+gjkyQ90WCbU4Z4JoSk4eneqgOE+mwwyeaYbZI1kaJidHXKJF4JjquniRgos0TBNh9QQCgVgEF6SyPJ4S0D2U6LD+EniDlnDLnrtHNfqbDymutfqCcEJbyk9AQHVdPEjBxUSiC0hTT4numxUMfBVVrhJDOiRAUmYn6kI988GSOtqfDPFWHWcWTsBJF28ww4huWyHdSHPsERYfYEENUAfqBCCEdIz1Bh7dvHY2UuBioVKGNrnjqGt1i9B4Jcl5OEdT5oAjqpESjJ0gZCaIIIoR0lDFFGWHZj7JCTInoE+TJGB3r5BXKowjqdETH1ZMETDSKIKUniCKIENJZECM66poD8wS5RoJYHt/ZiI6rJwkYZTosajxBFEGEkE6Ip9EZvvoEOS9neXznIzquniRgdBo10hN00GvVITcN+gvTYYSQzognT5CvAarO6TB6gjof9AR1UlQqFd6dNQatRjMS9NHxZxRDVHVadUjb3BNCSDDxVB3ma4CqXquGWgVYbNVhoWzqSEJDdFw9Sbs4NScp0ofgQP/cZBRlJWBUn3T2yiCEdBqSPRmjjd4HqKpUKsTrtGhsM0GjViEriY0SOxsUQSRoJOi1WHn3uSEvZyWEkGAiN0tscfQE+eoTBFhTYo1tJmQn6Xnz1wmhJ4gEFQogQkhnw3c6zLMIEo/RFN05oQgihBDSrXHXJ6isphkV9dbJ8DnJngWOXQTRD9QZoQgihBDSrRHVYcpI0DvrSyFJwNmnZHoVQSJVxsqwzgk9QYQQQro1olliY5sJFosEk0XCsg1lAIDpo3t7fa4on2c6rHPCSBAhhJBujYgESRLQ0GbCl9srcKLJgJxkPSYOyPb63OJeqdCoVTijT3o4DpUEGUaCCCGEdGtiYzTQa9VoM1lQ32LE2z8fBgBMO6MXtBrvsYJ7LzwNd47vFzX92khgMBJECCGk2yMqxDaVnsS6gzXQqFW4blQvn89TqVQUQJ0YiiBCCCHdHlEh9sqa/QCACf2zaXbuBlAEEUII6faIhom7KhoAADeM8W6IJl0DiiBCCCHdHuUg6t4Z8TirX2YEj4aEC4ogQggh3Z5khQi6flQvqDkCo1tAEUQIIaTbI8rkdRo1rh7RM8JHQ8IFRRAhhJBuT16qdezF5KE9kJHIafDdBdb1EUII6fbcMKYXkmK1+M2wvEgfCgkjFEGEEEK6PUmxMawI64aENB22YMECjBs3DvHx8UhNTXV5fMuWLbjuuutQUFCAuLg4DBgwAM8995zLemvWrMHw4cOh1+vRr18/LF682Ot+Dx06BJVK5fLz888/B+mVEUIIIaSzE9JIkMFgwNSpUzF27Fj8+9//dnl848aNyM7OxtKlS1FQUICffvoJs2bNgkajwdy5cwEABw8exOTJkzF79my8/fbbWLlyJW699Vb06NEDkyZN8rr/b775BoMGDZJ/z8jICO4LJIQQQkinRSVJkhTqnSxevBh33XUXamtrfa47Z84c7Ny5E6tWrQIA3Hffffj888+xbds2eZ1p06ahtrYWX375pdttHDp0CIWFhdi8eTOGDRvWrmOur69HSkoK6urqkJyc3K5tEEIIISS8BHL9jrrqsLq6OqSn26fxrl27FhMnTnRYZ9KkSVi7dq3Pbf3mN79BdnY2zjrrLHzyySde121ra0N9fb3DDyGEEEK6LlElgn766Se89957mDVrlrysoqICOTk5Duvl5OSgvr4eLS0tbreTmJiIp59+Gu+//z4+//xznHXWWZgyZYpXIbRw4UKkpKTIPwUFBcF5UYQQQgiJSgIWQffff79b07HyZ9euXQEfyLZt23D55Zfj4YcfxoUXXhjw85VkZmbi7rvvxujRo3HGGWfgiSeewA033ICnnnrK43MeeOAB1NXVyT9lZWUdOgZCCCGERDcBG6PvuecezJw50+s6RUVFAW1zx44dmDBhAmbNmoU///nPDo/l5ubi+PHjDsuOHz+O5ORkxMXF+b2P0aNHY8WKFR4f1+v10OvZIIsQQgjpLgQsgrKyspCVlRW0A9i+fTvOP/98zJgxAwsWLHB5fOzYsfjf//7nsGzFihUYO3ZsQPspKSlBjx49OnSshBBCCOk6hLREvrS0FDU1NSgtLYXZbEZJSQkAoF+/fkhMTMS2bdtw/vnnY9KkSbj77rtRUVEBANBoNLLQmj17Nl588UX88Y9/xM0334xVq1Zh2bJl+Pzzz+X9vPjii/joo4+wcuVKAMCSJUug0+lQXFwMAFi+fDneeOMNvP7666F8uYQQQgjpRIRUBD300ENYsmSJ/LsQJatXr8b48ePxwQcfoKqqCkuXLsXSpUvl9Xr37o1Dhw4BAAoLC/H5559j/vz5eO6559CzZ0+8/vrrDj2CqqursX//fod9//Wvf8Xhw4eh1WrRv39/vPfee7j66qtD+GoJIYQQ0pkIS5+gzgj7BBFCCCGdj07dJ4gQQgghJBxQBBFCCCGkW8Ip8h4QWUJ2jiaEEEI6D+K67Y/bhyLIAw0NDQDAztGEEEJIJ6ShoQEpKSle16Ex2gMWiwVHjx5FUlISVCpVULddX1+PgoIClJWV0XQdBni+wwvPd3jh+Q4vPN/hpT3nW5IkNDQ0IC8vD2q1d9cPI0EeUKvV6NmzZ0j3kZyczA9RGOH5Di883+GF5zu88HyHl0DPt68IkIDGaEIIIYR0SyiCCCGEENItoQiKAHq9Hg8//DAHtoYJnu/wwvMdXni+wwvPd3gJ9fmmMZoQQggh3RJGggghhBDSLaEIIoQQQki3hCKIEEIIId0SiiBCCCGEdEsoggghhBDSLaEICjMvvfQS+vTpg9jYWIwePRrr16+P9CF1CRYuXIgzzjgDSUlJyM7OxpQpU7B7926HdVpbWzFnzhxkZGQgMTERV111FY4fPx6hI+5aPPHEE1CpVLjrrrvkZTzfwaW8vBw33HADMjIyEBcXhyFDhuCXX36RH5ckCQ899BB69OiBuLg4TJw4EXv37o3gEXdezGYz/vKXv6CwsBBxcXHo27cv/vrXvzoM5OT5bj/fffcdLrvsMuTl5UGlUuHjjz92eNyfc1tTU4Pp06cjOTkZqampuOWWW9DY2BjwsVAEhZH33nsPd999Nx5++GFs2rQJp59+OiZNmoTKyspIH1qn59tvv8WcOXPw888/Y8WKFTAajbjwwgvR1NQkrzN//nx8+umneP/99/Htt9/i6NGjuPLKKyN41F2DDRs24J///CeGDh3qsJznO3icPHkSZ555JmJiYvDFF19gx44dePrpp5GWliavs2jRIjz//PN49dVXsW7dOiQkJGDSpElobW2N4JF3Tp588km88sorePHFF7Fz5048+eSTWLRoEV544QV5HZ7v9tPU1ITTTz8dL730ktvH/Tm306dPx/bt27FixQp89tln+O677zBr1qzAD0YiYWPUqFHSnDlz5N/NZrOUl5cnLVy4MIJH1TWprKyUAEjffvutJEmSVFtbK8XExEjvv/++vM7OnTslANLatWsjdZidnoaGBumUU06RVqxYIZ177rnS73//e0mSeL6DzX333SedddZZHh+3WCxSbm6u9NRTT8nLamtrJb1eL/3nP/8JxyF2KSZPnizdfPPNDsuuvPJKafr06ZIk8XwHEwDSRx99JP/uz7ndsWOHBEDasGGDvM4XX3whqVQqqby8PKD9MxIUJgwGAzZu3IiJEyfKy9RqNSZOnIi1a9dG8Mi6JnV1dQCA9PR0AMDGjRthNBodzn///v3Rq1cvnv8OMGfOHEyePNnhvAI838Hmk08+wciRIzF16lRkZ2ejuLgY//rXv+THDx48iIqKCofznZKSgtGjR/N8t4Nx48Zh5cqV2LNnDwBgy5Yt+OGHH3DxxRcD4PkOJf6c27Vr1yI1NRUjR46U15k4cSLUajXWrVsX0P44RT5MVFdXw2w2Iycnx2F5Tk4Odu3aFaGj6ppYLBbcddddOPPMMzF48GAAQEVFBXQ6HVJTUx3WzcnJQUVFRQSOsvPz7rvvYtOmTdiwYYPLYzzfweXAgQN45ZVXcPfdd+NPf/oTNmzYgHnz5kGn02HGjBnyOXX3/cLzHTj3338/6uvr0b9/f2g0GpjNZixYsADTp08HAJ7vEOLPua2oqEB2drbD41qtFunp6QGff4og0uWYM2cOtm3bhh9++CHSh9JlKSsrw+9//3usWLECsbGxkT6cLo/FYsHIkSPx+OOPAwCKi4uxbds2vPrqq5gxY0aEj67rsWzZMrz99tt45513MGjQIJSUlOCuu+5CXl4ez3cXg+mwMJGZmQmNRuNSHXP8+HHk5uZG6Ki6HnPnzsVnn32G1atXo2fPnvLy3NxcGAwG1NbWOqzP898+Nm7ciMrKSgwfPhxarRZarRbffvstnn/+eWi1WuTk5PB8B5EePXpg4MCBDssGDBiA0tJSAJDPKb9fgsMf/vAH3H///Zg2bRqGDBmC3/72t5g/fz4WLlwIgOc7lPhzbnNzc10KikwmE2pqagI+/xRBYUKn02HEiBFYuXKlvMxisWDlypUYO3ZsBI+sayBJEubOnYuPPvoIq1atQmFhocPjI0aMQExMjMP53717N0pLS3n+28GECRPw66+/oqSkRP4ZOXIkpk+fLv+f5zt4nHnmmS4tH/bs2YPevXsDAAoLC5Gbm+twvuvr67Fu3Tqe73bQ3NwMtdrx8qjRaGCxWADwfIcSf87t2LFjUVtbi40bN8rrrFq1ChaLBaNHjw5shx2ydZOAePfddyW9Xi8tXrxY2rFjhzRr1iwpNTVVqqioiPShdXruuOMOKSUlRVqzZo107Ngx+ae5uVleZ/bs2VKvXr2kVatWSb/88os0duxYaezYsRE86q6FsjpMkni+g8n69eslrVYrLViwQNq7d6/09ttvS/Hx8dLSpUvldZ544gkpNTVV+u9//ytt3bpVuvzyy6XCwkKppaUlgkfeOZkxY4aUn58vffbZZ9LBgwel5cuXS5mZmdIf//hHeR2e7/bT0NAgbd68Wdq8ebMEQHrmmWekzZs3S4cPH5Ykyb9ze9FFF0nFxcXSunXrpB9++EE65ZRTpOuuuy7gY6EICjMvvPCC1KtXL0mn00mjRo2Sfv7550gfUpcAgNufN998U16npaVFuvPOO6W0tDQpPj5euuKKK6Rjx45F7qC7GM4iiOc7uHz66afS4MGDJb1eL/Xv31967bXXHB63WCzSX/7yFyknJ0fS6/XShAkTpN27d0foaDs39fX10u9//3upV69eUmxsrFRUVCQ9+OCDUltbm7wOz3f7Wb16tdvv6xkzZkiS5N+5PXHihHTddddJiYmJUnJysnTTTTdJDQ0NAR+LSpIULTAJIYQQQroJ9AQRQgghpFtCEUQIIYSQbglFECGEEEK6JRRBhBBCCOmWUAQRQgghpFtCEUQIIYSQbglFECGEEEK6JRRBhBBCCOmWUAQRQgghpFtCEUQIIYSQbglFECGEEEK6Jf8PO7BG1ztohpsAAAAASUVORK5CYII=",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "pd.DataFrame(FDS.running_mean_last_epoch.numpy()).iloc[:, 7].plot(\n",
-    "    title=\"Running mean bina values for 'Longitude' feature\"\n",
-    ");"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "24a898df",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkEAAAGzCAYAAAAotsMiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAACJcElEQVR4nO3dd3xT5f4H8M9J2iSd6R60pRMou6WsMgQEAQUFB4oXlSFw8YJ7XFAvyFVEREQFEdcFRH8KKG7Zy0FlFyjQQqF075Hupk2e3x/JOW1ouiDJSZrv+/XK69WenJzz5LRJvnme7/N9OMYYAyGEEEKInZGI3QBCCCGEEDFQEEQIIYQQu0RBECGEEELsEgVBhBBCCLFLFAQRQgghxC5REEQIIYQQu0RBECGEEELsEgVBhBBCCLFLFAQRQgghxC5REERExXEcFi1aZPbzHD58GBzH4fDhw2Y/lzW5fv06OI7DO++80+a+r732GjiOs0CrzI/jOLz22mtiN6NFW7duRXR0NBwdHeHh4SF2c6zO6NGjMXr0aIucy1LvDbt370ZMTAwUCgU4jkNZWZlZz0fah4IgG3L+/Hk88MADCA0NhUKhQFBQEO644w6sW7dO7Ka16ujRo3jttdfoRU8IgOTkZMyaNQuRkZH49NNP8cknn5j1fK+99hrCwsIMtoWFhWHy5MlmPa8p5eTk4LXXXkNiYqLYTTGwefPmdn1xKC4uxoMPPggnJyd8+OGH2Lp1K1xcXEzenosXL+K1117D9evXTX7szspB7AaQ9jl69CjGjBmDrl27Yt68eQgICEBmZib+/vtvvP/++3jyySfFbmKLjh49iuXLl2PWrFn0rdeKvfrqq1i8eLHYzej0Dh8+DK1Wi/fffx9RUVFiN8cq7d271+D3nJwcLF++HGFhYYiJiRGnUbfgxIkTqKiowOuvv45x48aZ7TwXL17E8uXLMXr06GaBLzGOgiAbsWLFCiiVSpw4caJZIFFQUCBOo0in4uDgAAcHekswN/71asovBNXV1XB2djbZ8cQmk8nEboJJmeNvbklVVVVm6bmyBjQcZiOuXr2K3r17G30R+fn5GfzO59ns2LEDvXr1gpOTE+Lj43H+/HkAwMcff4yoqCgoFAqMHj3aaNfpjh07EBcXBycnJ/j4+OCRRx5BdnZ2s/0OHjyIkSNHwsXFBR4eHpgyZQouXbok3P/aa6/hxRdfBACEh4eD4zhwHNfsnD/88AP69OkDuVyO3r17Y/fu3c3OlZ2djTlz5sDf31/Y73//+1+z/bKysjB16lS4uLjAz88Pzz77LOrq6prtZwyfF3P58mU88sgjUCqV8PX1xX/+8x8wxpCZmYkpU6bA3d0dAQEBWLNmTbNj1NXVYdmyZYiKioJcLkdISAheeumlZm3YtGkTbr/9dvj5+UEul6NXr1746KOPmh2PH7r4888/MXjwYCgUCkREROCLL75o13PirV27FqGhoXBycsKoUaOQlJRk9Lk3xf8vtfX3SU9Px7/+9S/06NEDTk5O8Pb2xrRp09rslq+vr4eXlxdmz57d7L7y8nIoFAq88MILAAC1Wo2lS5ciLi4OSqUSLi4uGDlyJA4dOtTmc581a5bRb8Yt5UF9+eWXwv+/l5cXpk+fjszMTIN9rly5gvvvvx8BAQFQKBQIDg7G9OnToVKpWmxHWFgYli1bBgDw9fVtlru0YcMG9O7dG3K5HF26dMHChQubDSOPHj0affr0walTp3DbbbfB2dkZL7/8cpvXoKMaGhrw+uuvIzIyEnK5HGFhYXj55Zeb/R935P/z3LlzGDVqFJycnBAcHIw33ngDmzZtavae0DQn6PDhwxg0aBAAYPbs2cJ7yObNm4Xzz5o1q9m5jOUVdeS94dixY5g4cSKUSiWcnZ0xatQo/PXXX+27eDe0Y+bMmQCAQYMGgeM4g/a25zzteX1t3rwZ06ZNAwCMGTNGuE58rlNLeXI3Xj9+iO/IkSP417/+BT8/PwQHBwv379q1S3jPd3Nzw6RJk3DhwoUOXxerwYhNGD9+PHNzc2Pnz59vc18ArF+/fiwkJIS99dZb7K233mJKpZJ17dqVrV+/nvXq1YutWbOGvfrqq0wmk7ExY8YYPH7Tpk0MABs0aBBbu3YtW7x4MXNycmJhYWGstLRU2G/fvn3MwcGBde/enb399tts+fLlzMfHh3l6erK0tDTGGGNnz55lDz/8MAPA1q5dy7Zu3cq2bt3KKisrhbb279+fBQYGstdff5299957LCIigjk7O7OioiLhXHl5eSw4OJiFhISw//73v+yjjz5i99xzj3BcXnV1NevevTtTKBTspZdeYu+99x6Li4tj/fr1YwDYoUOHWr12y5YtYwBYTEwMe/jhh9mGDRvYpEmTGAD27rvvsh49erAnnniCbdiwgQ0fPpwBYEeOHBEer9Fo2Pjx45mzszN75pln2Mcff8wWLVrEHBwc2JQpUwzONWjQIDZr1iy2du1atm7dOjZ+/HgGgK1fv95gv9DQUNajRw/m7+/PXn75ZbZ+/Xo2YMAAxnEcS0pKavX5pKWlMQCsb9++LCwsjK1atYotX76ceXl5MV9fX5aXl9fsuTfV3r/Pjh07WP/+/dnSpUvZJ598wl5++WXm6enJQkNDWVVVVattnDNnDvPw8GB1dXUG27ds2cIAsBMnTjDGGCssLGSBgYHsueeeYx999BF7++23WY8ePZijoyM7c+ZMs3YvW7ZM+H3mzJksNDS02bmNPec33niDcRzHHnroIbZhwwbh/7rp/39dXR0LDw9nXbp0YW+88Qb77LPP2PLly9mgQYPY9evXW3yu33//Pbv33nsZAPbRRx+xrVu3srNnzxq0Zdy4cWzdunVs0aJFTCqVskGDBjG1Wi0cY9SoUSwgIID5+vqyJ598kn388cfshx9+aPGcy5Yta/bcQ0ND2aRJk1p8DH/NALAHHniAffjhh+yxxx5jANjUqVObHas9/59ZWVnMy8uLeXt7s+XLl7N33nmHRUdHs/79+zMAwnsG/xxHjRrFGNO99v/73/8yAGz+/PnCe8jVq1eF88+cObNZ+5seg7GOvTccOHCAyWQyFh8fz9asWcPWrl3L+vXrx2QyGTt27JiwH/9e2Zq9e/ey+fPnMwDsv//9L9u6dSs7evRoh87TntfX1atX2VNPPcUAsJdfflm4Tvxr/MbXRNO/X9Prxz+nXr16sVGjRrF169axt956izHG2BdffME4jmMTJ05k69atY6tWrWJhYWHMw8PD4O9nSygIshF79+5lUqmUSaVSFh8fz1566SW2Z88egzdHHgAml8sN/ik//vhjBoAFBASw8vJyYfuSJUsM3oDUajXz8/Njffr0YTU1NcJ+v/zyCwPAli5dKmyLiYlhfn5+rLi4WNh29uxZJpFI2GOPPSZsW716dbM3uaZtlclkLDU11eAYANi6deuEbY8//jgLDAw0+OBljLHp06czpVLJqqurGWOMvffeewwA2759u7BPVVUVi4qK6lAQNH/+fGFbQ0MDCw4OZhzHCW8GjDFWWlrKnJycDN5Atm7dyiQSCfvjjz8Mjrtx40YGgP3111/CNr7NTU2YMIFFREQYbAsNDWUA2O+//y5sKygoYHK5nD3//POtPh8+CHJycmJZWVnC9mPHjjEA7Nlnn2323Jtq79/H2HNJSEhgANgXX3zRahv37NnDALCff/7ZYPtdd91lcC0aGhqaBUqlpaXM39+fzZkzp1m7byYIun79OpNKpWzFihUG+50/f545ODgI28+cOcMAsB07drT63Izhz1lYWChsKygoYDKZjI0fP55pNBph+/r16xkA9r///U/YNmrUKAaAbdy4scPn5rUVBCUmJjIAbO7cuQbbX3jhBQaAHTx40OBY7fn/fPLJJxnHcQYBa3FxMfPy8mo1CGKMsRMnTjAAbNOmTUafS3uCoPa+N2i1WtatWzc2YcIEptVqhX2rq6tZeHg4u+OOO5qdqy18YMEH9B09T3tfXzt27Gjxfa6jQdCIESNYQ0ODsL2iooJ5eHiwefPmGTw+Ly+PKZXKZtttBQ2H2Yg77rgDCQkJuOeee3D27Fm8/fbbmDBhAoKCgvDTTz8123/s2LEG3f9DhgwBANx///1wc3Nrtv3atWsAgJMnT6KgoAD/+te/oFAohP0mTZqE6Oho/PrrrwCA3NxcJCYmYtasWfDy8hL269evH+644w789ttv7X5u48aNQ2RkpMEx3N3dhTYxxvDdd9/h7rvvBmMMRUVFwm3ChAlQqVQ4ffo0AOC3335DYGAgHnjgAeF4zs7OmD9/frvbAwBz584VfpZKpRg4cCAYY3j88ceF7R4eHujRo4fQTkA3jNizZ09ER0cbtPP2228HAIOhGycnJ+FnlUqFoqIijBo1CteuXWs2pNKrVy+MHDlS+N3X17fZuVszdepUBAUFCb8PHjwYQ4YMadffqa2/z43Ppb6+HsXFxYiKioKHh4fwt2nJ7bffDh8fH2zbtk3YVlpain379uGhhx4StkmlUiFXRKvVoqSkBA0NDRg4cGCb52ivnTt3QqvV4sEHHzT4+wUEBKBbt27C30+pVAIA9uzZg+rq6ls+7/79+6FWq/HMM89AIml8W543bx7c3d2F1x1PLpcbHUI0Ff7/4rnnnjPY/vzzzwNAs/a05/9z9+7diI+PN0hs9vLywowZM0zdfKPa+96QmJiIK1eu4B//+AeKi4uF/4GqqiqMHTsWv//+O7Ra7S23pyPnuZXX182aN28epFKp8Pu+fftQVlaGhx9+2OC1IZVKMWTIkHYNS1sjyoK0IYMGDcLOnTuhVqtx9uxZfP/991i7di0eeOABJCYmolevXsK+Xbt2NXgs/6YdEhJidHtpaSkA3dgzAPTo0aPZ+aOjo/Hnn3+2uV/Pnj2xZ8+edifT3dhWAPD09BTaVFhYiLKyMnzyySctTifmEw/T09MRFRXVLM/DWDs70ialUgmFQgEfH59m24uLi4Xfr1y5gkuXLsHX17fVdgLAX3/9hWXLliEhIaHZB6lKpRL+NsbaAxheo7Z069at2bbu3btj+/btbT62PeeuqanBypUrsWnTJmRnZ4MxJtzXWo4MoEvIvv/++/F///d/qKurg1wux86dO1FfX28QBAHAli1bsGbNGiQnJ6O+vl7YHh4e3ubzaI8rV66AMWb0egGAo6OjcL7nnnsO7777Lr766iuMHDkS99xzj5BH1lEtvZ5kMhkiIiKE+3lBQUFmTR5OT0+HRCJpNnstICAAHh4ezdrTnv+R9PR0xMfHN9vPUjPk2vvecOXKFQAQ8niMUalU8PT0vKX2dOQ8t/L6ulk3vqb49vJf6G7k7u5ulnaYGwVBNkgmk2HQoEEYNGgQunfvjtmzZ2PHjh1CwiUAgwi+qZa2N31RWVpbbeK/DT3yyCMtvmH069fP7G1qz7XTarXo27cv3n33XaP78kHo1atXMXbsWERHR+Pdd99FSEgIZDIZfvvtN6xdu7bZN00x/27tOfeTTz6JTZs24ZlnnkF8fDyUSiU4jsP06dPb9a15+vTp+Pjjj7Fr1y5MnToV27dvR3R0NPr37y/s8+WXX2LWrFmYOnUqXnzxRfj5+UEqlWLlypW4evVqq8dvqZaLRqMx+F2r1YLjOOzatcvo83Z1dRV+XrNmDWbNmoUff/wRe/fuxVNPPYWVK1fi77//NkgkNYemPQPm1N7imWL+f7b2t22pXa3h/19Xr17d4nT8pv8HN6sj57nV11drbnwN8G78H+PPs3XrVgQEBDTb31Znltpmq4lg4MCBAHTDU6YQGhoKAEhJSWkW8aekpAj3N93vRsnJyfDx8RF6gW61CrGvry/c3Nyg0WjarLERGhqKpKQkMMYMzmusneYQGRmJs2fPYuzYsa0+759//hl1dXX46aefDL5Fm6tLmf8W19Tly5dNVkvk22+/xcyZMw1my9XW1ra7QOZtt92GwMBAbNu2DSNGjMDBgwfxyiuvNDtHREQEdu7caXBtmwb/LfH09DTalht7NCIjI8EYQ3h4OLp3797mcfv27Yu+ffvi1VdfxdGjRzF8+HBs3LgRb7zxRpuPbarp6ykiIkLYrlarkZaWZtbaMi21R6vV4sqVK+jZs6ewPT8/H2VlZUJ7O3rM1NTUZtuNbbtRa6+l1v62Ta9le98b+KFfd3d3s173jpynva+vjl4ntVrd7s8Ovr1+fn4W/380J8oJshGHDh0y+q2KH7vv6HBPSwYOHAg/Pz9s3LjRYOrorl27cOnSJUyaNAkAEBgYiJiYGGzZssXghZWUlIS9e/firrvuErbxwdDNVoyWSqW4//778d133zWb1g3ohst4d911F3JycvDtt98K26qrq81elZf34IMPIjs7G59++mmz+2pqalBVVQWg8Zvzjd3amzZtMku7fvjhB4MSB8ePH8exY8dw5513muT4Uqm02f/nunXrWvyWeSOJRIIHHngAP//8M7Zu3YqGhoZmQ2HGrtmxY8eQkJDQ5vEjIyOhUqlw7tw5YVtubi6+//57g/3uu+8+SKVSLF++vNnzYYwJQ5/l5eVoaGgwuL9v376QSCTtLsfQ1Lhx4yCTyfDBBx8YnPfzzz+HSqUSXneWwr9+33vvPYPtfA/nzbRnwoQJSEhIMKj6XFJSgq+++qrNx7b2HhIZGYm///4barVa2PbLL780K2nQ3veGuLg4REZG4p133kFlZWWz8zV9v7kVHTlPe19fbV2n33//3WDbJ5980u7X6IQJE+Du7o4333zTYCjaWHttCfUE2Ygnn3wS1dXVuPfeexEdHQ21Wo2jR49i27ZtCAsLM1mSpKOjI1atWoXZs2dj1KhRePjhh5Gfn4/3338fYWFhePbZZ4V9V69ejTvvvBPx8fF4/PHHUVNTg3Xr1kGpVBrUo4iLiwMAvPLKK5g+fTocHR1x9913d6j41ltvvYVDhw5hyJAhmDdvHnr16oWSkhKcPn0a+/fvR0lJCQBdMt/69evx2GOP4dSpUwgMDMTWrVstVkju0Ucfxfbt27FgwQIcOnQIw4cPh0ajQXJyMrZv3449e/Zg4MCBGD9+PGQyGe6++27885//RGVlJT799FP4+fmZrFevqaioKIwYMQJPPPEE6urq8N5778Hb2xsvvfSSSY4/efJkbN26FUqlEr169UJCQgL2798Pb2/vdh/joYcewrp167Bs2TL07dvXoAeCP8fOnTtx7733YtKkSUhLS8PGjRvRq1cvox8iTU2fPh3//ve/ce+99+Kpp55CdXU1PvroI3Tv3t0gsTQyMhJvvPEGlixZguvXr2Pq1Klwc3NDWloavv/+e8yfPx8vvPACDh48iEWLFmHatGno3r07GhoasHXrViFg7yhfX18sWbIEy5cvx8SJE3HPPfcgJSUFGzZswKBBg/DII490+JhtSU1NNdpjFRsbi0mTJmHmzJn45JNPUFZWhlGjRuH48ePYsmULpk6dijFjxnT4fC+99BK+/PJL3HHHHXjyySfh4uKCzz77DF27dkVJSUmrvRiRkZHw8PDAxo0b4ebmBhcXFwwZMgTh4eGYO3cuvv32W0ycOBEPPvggrl69ii+//NIgmR9o/3uDRCLBZ599hjvvvBO9e/fG7NmzERQUhOzsbBw6dAju7u74+eefO/z8b9SR87T39RUTEwOpVIpVq1ZBpVJBLpcLtcjmzp2LBQsW4P7778cdd9yBs2fPYs+ePc3yHFvi7u6Ojz76CI8++igGDBiA6dOnw9fXFxkZGfj1118xfPhwrF+//pavi8VZcCYauQW7du1ic+bMYdHR0czV1ZXJZDIWFRXFnnzySZafn2+wLwC2cOFCg238VOnVq1cbbD906JDRqb7btm1jsbGxTC6XMy8vLzZjxgyDKda8/fv3s+HDhzMnJyfm7u7O7r77bnbx4sVm+73++ussKCiISSQSg+mwxtrKmPFpr/n5+WzhwoUsJCSEOTo6soCAADZ27Fj2ySefGOyXnp7O7rnnHubs7Mx8fHzY008/zXbv3t2hKfJNpy8zppti7eLi0mz/UaNGsd69extsU6vVbNWqVax3795MLpczT09PFhcXx5YvX85UKpWw308//cT69evHFAqFUMPnf//7X7Ppwi1NZ75xCrAxTf/ua9asYSEhIUwul7ORI0cK9WlufO5NtffvU1paymbPns18fHyYq6srmzBhAktOTm5x+rIxWq2WhYSEMADsjTfeMHr/m2++yUJDQ5lcLmexsbHsl19+MTr9HUamA+/du5f16dOHyWQy1qNHD/bll18afc6MMfbdd9+xESNGMBcXF+bi4sKio6PZwoULWUpKCmOMsWvXrrE5c+awyMhIplAomJeXFxszZgzbv39/m8+zpf8xxnRT4qOjo5mjoyPz9/dnTzzxhEFtLsaM/891FD+t3djt8ccfZ4wxVl9fz5YvX87Cw8OZo6MjCwkJYUuWLGG1tbXNjtXe/88zZ86wkSNHMrlczoKDg9nKlSvZBx98wAAY1Kwy9tgff/yR9erVizk4ODSbLr9mzRoWFBTE5HI5Gz58ODt58qTRY3TkveHMmTPsvvvuY97e3kwul7PQ0FD24IMPsgMHDrTvIjdhbIp8R87TkdfXp59+yiIiIphUKjV4XhqNhv373/9mPj4+zNnZmU2YMIGlpqa2OEXeWFsZ031mTJgwgSmVSqZQKFhkZCSbNWsWO3nyZIevizXgGBMxI5YQQohde+aZZ/Dxxx+jsrLyphKZCbkVlBNECCHEImpqagx+Ly4uxtatWzFixAgKgIgoKCeIEEKIRcTHx2P06NHo2bMn8vPz8fnnn6O8vBz/+c9/xG4asVMUBBFCCLGIu+66C99++y0++eQTcByHAQMG4PPPP8dtt90mdtOInaKcIEIIIYTYJcoJIoQQQohdMlsQtGLFCgwbNgzOzs7w8PAwus9TTz2FuLg4yOXyFsuG79mzB0OHDoWbmxt8fX1x//334/r1662e+/Lly5gyZQp8fHzg7u6OESNG2OziboQQQggxD7PlBKnVakybNg3x8fH4/PPPW9xvzpw5OHbsmEElV15aWhqmTJmC5557Dl999RVUKhWeffZZ3Hfffa2unDt58mR069YNBw8ehJOTE9577z1MnjwZV69eNbrmiTFarRY5OTlwc3O75WUfCCGEEGIZjDFUVFSgS5cukEja6OsxdyGiTZs2MaVS2eo+y5YtY/3792+2fceOHczBwYFpNBph208//cQ4jmNqtdrosQoLCxkA9vvvvwvbysvLGQC2b9++drc7MzOzxWJidKMb3ehGN7rRzbpvmZmZbX7WW/XssLi4OEgkEmzatAmzZs1CZWUltm7dinHjxsHR0dHoY7y9vdGjRw988cUXGDBgAORyOT7++GP4+fkJyzcYU1dXZ7DmD9Pni2dmZsLd3d20T4wQQgghZlFeXo6QkBC4ubm1ua9VB0Hh4eHYu3cvHnzwQfzzn/+ERqNBfHy8sGioMRzHYf/+/cKaPxKJBH5+fti9ezc8PT1bfNzKlSuxfPnyZtvd3d0pCCKEEEJsTHtSWTqUGL148WJwHNfqLTk5+aYbfKO8vDzMmzcPM2fOxIkTJ3DkyBHIZDI88MADRldUB3Q9OAsXLoSfnx/++OMPHD9+HFOnTsXdd9/d6sKUS5YsgUqlEm43rkBMCCGEkM6lQz1Bzz//PGbNmtXqPhEREbfSHgMffvghlEol3n77bWHbl19+iZCQEBw7dgxDhw5t9piDBw/il19+QWlpqdCDs2HDBuzbtw9btmzB4sWLjZ5LLpdDLpebrO2EEEIIsW4dCoJ8fX3h6+trrrY0U11d3Syzm19fRqvVtvgYAM0eJ5FIWnwMIYQQQuyP2eoEZWRkIDExERkZGdBoNEhMTERiYiIqKyuFfVJTU5GYmIi8vDzU1NQI+6jVagDApEmTcOLECfz3v//FlStXcPr0acyePRuhoaGIjY0FABw/fhzR0dHIzs4GoFubxtPTEzNnzsTZs2dx+fJlvPjii0hLS8OkSZPM9XQJIYQQYmvaPWe8g2bOnGl0ytqhQ4eEfUaNGmV0n7S0NGGfr7/+msXGxjIXFxfm6+vL7rnnHnbp0iXh/kOHDjV7zIkTJ9j48eOZl5cXc3NzY0OHDmW//fZbh9qvUqkYAKZSqW72EhBCCCHEwjry+U1rh7WgvLwcSqUSKpWKZocRQgghNqIjn9+0dhghhBBC7BIFQYQQQgixSxQEEUIIIcQuURBECCGEELtEQRAhhBBC7BIFQYQQQgixS1a9gGpndDm/AttPZMLHTY4FoyLFbg4hhBBit6gnyMJyVbX47M80/JiYI3ZTCCGEELtGQZCFeTg5AgBU1WqRW0IIIYTYNwqCLMzDWRcEldXUi9wSQgghxL5REGRhHs4yAEC1WoO6Bo3IrSGEEELsFwVBFuYmd4CE0/2sot4gQgghRDQUBFmYRMJBKeQFURBECCGEiIWCIBHwQ2KUF0QIIYSIh4IgEfA9QWXUE0QIIYSIhoIgEQgzxGiaPCGEECIaCoJE4EE9QYQQQojoKAgSQWNOEPUEEUIIIWKhIEgElBNECCGEiI+CIBFQ1WhCCCFEfBQEiYAPgqhOECGEECIeCoJE4OFEOUGEEEKI2CgIEoHSmXKCCCGEELFRECQCD1o2gxBCCBEdBUEi4KfIV9Q1oF6jFbk1hBBCiH2iIEgE7goH4edymiFGCCGEiIKCIBE4SCVCIETT5AkhhBBxUBAkEqFqNOUFEUIIIaKgIEgkQq0gmiZPCCGEiIKCIJHQ0hmEEEKIuCgIEgkNhxFCCCHioiBIJHytIEqMJoQQQsRBQZBIGtcPo5wgQgghRAwUBImEzwkqpeEwQgghRBQUBIlEyAmi4TBCCCFEFBQEiaRx/TAaDiOEEELEQEGQSPicIOoJIoQQQsRBQZBIhCCIcoIIIYQQUVAQJBKlky4nqLy2HhotE7k1hBBCiP2hIEgk/OwwxoCKWuoNIoQQQiyNgiCRyBwkcJFJAdCQGCGEECIGswZBK1aswLBhw+Ds7AwPDw+j+zz11FOIi4uDXC5HTEyM0X22b9+OmJgYODs7IzQ0FKtXr27z3CUlJZgxYwbc3d3h4eGBxx9/HJWVlbfwbEyPpskTQggh4jFrEKRWqzFt2jQ88cQTre43Z84cPPTQQ0bv27VrF2bMmIEFCxYgKSkJGzZswNq1a7F+/fpWjzljxgxcuHAB+/btwy+//ILff/8d8+fPv+nnYg6NydE0TZ4QQgixNAdzHnz58uUAgM2bN7e4zwcffAAAKCwsxLlz55rdv3XrVkydOhULFiwAAERERGDJkiVYtWoVFi5cCI7jmj3m0qVL2L17N06cOIGBAwcCANatW4e77roL77zzDrp06XKrT80khKUzqCeIEEIIsTirzwmqq6uDQqEw2Obk5ISsrCykp6cbfUxCQgI8PDyEAAgAxo0bB4lEgmPHjrV4nvLycoObuXk40UryhBBCiFisPgiaMGECdu7ciQMHDkCr1eLy5ctYs2YNACA3N9foY/Ly8uDn52ewzcHBAV5eXsjLyzP6mJUrV0KpVAq3kJAQ0z4RI5RUK4gQQggRTYeDoMWLF4PjuFZvycnJJmvgvHnzsGjRIkyePBkymQxDhw7F9OnTdY2XmC6GW7JkCVQqlXDLzMw02bFbwi+dUVZDOUGEEEKIpXU4J+j555/HrFmzWt0nIiLiZtvTDMdxWLVqFd58803k5eXB19cXBw4caPU8AQEBKCgoMNjW0NCAkpISBAQEGH2MXC6HXC43WbvbQ8gJop4gQgghxOI6HAT5+vrC19fXHG1plVQqRVBQEADg66+/Rnx8fIvtiI+PR1lZGU6dOoW4uDgAwMGDB6HVajFkyBCLtbktQk4QJUYTQgghFmfW2WEZGRkoKSlBRkYGNBoNEhMTAQBRUVFwdXUFAKSmpqKyshJ5eXmoqakR9unVqxdkMhmKiorw7bffYvTo0aitrcWmTZuwY8cOHDlyRDjP8ePH8dhjj+HAgQMICgpCz549MXHiRMybNw8bN25EfX09Fi1ahOnTp1vNzDCgMSeolKbIE0IIIRZn1iBo6dKl2LJli/B7bGwsAODQoUMYPXo0AGDu3LkGAQ2/T1paGsLCwgAAW7ZswQsvvADGGOLj43H48GEMHjxYeEx1dTVSUlJQX9/Yo/LVV19h0aJFGDt2LCQSCe6//35hOr614HOCaDiMEEIIsTyOMUardxpRXl4OpVIJlUoFd3d3s5wjJa8CE977HV4uMpz+zx1mOQchhBBiTzry+W31U+Q7s6YVo7W0kjwhhBBiURQEiYhfSV7LgEp1g8itIYQQQuwLBUEiUjhKoXDU/QkoL4gQQgixLAqCREZLZxBCCCHioCBIZEJeEFWNJoQQQiyKgiCR8XlB1BNECCGEWBYFQSLzdKaq0YQQQogYKAgSWeP6YTQcRgghhFgSBUEiUzrTcBghhBAiBgqCREaLqBJCCCHioCBIZB7UE0QIIYSIgoIgkQmLqNIUeUIIIcSiKAgSGeUEEUIIIeKgIEhklBNECCGEiIOCIJE1XUmeMVpJnhBCCLEUCoJExgdB9RqGarVG5NYQQggh9oOCIJE5OUohk+r+DDQkRgghhFgOBUEi4ziuSXI0zRAjhBBCLIWCICsgTJOnGWKEEEKIxVAQZAWE5GgaDiOEEEIshoIgK6DUT5MvqaLhMEIIIcRSKAiyAv7ucgBAfnmtyC0hhBBC7IeD2A0gQBcPJwBAThkFQUQnv7wWa/ddRk29Bj0D3REd4IZege7wdZOD4zixm0cIIZ0CBUFWoIuHAgCQq6oRuSXEGpzJKMU/t55CQUUdAODHxBzhviAPJ3z0yAD0C/YQqXWEENJ50HCYFeii5HuCKAiyd9+dysJDn/yNgoo6dPNzxQvju2Nyv0BE+rpAwgHZZTV46dtzaNBoxW4qIYTYPOoJsgLCcJiqFowxGu6wQw0aLVbtTsanf6QBAMb19MPah2LgpnAU9imurMO4d48gOa8CW/9Ox+zh4WI1lxBCOgXqCbIC/u4KcBygbtCimGaI2aVXf0gSAqBFY6LwyaMDDQIgAPB2lePFCdEAgHf3XkahfriMEELIzaEgyArIHCTwddXNEMul5Gi7U1Fbj+9OZwEA3p8egxcm9IBEYrw38KFBIegbpERFXQNW7U62ZDMJIaTToSDISgTqh8SyKS/I7vx+uQj1GoYIHxdMiQlqdV+phMPyKb0BAN+eysKp9FJLNJEQQjolCoKsRBDNELNb+y/lAwDG9fJv1/4DunriwYHBAIBlPyVBo2VmaxshhHRmFARZiUD9DLFclfHhsMKKOrzy/Xlczq+wZLOImTVotDiUUgAAGBvt1+7HvTQxGu4KByRll+Pr4xnmah4hhHRqFARZiUClrieopeGwL/9Ox1fHMvDk/52hb/6dyKn0UpRV18PD2RFxoZ7tfpyPqxzPj+8BAFjx6yVs+iuN/i8IIaSDKAiyEkH6nKDcFoKg1IJKAEBKfgV26pNoie3jh8Ju7+EHB2nHXo4zhnTFyG4+qKnXYPnPF3H/R0eRnFdujmYSQkinREGQlQhsY+mMq4WVws/v7ruM2nqNRdpFzOvAJd1QWHvzgZpykEqwZfZgvD61D1zlDkjMLMPkD/7E27uT6f+DEELagYIgK8EvnVFQUYv6G6oBa7QM14qqAABuCgfkqmqx5eh1SzeRmNjVwkpcK6qCo5TDyG4+N3UMiYTDo0NDsf+5URjfyx8NWoYNh6/ivf1XTNxaQgjpfCgIshI+LnI4SjloWfPV5LNLa6Bu0ELuIMGrk3oCAD48lApVdb0YTSUmsv+ibihsaIR3s8KIHRWgVOCTxwYK/x+H9cnWhBBCWkZBkJWQSDgEKPlp8oZBED8UFu7jggfiQtDD3w3ltQ3YcDjV4u0kpsMPhd1xE0NhLZnULxAAcKWgEjVqGhIjhJDWUBBkRVpaSJVPio70c4VUwuHfd+pmBW06ep0WXbVRJVVqnEwvAQCM7Wm6ICjAXQFfNzk0WoYLOSqTHZcQQjojCoKsSJcWkqP5nqBIX1cAwJgefhgc7gV1gxbv7rts2UYSkziUXAAtA3oGugszA02B4zj0D1YCAM5mURBECCGtoSDIinRpoWp0YxDkAkD3QbfkTt1Cmt+dzqKlNmzQgWRdPtAdPdtfILG9+gV7AADOZZWZ/NiEENKZUBBkRQJbGA67WqibGcb3BAFAbFdPRPi4gDEgs6Taco0kt6yuQYMjKYUAbm5qfFv66nuCzlNPECGEtMqsQdCKFSswbNgwODs7w8PDw+g+Tz31FOLi4iCXyxETE2N0n+3btyMmJgbOzs4IDQ3F6tWrWz3v9evX8fjjjyM8PBxOTk6IjIzEsmXLoFarb/EZmVeQkeGwkio1Sqp07Y7Q9wTx3BQOAICqugYLtZCYwtGrxahSa+DvLkefLkqTH7+/vifoWlEVVDU0g5AQQlpi1iBIrVZj2rRpeOKJJ1rdb86cOXjooYeM3rdr1y7MmDEDCxYsQFJSEjZs2IC1a9di/fr1LR4vOTkZWq0WH3/8MS5cuIC1a9di48aNePnll2/p+ZhboH44LKfJcNg1/VBYkIcTnGUOBvu7yHW/V1IQZDMaNFqs3p0CAJjYOwASCWfyc3i5yBDsqQuok7KpN4gQQlri0PYuN2/58uUAgM2bN7e4zwcffAAAKCwsxLlz55rdv3XrVkydOhULFiwAAERERGDJkiVYtWoVFi5cCI5r/iEyceJETJw4Ufg9IiICKSkp+Oijj/DOO+8YbUddXR3q6uqE38vLLb/8AD8cVlZdjxq1Bk4yaWM+kJ9rs/1dKQiyOZv+uo6LueVQOjniybHdzHae/sEeyCqtwdmsMgyPurlCjIQQ0tlZfU5QXV0dFAqFwTYnJydkZWUhPT293cdRqVTw8vJq8f6VK1dCqVQKt5CQkJtu881yVzgIgQ3fG9SYD+TSbH9+XxoOsw2ZJdXCbL6X74qGj6vcbOfqp88LOpdJPUGEENISqw+CJkyYgJ07d+LAgQPQarW4fPky1qxZAwDIzc1t1zFSU1Oxbt06/POf/2xxnyVLlkClUgm3zMxMk7S/IziOE1aT55OjhRpBvs17goThsFoKgqwdYwxLf0xCTb0Gg8O98OBA8wbZNEOMEELa1uEgaPHixeA4rtVbcnKyyRo4b948LFq0CJMnT4ZMJsPQoUMxffp0XeMlbTc/OzsbEydOxLRp0zBv3rwW95PL5XB3dze4iaGLsJq8Ljn6xhpBTbkq+OEwqgxs7X49n4tDKYWQSSV4896+RodxTalvsBIcB+SoalFYUdf2AwghxA51OCfo+eefx6xZs1rdJyIi4mbb0wzHcVi1ahXefPNN5OXlwdfXFwcOHGjXeXJycjBmzBgMGzYMn3zyicnaZE58raDsshrU1muE6e+RfjQcZqtUNfVY/vNFAMAToyMRZSS/y9Rc5Q6I9HVFakElzmWVmbQqNSGEdBYdDoJ8fX3h6+trjra0SiqVIigoCADw9ddfIz4+vtV2ZGdnY8yYMYiLi8OmTZva1WtkDfjk6FxVDdKLq6FluqnwvkbyR1xkUgCUGG3t3t6djMKKOkT4uuBfYyItdt5+wUqkFlTibJaKgiBCCDHCrLPDMjIyUFJSgoyMDGg0GiQmJgIAoqKi4Oqq+zacmpqKyspK5OXloaamRtinV69ekMlkKCoqwrfffovRo0ejtrYWmzZtwo4dO3DkyBHhPMePH8djjz2GAwcOICgoCNnZ2Rg9ejRCQ0PxzjvvoLCwUNg3ICDAnE/5lgnDYapag6EwY8MnrvqVxykIsl5ZpdX4v+MZAIA37+0LuYPUYufuH+yBnaezcZ7yggghxCizBkFLly7Fli1bhN9jY2MBAIcOHcLo0aMBAHPnzjUIaPh90tLSEBYWBgDYsmULXnjhBTDGEB8fj8OHD2Pw4MHCY6qrq5GSkoL6el1huH379iE1NRWpqakIDg42aBNjzOTP05S6KBuHw662khQNAK5y3QcqDYdZr29PZYExYFikN4ZGeFv03Hzl6HNZKjDGzJ6HRAghtsasQdDmzZtbrREEAIcPH271fh8fHyQkJLS6z+jRow2Cm1mzZrWZt2StmiZGpwo1gprnAwFULNHaabUMO05mAQAeGmT5kgu9At3hIOFQXKVGdlkNgj2dLd4GQgixZraRKGNHAvQ9QTX1GpxKLwUARLXYE0RBkDX762oRsstq4K5wwITelh+GVThK0SPADYCuN4gQQoghCoKsjMJRCm8XGQAgq1RXK8hYtWiAZodZu20ndLWmpsYGQeFouVygpvh6QWcpL4gQQpqhIMgK8UNiAOAg4dDVy/gwBg2HWa/SKjX2XsgHALMXRmxNf6ocTQghLaIgyArxVaMBINTbGY5S438mvlhivYahroEKJlqTHxKzodZo0buLO/oEmX6l+Pbie4KSslXQaq17UgAhhFgaBUFWqGlPUEszwwDApcmq8lVUNdpqMMaEoTAxe4EAoLu/KxSOElTUNeBiruUXBSaEEGtGQZAV4qtGAy3nAwGAVMLBSZ9rQuuHWY+k7HIk51VA5iDB1JggUdviIJVgZDddUdHFO89RjyEhhDRBQZAVam9PENB0/TAKgqzFtpO64ogTewdA6ewocmuA/07pDQ9nRyRll2P17hSxm0MIIVaDgiArxC+dAQCRvsZrBPGEGWJqCoKsQW29Bj8m5gAQpzaQMYFKJ6x+oD8A4LM/03AopUDkFhFCiHWgIMgKBXs2BkERbfQEuchpOMya7ErKRUVtA0K8nBBv4QrRrbmjlz9mxocCAF7YfhYF5bUit4gQQsRHQZAV8ndX4InRkXhxQg8onVofTqGCidbl57O5AIAHBoRAIrGuZSqW3NUTPQPdUVylxrPbE2m2GCHE7lEQZKX+PTEaC8dEtbkfFUy0HrX1Ghy9WgQAmNjH+hbqVThKse7hWDg5SvFXajE+OnJV7CYRQoioKAiycVQw0Xr8fa0YtfVadFEq0N2/9WFMsUT5ueK1e3oBANbuu4xzVEmaEGLHKAiycTQcZj0OpxQCAEZH+1n1iu0PDgzBnX0C0KBleGZbImrUNG2eEGKfKAiycTQcZh0YYziYrJt1NaaHn8itaR3HcXjz3r7wc5PjWmEVVu66JHaTCCFEFBQE2TgaDrMO14qqkFFSDZlUgmGR1jMrrCWeLjKsnqabNv9FQjoO07R5QogdoiDIxjUOh9GQhpgO6XuBhkR4CYGptRvV3VeYNv/it+dQUqUWuUWEEGJZFATZOBoOsw5CPpCVD4XdaPGdPRHp64LCijq8vPM8GKNp84QQ+0FBkI0ThsOoWKJoquoacCytGAAwpoevyK3pGCeZFO9Pj4WDhMPuC3n45Vyu2E0ihBCLoSDIxgkVo6knSDR/pRahXsMQ6u2McJ/WlzmxRn2ClEJNqrX7LqNBoxW5RYQQYhkUBNk4NwWtHSa2Q/qhsDE9rHtqfGvmjgyHh7MjrhVV4aezOWI3hxBCLIKCIBtHw2HiYowJM6tG29hQWFNuCkfMvy0CAPDBgSvUG0QIsQsUBNk4FxlNkRdTSn4FclW1UDhKMNSKFky9GTPjw+DlIsP14mr8kEi9QYSQzo+CIBvHD4fVNWhRT9/eLe5Qsm4obFikDxSOUpFbc2tc5A4GvUH0/0QI6ewoCLJxTWvS0DR5yzuUwleJtt2hsKYeiw+Ft4sMGSXV+P50ttjNIYQQs6IgyMY5SiWQOej+jDQkZlmlVWqcSi8FYHv1gVriLHPAglGRAIAPDlJvECGkc6MgqBNwEwomUtVoSymvrcecLSeg0TJEB7ghxMtZ7CaZzCNDQ+HjKkdWaQ2+PZXV6r4/nMnGy9+fR0FFrYVaRwghpkNBUCfQuH5YvcgtsQ+qmno8+vlxnMkog9LJEe/o1+DqLJxkUiwY1ZgbVFBuPMD56WwOntmWiP87loEp6//C+SyVJZtJCCG3jIKgTsCF1g+zmLJqNR757BjOZpbB09kR/zdvCPoEKcVulsk9MjQUIV5OyFXVYvonfyNPZRgIHbtWjBe2nwWg64nMVdXigY1H8WMi5RERQmwHBUGdgButH2YRpVVqzPjsGM5nq+DlIsP/zRuK3l06XwAEAApHKb56fCiCPJxwragKD32SgJyyGgBAakEl5m89BbVGi4m9A/DHv8dgTA9f1DVo8fQ3iVi1OxkaLa1BRgixfhQEdQLC0hlUMNFsGGP455encCGnHD6uMnw9byh6BrqL3Syz6urtjG/mD0WIlxPSi6vx0CcJSMwsw6xNx6GqqUdsVw+8Nz0GHs4yfDZzEJ4YrUuo/ujwVbz07TmRW08IIW2jIKgTaBwOoyDIXA5cKsDxtBIoHCX4et5Q9AhwE7tJFhHi5Yxt8+MR6u2MzJIaTP3wL2SV1iDM2xmfPTZQqI0klXD498RovD89BhIO+O50Fs5llYnbeEIIaQMFQZ2AsH4YBUFmodUyvLM3BQAwe3g4uvnbRwDE6+LhhG3z44XFYb1cZNg8ezC8XeXN9p0SE4SpsUEAdIuxEkKINaMgqBOgpTPM66ezOUjOq4C7wgELbosUuzmiCFAqsO2fQ/HU2G74Zv5QhOkDImOeur0bpBIOh1IKcTqj1IKtJISQjqEgqBOg4bBbV61uQLW6+fVTN2jxrr5H45+jIqF0drR006yGn5sCz93RHd3b6AkL83HB/QOoN4gQYv0oCOoEaDjs1tQ1aDB2zREMe+sg/kotMrhv28lMZJRUw8dVjtnDw8RpoA168vZucJBw+ONKEU5cLxG7OYQQYhQFQZ0A9QTdmlPppchV1aKsuh6P/e84Nv+VBsYYatQarDtwBQDw5O1RcJY5tHEkwgvxcsa0gSEAgHf3Um8QIcQ60bt6J0BB0K05mloMQNejVlHbgNd+vojkvAp08XBCQUUdgj2d8PDgriK30vYsuj0K357KRMK1Yhy9WoRhkT5iN4kQQgxQT1AnQGuH3Zo/9UNg/5ncC6/c1RMSDvjmRKaQC/TsuO7CIrWk/YI8nDB9kC54fG/fFTBGBRQJIdaF3tk7AeoJunnltfVCPZsRUT6Yd1sE/jdrkJBn1d3fVZjyTTruX2MiIXOQ4Pj1Ehy5XCh2cwghxAAFQZ2AUDGagqAO+/tqMbQMiPBxQRcPJwDA6B5++HHhcMwZHo4P/zEAUgkncittV6DSCY8ODQUAvPpDEiXvE0KsilmDoBUrVmDYsGFwdnaGh4eH0X2eeuopxMXFQS6XIyYmxug+27dvR0xMDJydnREaGorVq1e3uw11dXWIiYkBx3FITEzs+JOwAW5y3bRt+oDpuKNXdflAw6K8DbZH+Lpi6d297K4wojk8e0d3BHk4Iau0Bm/vTha7OYQQIjBrEKRWqzFt2jQ88cQTre43Z84cPPTQQ0bv27VrF2bMmIEFCxYgKSkJGzZswNq1a7F+/fp2teGll15Cly5dOtx2W8L3BFWrNbRwZQfxU+KHU9Ku2bjKHbDq/n4AgC0J6fj7WrHILSKEEB2zBkHLly/Hs88+i759+7a4zwcffICFCxciIiLC6P1bt27F1KlTsWDBAkRERGDSpElYsmQJVq1a1Wai5a5du7B371688847t/Q8rB2fEwQAVUYK/hHj8strcaWgEhwHxEd6t/0ActNGdPPBw4N1U+Zf+vac0cKUhBBiaVafE1RXVweFQmGwzcnJCVlZWUhPT2/xcfn5+Zg3bx62bt0KZ2fndp2nvLzc4GYr5A4SOEp1eSs0JNZ+R6/qeoH6dFHCw1kmcms6v5fv6okuSgUySqqxek+K2M0hhBDrD4ImTJiAnTt34sCBA9Bqtbh8+TLWrFkDAMjNzTX6GMYYZs2ahQULFmDgwIHtOs/KlSuhVCqFW0hIiMmeg7lxHNc4Q6yWgqD2+vOK8XwgYh5uCkes1A+LbT56HcfTqJI0IURcHQ6CFi9eDI7jWr0lJ5su+XHevHlYtGgRJk+eDJlMhqFDh2L69Om6xkuMN3/dunWoqKjAkiVL2n2eJUuWQKVSCbfMzEyTtN9SaBHVjmGMCT1BI6IoH8hSRnX3xYMDg8EY8O/vzlEOGyFEVB2uGP38889j1qxZre7TUn7PzeA4DqtWrcKbb76JvLw8+Pr64sCBA62e5+DBg0hISIBcLjfYPnDgQMyYMQNbtmxp9hi5XN5sf1vSuH4YFUxsj7SiKuSqaiGTSjAw1Evs5tiVVyb1wq/ncpFWVIXL+RXoGegudpMIIXaqw0GQr68vfH19zdGWVkmlUgQF6YrWff3114iPj2+xHR988AHeeOMN4fecnBxMmDAB27Ztw5AhQyzSXkujgokdw88KGxDqASeZVOTW2BelkyNiunrgr9RinEovpSCIECIas64dlpGRgZKSEmRkZECj0Qh1eqKiouDq6goASE1NRWVlJfLy8lBTUyPs06tXL8hkMhQVFeHbb7/F6NGjUVtbi02bNmHHjh04cuSIcJ7jx4/jsccew4EDBxAUFISuXQ3XeeLPFRkZieDgYHM+ZdFQENQxf+nXC6OhMHHEdfXEX6nFOJ1eikf0xRQJIcTSzBoELV261GDoKTY2FgBw6NAhjB49GgAwd+5cg4CG3yctLQ1hYWEAgC1btuCFF14AYwzx8fE4fPgwBg8eLDymuroaKSkpqK+vN+fTsWqN64dRENQWjZYh4RqfFE1BkBgGhHoCAE5llIrcEkKIPTNrELR582Zs3ry51X0OHz7c6v0+Pj5ISEhodZ/Ro0e3WjMoLCys0y/eSEtntN+FHBVUNfVwkzugX5BS7ObYpdiuuiAovbgahRV18HWz3Xw8QuwZYwwfHkpFkKcT7o21vZEWq58iT9qHhsPajx8KGxLhDQcpvQTEoHRyRHd/3TD1aeoNIsRmJWWX4529l/HstrP47I9rYjenw+gToJNwpeGwdimpUuPzP9MAAKO601CYmOL0Q2IUBBFiuy7mqoSf3/j1Er5IuC5eY24CBUGdhCv1BLWJMYZXvj+Poso6dPNzxbSBtlMQszMaoB8SO51OQRAhtupSbgUAwN9dN6S99McL+Pp4hphN6hAKgjoJqhjdth8Ss7ErKQ8OEg5rH4qBwpGmxouJ7wk6m6WCukErcmsIITcjJU8XBD0/vgfmjggHALz8/Xl8eypLzGa1GwVBnYQwHEYLUxqVU1aDpT9eAAA8PbYb+lBCtOjCfVzg6ewIdYMWF3JUbT+AEGJVGGNIztOts9kr0B2vTOqJmfGhYAx46duzOKqvx2bNKAjqJBqHw6hi9I20WoYXdpxFRW0DYkI88MToSLGbRKCrBs8PiZ2iITFCbE5BRR1Kq+sh4YAoP1dwHIdld/fGpH6B0DLg53M5YjexTRQEdRKNw2H2WyupJVsSruPo1WIoHCV498H+NCPMigyg5GhCbFayfigs3MdFSC+QSDhM7B1gcL81o0+DTqJxdhj1BDWVU1aDt3bpFvR95a6eiPB1FblFpCk+L+hUemmnr+VFSGeTnKsbCou+Yemb6AA3ALp8Ia2VL5JMQVAn4aqgKfLGHLlciLoGLfoFK2l5BivUP9gDUgmH/PI65KhqxW4OIaQD+KToaH83g+3hPi6QSSWoVmuQVVojRtPajYKgTkKoGK1uoG/UTZy4XgIAuK2bLziOE7k15EZOMil6d9F9i6S8IEJsyyU+CLqhJ8hBKkGUn67XnU+ctlYUBHUS/HAYY0C1mobEePwHa1yYp8gtIS2hekGE2J56jRapBfogKMCt2f3Rgbpt1p4XREFQJ+HkKIVE39FBQ2I6BRW1SC+uBsc1ftAS6zMglGaIEWJr0oqqUK9hcJU7INjTqdn9TfOCrBkFQZ0Ex3HCDLEKCoIAAKeu6z5Ue/i7QenkKHJrSEv45OiLueWopjpXhNiES/qk6B4BbkZTDXoE6IbILtFwGLEUWj/M0Al9EDSQhsKsWhelAgHuCmi0DGczqWgiIbaAH+YyNhQGAD31268XVaG23npTNCgI6kRo/TBDp9J1SdEDQ71EbglpDcdxtJgqITYmpY0gyNdNDk9nR2gZkFpQacmmdQgFQZ0IrR/WqFrdgAs5um5Y/gOWWK/Yrh4AgDMZZaK2gxDSPi3VCOJxHIdofkgs13qHxCgI6kRo/bBGiZllaNAyBLgrjCbtEevCr+VmzW+WhNij0ip1s4KHqup6oa5XjxZ6gpreZ83J0RQEdSK0flgjPik6LsyT6gPZgF76WkHZZTUoq1aL3BpCCACczSxD7Ov7sHjnOYPtKfm6oCbIwwnuipYnnfS0gWnyFAR1IjQc1uiEfrr1IBoKswnuCkeEeOl67C7mUG8QIdbgjD5Hb/vJLIN8Pb4AYkv5QDx+hhgFQcQiXPVVo+19dphGy3AmnZ8ZRknRtqKXPrfgIg2JEWIVSqoae2Xf/PWSsBrBpVy+UnTrQVB3f1dwHFBUWYeiyjrzNfQWUBDUifDrh9n77LCUvApU1DXARSZt85sKsR69u+jygqgniBDrUNJkaPpkein2XMgHAKTk8TWCjCdF85xlDgj1ctY/xjp7gygI6kRcaIo8gMap8bFdPeEgpX9xW0E9QYRYl9KqegBAoFIBAFi1Oxl1DRohoOnZji+Z0VY+JEafEJ0IFUvUOZlORRJtEZ8cfaWg0qqLqxFiL4qrdENYT43tBh9XGdKKqrB6dwqq1BrIpBKE+7i0eQx+hliylX65oSCoE6FiiTon+UrRVCTRpgQqFfBwdoRGy3Al33qLqxFiL/ieoBBPZzwzrjsA4LM/0wAAUX6u7eppF9YQy6eeIGJmNBwG5JTVILusBlIJhxh9AT5iGziOQ+8u/JAYLZ9BiNj4nCBPF0dMHxSCSN/Gnp+2kqIb99O9plPyKqC5od6QNaAgqBPx0C8SWlxpv3VW+KGwnoFuQs8YsR18XtAFSo4mRFSMMZTqZ4d5u8jhIJVgyZ09hft7tpEUzevq5QyFowR1DVqkF1eZpa23goKgTiRMPz6bVVqNugb7zKk4dZ3WC7NlfF4QzRAjRFzltQ1o0PfceDjrvmCP7emH0T18wXFAfKR3u44jlXDo4W+9RRMpCOpE/NzkcJFJoWVAenG12M0RxbE0fRBESdE2qVdg4/IZN5bqJ4RYDt8L5CKTQuGoq0HHcRw+eXQg/vr37cJSN+0hJEdTEETMieM4RPq5AgCuFdpfYmlmSTWS8yog4YChEe37lkKsS6SvC2QOElSpNcgosc9AnhBrwOcDebnKDLbLHCTo4tGx9Rj5afJ8fSFrQkFQJxOhHxK7Wmh9Y6/mtudCHgBgUJgXfFzlIreG3AwHqUSYTUJ5QYSIp0SfW+rlLGtjz7ZFU08QsZQIX74nyP6CoL36aqYT+wSI3BJyKxqLJtIMMULE0jgz7NaDIH44LKOk2urq2FEQ1MlE6KcwXiuyr+Gwwoo6nNBXih7fm4IgW9abkqMJER2fE+RlgiDI21UOLxcZGIPVDXNTENTJRPg09gTxi93Zg30X88EY0C9YiaAOjlcT6yLMELPSCrOE2AN+8VRTDIcBQLCn7n05q7TGJMczFQqCOplwHxdwHKCqqTdYAbiz4/OBJlAvkM3rEeAOjgPyy6135WlCOjv+88MUw2EAhC+nWaXUE0TMyEkmRRel7p/NXpKjVTX1OHq1CAAFQZ2Bq9wBYd66YV0aEiNEHKXVfKFE0/YEZVNPEDE3IS/ITqbJH0ouQL2GIcrPFVH6EgHEttGQGCHiKjZxT1CwpzMAGg4jFhDJzxArso+eIH4obCL1AnUatHwGIeIyZWI00CQnqIyGw4iZ2VNPUI1ag8MphQBoKKwzaVw+g6bJEyIGISfIRInRQZQYTSwl0o5qBf1+pRA19RoEeTihT1D7FvQj1q+3vifoWlEVqtXWVVeEkM6uXqNFea3udWeqnCA+Mbqsuh6VVlQriIKgTojvCcooqUa9Ritya8xrT1LjrDCO40RuDTEVP3cFfFzlYAy4lGt9VWYJ6cz4pGgJB7g7OZrkmG4KR2EhVmtKjqYgqBMKcFfAWSZFg5YZXUj157M52KvPo7Fl9Rot9l/SVYme0Ntf5NYQU+sXrFugMTGzTNyGEGJnSqvqAQAezjJIJab7cmmN0+TNFgStWLECw4YNg7OzMzw8PIzu89RTTyEuLg5yuRwxMTFG99m+fTtiYmLg7OyM0NBQrF69ul3n//XXXzFkyBA4OTnB09MTU6dOvbknYoM4jkO4j/G8oOtFVXjy6zNY+H+nbX6Y4e9rxSivbYC3iwwDw7zEbg4xsQFdPQAApzNKxW0IIXamxMRJ0TxhmnyZHfQEqdVqTJs2DU888USr+82ZMwcPPfSQ0ft27dqFGTNmYMGCBUhKSsKGDRuwdu1arF+/vtVjfvfdd3j00Ucxe/ZsnD17Fn/99Rf+8Y9/3PRzsUURLcwQ+/V8LgCgXsOsrnx5R+3WD4Xd0cvfpN9WiHUY0NUTAHAmnYIgQizJ1NWiedY4Td7BXAdevnw5AGDz5s0t7vPBBx8AAAoLC3Hu3Llm92/duhVTp07FggULAAARERFYsmQJVq1ahYULFxrNAWloaMDTTz+N1atX4/HHHxe29+rV61aejs2JbGGG2G/6IAgArhdVIzrANpOJtVqGvRf1Q2G0YGqn1D/EAxIOyFHVIldVg0AlLYdCiCU0Lp5qmnwgnl0Nh5lCXV0dFAqFwTYnJydkZWUhPT3d6GNOnz6N7OxsSCQSxMbGIjAwEHfeeSeSkpLaPFd5ebnBzZYZW00+vbjKoO5KerHtzh47k1mKwoo6uMkdMCzSW+zmEDNwkTsIQfrp9DJxG0OIHWmsESQ36XGtcf0wqw6CJkyYgJ07d+LAgQPQarW4fPky1qxZAwDIzc01+phr164BAF577TW8+uqr+OWXX+Dp6YnRo0ejpKSkxXOtXLkSSqVSuIWEhJj+CVlQBJ8T1GQ47NfzhtfsupGkaVux54KuF2hMtB/kDlKRW0PMZUCoBwDKCyLEkhpzgkzbE8QPh9ns7LDFixeD47hWb8nJySZr3Lx587Bo0SJMnjwZMpkMQ4cOxfTp03UNlxhvularmxL+yiuv4P7770dcXBw2bdoEjuOwY8eOFs+1ZMkSqFQq4ZaZmWmy5yEGfpp8SZVaiOp3ndfl0AwM1eVa2GpPEGOssUo0DYV1anxeEAVBhFiOqQsl8viCicVVaquZmNOhnKDnn38es2bNanWfiIiIW2mPAY7jsGrVKrz55pvIy8uDr68vDhw40Op5AgMDARjmAMnlckRERCAjI6PFc8nlcsjlpu36E5OzzAGBSgVyVbW4VlQJ31oFzmerIOGA+bdF4OTWU0anz9uC5LwKpBdXQ+YgwajuvmI3h5hRnD5gv5Bdjtp6DRSO1OtHiLkJi6e6mjYIUjo5wk3hgIraBmSX1qCbv5tJj38zOhQE+fr6wtfX8h86UqkUQUFBAICvv/4a8fHxLbaDn3KfkpKCESNGAADq6+tx/fp1hIaGWqzN1iDC1wW5qlpcLazCieu6b9JDI7yFD5YcVY1NfrDwvUC3dfOFi9xsuf3ECnT1coa3iwzFVWpcyFEhLpRKIRBibsWV5ukJAnRDYpdyy5FlJUGQ2XKCMjIykJiYiIyMDGg0GiQmJiIxMRGVlY2zlVJTU5GYmIi8vDzU1NQI+6jVuj9AUVERNm7ciOTkZCQmJuLpp5/Gjh078N577wnHOH78OKKjo5GdnQ0AcHd3x4IFC7Bs2TLs3bsXKSkpwjT9adOmmevpWqWmy2fws8Lu6hsILxcZ3OQOYMy6svTba7dQJZoKJHZ2HMchlh8So+RoQiyC7wkydZ0goOlCqtaRF2S2r9FLly7Fli1bhN9jY2MBAIcOHcLo0aMBAHPnzsWRI0ea7ZOWloawsDAAwJYtW/DCCy+AMYb4+HgcPnwYgwcPFh5TXV2NlJQU1NfXC9tWr14NBwcHPProo6ipqcGQIUNw8OBBeHp6muvpWiU+Ofr3y4W4mFsOCde4vESojzOSsstxvagaUX7iR+PtlV5cheS8CkglHMb1pCDIHgwI9cD+S/mUF0SIBTDGzFYsEbC+afJmC4I2b97cao0gADh8+HCr9/v4+CAhIaHVfUaPHg3GmME2R0dHvPPOO3jnnXfa09ROi58mfzFXNy1+cLgXfN10eU+h3i66IMjGkqP5obAh4V7wNMMLlFifuCbJ0YwxWiOOEDOqVmtQ16CbYGTWniArmSFm1VPkya3hZ4jxJvUNFH4O89ZNVbS15Gh+ajzNCrMf/YI94CDhkF9ehxxVrdjNIaRT43uB5A4SOJkhX9TaqkZTENSJdVE6QeGo+xNznGFl5VBvXYBkSz1BBeW1OKVfQmF8LwqC7IWTTIqegbqiiadoCQ1CzEqYGeYiM0uvq7B+GAVBxNwkEg7hProhsUFhXvBza6y+HWaDQRC/TEZMiAcClIo29iadibCYKgVBhJhVMV8jyEzpBnwQVFRZh9p6jVnO0REUBHVyMSEeAID7YoMMtvPDYdmlNVDrx3+tHRVItF8D9GUdzlByNCFmVWrGpGhAVyvIVV/axBqGxCgI6uQW3xmNrY8PxkODDJcB8XWTw8lRCq2NTJO/VliJhKvFAHQz3Ih94StHX8gpt4pvj4R0VuaqFs3jOK5xSMwKpslTENTJKZ0cMbKbb7OxXY7jEGojydGMMbz6QxIatAyje/gi3Mel7QeRTiXY0wm+bnI0aBnOZ6vEbg4hnZY5awTxrGmaPAVBdsxW8oK+P5ONo1eLoXCU4PUpfcRuDhEBx3FCXhAlRxNiPuasEcSzpmnyFATZsVAf6+8JKq1S441fLwEAnhrbDSFeziK3iIiFX+6FkqMJMZ8SMydGA9Y1TZ6CIDtmCz1Bb+1KRkmVGj383TBvpOkW5yW2h88LOpNZ1qxAKiHENEqrdKsveJtzOEyYJi/+F3AKguyYtecEHU8rwbaTmQCAN+/rA0cp/bvas95dlJBKOBRW1CGvnIomEmIOxVV1AMyXGA3QcBixEnxPUGZJNRo01jVNXt2gxcvfnwcAPDy4K60eTuAkk6K7ftXps5mUHE2IOZRW63qCzJsTpPsCXlAhfq0gCoLsWIC7AjIHCRq0DDll1vXNevPRNKQWVMLHVYbFE6PFbg6xEv2DlQCAs1ll4jaEkE5Io2Uos8DsME9nR2FJjhyRp8lTEGTHJBIOofpEY2vLC9qnrw799NhuUDo7itwaYi3664t/nqMgiBCTU9XUQ6tPt/Mw4/uuNdUKoiDIzvFriKVbURCk1TJcyCkHAAyJ8Ba5NcSa9NP3BJ3LUkGrpeRoQkyJnxnmrnAwew6mteQFURBk5/jlM65bUXL0taIqVKs1UDhKEOnrKnZziBXp7u8GuYMEFbUNVtd7SYitExZPdZWb/VyN0+TF/eyhIMjOhfpYX0/QhRxd0muvQHdIJaZfxZjYLkepBL276FaUp7wgQkyruJJfMsP8KQhB1BNErIE19gSdz9IFQX2DlCK3hFgjPi+IZogRYlqWWDKDJ+QEiRwEOYh6diI6fpp8RnE1NFpmFT0vSfqeoN4UBBEj+gd7AKDkaEJMzRJLZvDiI7yxbf5QdPUWdxUACoLsXKBSAUcpB7VGi7zyWmFhO7FotQwXsnVJ0dQTRIzhk6Mv5JSjXqOlIpqEmEipBZbM4Hm7yi2Se9QWevewcw5SCUL0CWrpReLnBWWUVKOirgEyBwmi/CgpmjQX5u0Cd4UD6hq0SMmrELs5hHQaQk+QGatFWxsKgoiwfIY15AWdz9YNhfUMdKdv+MQoiYRDP2FIjPKCCDGVEgvmBFkL+pQhQq0ga5hyzOcD9dHPACLEmMZ6QWXiNoSQTqTUgjlB1oKCIIJIX10QdCVf/KGFpGyaGUbaxvcEJWaWidoOQjqTYgvmBFkLCoIIogN1vS6XcsUNghhjSNInRfehIIi0IkY/Tf5KQSVq1OIuwEhIZ0E5QcQuRQfoVubOK68VukPFkFVaA1VNPRylnLBaOCHGBCgV8HOTQ6NlQnFNQsjNU1XXo1r/hSJAqRC5NZZDQRCBm8IRIV66qfGXcstFawc/FNYjwA0yB/rXJK3jh8TOUnI0IbcsU798hY+rHAr9Cu/2gD5pCACgZ4B+SEzEKcfnKR+IdEB/fXL0WTvKC8osqcbeC3liN4N0QvzyFXwlZ3tBQRABoJuSDojcE6RfOb53FwqCSNv45TPsZYZYbb0G0z/5G/O3nsJfqUViN4d0MvxCpiFe4lZwtjQKgggA8YMgXVI09QSR9uOnyV8vroaqul7k1pjfFwnXkV2m+7Z+MLlA5NaQzoZ6gohd66UPgq7kV6Jeo7X4+XNVtSipUsNBwqFHACVFk7Z5OMuEQp/nssvEbUwH/X65EA98dBSn0kvbtX9ZtRrrD6YKv/95hXqCiGllluh6gigIInYp2NMJrnIHqDVaXCu0fNFEPh+om7+bXSXlkVvDL6ba3mDCWnz2ZxpOppfin1tPIk9V2+b+Hx5KRXltAyJ8XMBxQEp+BQrK234cIe3F9wTxyyjZCwqCCADdUgT8VHkxhsQuZFOlaNJxQyK8AABHrxaL3JL202gZTuuDtqJKNZ746hTqGlqudZRZUo0tR9MBAEvv7oU++py5PykviJgIY0zICaKeIGK3xMwLEmaGBVM+EGm/4ZE+AIAzGaWoVjeI3Jr2Sc4rR2VdA5xlUrgrHHAmowz//flii/uv2ZsCtUaL4VHeGNXdFyO66Z4zBUHEVEqr61GlrxHUxYOCIGKnogP1PUEiTJOnmWHkZoR6OyPIwwn1GoYT121jSIwfuosL9cT7D8eC44CvjmVg+4nMZvsmZavwQ2IOAGDJnT3BcRxGROmDoCtFYIxZruHEZnx3Kgtzt5xAeW37JgzwvUD+7vZVIwigIIg0IVZPUH55LQor6iDhGhO0CWkPjuMwLNIbAHDURnpG+GBtUJgXxvTww3PjugMAXv0hCafSS1CtbhBub/52CQAwNaaLsJRMXKgn5A4SFFTU4UpBpThPglitanUDXvvpAvZfKsCu87ntekzjzDD7ygcCAAexG0CsR3SAGzgOKKyoQ1FlHXxc5RY5b7K+5ynC1xVOMvv6FkJu3fAoH+w4lYW/rlp/EMQYw4m0EgDAwDBPAMDCMVE4l63Cvov5uP+jhGaPkUkleH58D+F3haMUg8O98MeVIvxxpYiWmCEGfj6bg4o63dDwhZz2faG115lhAPUEkSacZQ4I89atKG/J3qAM/Qsw3MfFYucknQffE3Qhp1zUte/aI7usBnnltZBKOGERWImEw5oH+2NgqKfRxyy6PapZAbuRfF7QlUKztpfYnq+OZQg/87XX2mKvM8MA6gkiN+gZ6Ia0oipcyi3HyG6+Fjkn/y2kq51VKiWm4eeuQHd/V1zOr0TCtWLc1TdQ7Ca1iM8H6tPFHc6yxrdfd4UjdiyIR229YY0ujoPRHI0RUb4AknEsrQTqBi2ttUcA6Kqnn2uylt7F3HJotAxSCdfq4zLtdGYYQD1B5AbCGmK5lkuOziimIIjcmmH6WWLWvpzEiev8UJhXs/s4joOTTGpwaylJNTrADd4uMlSrNTiTYRsJ4cT8vvpb1ws0uV8gnGVS1NZrca2w7bwxe84JoiCIGBAjOTqDeoLILRqunzFl7fWCTgpJ0caHvtpLIuGE50xT5QkAqGrq8dNZ3UzCx+LDhEkmSTmtD4k1rREU4kU9QcTO9dQXK0wtqIS6wfzLZzDGhOEwe1u4j5jOkAgvSDggragKOfr1tayNqroeKfm6Hta40OY9QR3F1wv6g5bQIAB+OJONmnoNuvu7YlCYpzCbMCm79S+0RZVq1NZrwXFAoJKCIJNZsWIFhg0bBmdnZ3h4eBjd56mnnkJcXBzkcjliYmKM7rN9+3bExMTA2dkZoaGhWL16dZvnvnz5MqZMmQIfHx+4u7tjxIgROHTo0C08G/vRRamAu8IBDVqGVAtMvy2rrhdmMtjjeDQxDXeFI/rpl9Cw1iGx0xmlYAwI83aGr9utz7zkk6PPZZV1ygVkGWNY+H+nMeOzvy3yhcyWMcbw1TFdVfEZQ0LBcRx667/QtpUczfcCBbgr7DK3zGzPWK1WY9q0aXjiiSda3W/OnDl46KGHjN63a9cuzJgxAwsWLEBSUhI2bNiAtWvXYv369a0ec/LkyWhoaMDBgwdx6tQp9O/fH5MnT0ZeXt5NPx97wXGcRYfE+KEweyzSRUxreJS+XpCVDomdTG85H+hmBCqdEOnrAi0DEq5Z53O+FbmqWvx6Lhd/pRbj7074/EzpZHopLudXwslRinsHBAGA0BN0MaccWm3LRTUz7XhmGGDGIGj58uV49tln0bdv3xb3+eCDD7Bw4UJEREQYvX/r1q2YOnUqFixYgIiICEyaNAlLlizBqlWrWqyUWlRUhCtXrmDx4sXo168funXrhrfeegvV1dVISkoyyXPr7MQIgigfiNyq4U2So62xkvIJE+UDNSVUj07tfFPlz2WVCT/vvUhfYHmpBZXYdzEfGcXVwv/5V3/reoHu6d8F7gpHAECUnytkDhJU1DUIs7+Msdc1w3hWPUW+rq4Ozs6GH45OTk7IyspCeno6wsLCmj3G29sbPXr0wBdffIEBAwZALpfj448/hp+fH+Li4lo9V11dnfB7ebnl18+yFnxC3aU8ywVBlA9EbtWAJpWUrxZWIsrPeooI1jVocDazDIDpeoIAYEQ3X2xJSMfBSwVouFsLB2nnGc5IzGwcxtl/sQCvT2HguNanend2qpp63LfhL5TX6lIIXOUO6BnohrP6azVjaFdhX0epBD0D3HA2S4Wk7HKEehuvwybMDLPT92CrfsVMmDABO3fuxIEDB6DVanH58mWsWbMGAJCba7wcOMdx2L9/P86cOQM3NzcoFAq8++672L17Nzw9W/4GtnLlSiiVSuEWEhJiludkCxp7girM/o2aagQRU1E4SoUqzH9aWbJwUnY56hq08HKRIcKERUFHdvOBl4sMOapa7L7QuXpLmvYE5ZXXCoss27NvjmegvFa3+K5MKkFlXQNOXC+FWqNFv2ClkBfH680nR7cyQ8yeq0UDHQyCFi9eDI7jWr0lJyebrHHz5s3DokWLMHnyZMhkMgwdOhTTp0/XNVxivOmMMSxcuBB+fn74448/cPz4cUydOhV33313i4ETACxZsgQqlUq4ZWY2X8zQXnTzd4WEA0qq1CioqGv7AbeAhsOIKQn1gqwsL+iUPh8oLtTTpL0ZCkcpHosPBQB8+vs1qxwGvBlaLcN5fdG/CF9d0LjvYr6YTRJdvUaLzUevAwBeu7s3Lvx3AnY/MxJrH+qPZ8Z1w7sPxjR7TJ8u/AyxloOgbKFGEAVBbXr++edx6dKlVm8t5ffcDI7jsGrVKlRWViI9PR15eXkYPHgwALR4noMHD+KXX37BN998g+HDh2PAgAHYsGEDnJycsGXLlhbPJZfL4e7ubnCzVwpHqbCERYqZV5SnIIiYEl875+9rxWjQWM+MInPkA/EeHRoKuYMEZ7NUOKZfl8zWXSuqQkVdAxSOEjwxKhIABUG/nstFrqoWPq5yTIntAkepBNEB7rg3NhjPjOuOKD/XZo/pE6T7HLuQU240QNZqGbLK7DsxukM5Qb6+vvD1tcxSCk1JpVIEBeky3r/++mvEx8e32I7qat2H6o09RRKJBFqt9bwpWrsoP1dcLazCtcJK3NbdPH/zeo1WqOlCQRAxhb5BSng4O6Ksuh4n00sxNMJb7CaBMYaTrVSKvlXernI8EBeMr45l4NPfr1nFc75V/FBY7y5K3NHLH1IJh+S8CmQUV6Ort/29VzDG8Okf1wAAM+NDIXdo30za7v5ucJBwKKlSI1dViy4ehr09hZV1UDdoIZVwCFQqTN5uW2C2nKCMjAwkJiYiIyMDGo0GiYmJSExMRGVlY+2Z1NRUJCYmIi8vDzU1NcI+arVuEcSioiJs3LgRycnJSExMxNNPP40dO3bgvffeE45x/PhxREdHIzs7GwAQHx8PT09PzJw5E2fPnsXly5fx4osvIi0tDZMmTTLX0+10Inx13yquFVWZ7Rw5ZTXQMkDuIDFJ3RRCpBIOY6P9AQB7L1hHz8HVwkqUVtdD7iARhidMbe7ICHAccCC5AKkFllvypr2+SLiORz8/1u56RnwSef9gD3g4yzBYHzza6yyxhGvFuJBTDoWjBI8MDW334xSOUqGHyNiQWNMaQZ0pqb4jzPasly5ditjYWCxbtgyVlZWIjY1FbGwsTp48Kewzd+5cxMbG4uOPP8bly5eFfXJycoR9tmzZgoEDB2L48OG4cOECDh8+LAyJAbqen5SUFNTX615cPj4+2L17NyorK3H77bdj4MCB+PPPP/Hjjz+if//+5nq6nU6kPgi62o51Z25WZkljL5C9z/ogpnNHL10QtO9SnlXkyPAVneNCPc1WjC7cxwXj9c/7sz/SzHKOm1VUWYcVv17CH1eK8OPZ7HY95qw+H6h/iC5oFP6mdjokxv9NH4gLhqeLrEOPFSpH5zSf7cu/B9vjchk8s02R37x5MzZv3tzqPocPH271fh8fHyQkJLS6z+jRo5u90Q0cOBB79uxpTzNJC/hkxGuF5usJonwgYg63dfeB3EGCzJIaJOdVCLMdxXLksq6GzygzDSvz5t8WgT0X8rHzdDaeG98dfm7WMbyxNSEddfqKz0dSCvFYfFir+6sbtLior1HWXz/b6Y5e/vjvLxdx4noJSqvUHQ4EbFlqQQUOJheA44A5w8M7/Pg+Xdzx7SngQis9Qfa4cCrPPvu/SJsifXQ9QbmqWlTpl7UwNaoRRMzBWeaAkd10AYfYQ2K19Rqh2vGoHuYNguJCvTCgqwfUGi2+OJpu1nO1V41ag61/N7bl6NVi1NZrWn1MSl4F1A1aKJ0cEarP/wnxckbPQHdoGXAwucCsbbY2n/95HQAwNtpfSFPoiD6tTJPPsvOZYQAFQaQFSmdH+Ljqvm2lmSkviGoEEXMZ31ufFyRyDsmJ6yWordfC312OHv7mL944/zbdTKqtf6ejWm2eLy8d8d3pLJRUqRHi5QR/dzlq6jU4qZ8p15Kz+qTofsFKg2FyfkhM7L+pJRVX1mHn6SwAwLyRHe8FAnR13zgOyC+vQ+ENJU/4StL2OjMMoCCItCLCzHlB1BNEzGVstB8knG5qcFYrSwaY25EU3VDYbd18LZL3dkcvf4R5O0NVU4+tCeL2Bmm0DJ//qctleXx4uDAceDil9Z6cpknRTfE5T79fLmqzN6mz+OZEJuoadIUQB4ff3MxCF7mDUKDzwg29QdQTREEQaUWkPi/oqpnygigniJiLt6tcmI4uZjLt71f0+UBmHgrjSSUc/jUmCgDwwYEryFPVWuS8xuy/lI+0oioonRwxbWAIRnX3A9CYI9WSc0JStIfB9t5d3NFFqUBNvQZ/pVpXRXBz4YdSp8UF31IQzQ+JXWiSHK3RMqFEib0umQFQEERaYc4ZYqrqeqhqdDP67HlmAjEfvudArLygnLIaXM6vhIRrXOjUEh4YEIzYrh6oUmuw4rdLFjvvjT79XVfX5pGhXeEid8CIKB9IOOBKQSWy9R++N6qqa8AV/RT//sGG5QQ4jsM4O5olxhgTlgqJCbm1IpvGKkfnl9eiXsPgIOEQ4G4dSfRioCCItIifIXa1wPRBED8W7eMqh7PMqtfxJTZqfK8AAMBx/YwiS/td3+PRP0RX68ZSJBIOr0/pAwkH/Hw2R5Rek1PppTiZXgqZVIKZ+tlgSmdHDOiq+zDnhwlvlJStgpYBgUoF/Ix8MI+J1vUmHW1jWRSNlllFeYRbkVFSjbLqesikEvQIuLV8st76ytEHkgvw9u5kqKrrhaGwLh5OkErst0QJBUGkRXxPUFpRFbRa076hNA6FUS8QMY+u3s6IDnCDRstEmVEkDIWZeWq8MX2ClEJRvaU/JkHdYNlq+Z/pqxtPje1iEMzw1+LIZeN/D34orF+w8aKSA0M9IeF07x+5KuO9SUWVdRi68gD+ufXUTbffGvDXomeg2y3XlxoU5oWR3XygbtBiw+GrGPn2QWw4nArAvvOBAAqCSCuCPZ0hk0pQ16Btsfv6ZlE+ELGE8SLNKGrQaIUiiWIEQQDw/Pge8HaR4WphlZCgbAnpxVXCivZzRxqu8Ti6h64n56/UYqOBGT8z7MZ8IJ6bwlHIbzl2zfg6afsv5qOwog4HkgtQ12C7CdT80iF9WwgIO8JRKsEXcwbjk0fj0N3fFeW1DTis742z55lhAAVBpBVSCYcwH90LxNTLZ1AQRCxhfG/dkJilZxSdzSpDRW0DPJwd0e+GWU6WonRyxJK7egLQJUnnmPiLTEt+PZ8LxoCR3XzQ/YayAL27uMPbRYbKugaczmg+VV4Iglq5ZkP0s6SOpRkfEuODT42WmbXYq7k19op5mOR4HMdhfO8A7Hr6Nqx9qL+Qi9nHBEGWLaMgiLQqQl800dR5QZk0PZ5YQNMZRX9esVxuDJ/zMiLKR9R8i/sHBGFQmCdq6jV4/ZeLFjknXwdojL7XpymJhBMWZD58Q15QSZVaWMaB7+0xZki4boFYYz1BGi3Dn01yoC7nW986au2h0TIhibmlocGbJZVwuDc2GAeeG43fnhqJR4Z0NenxbQ0FQaRVkX765TOKTBsEUU8QsQT+2y+gK9xnKZZaKqMtHMfhv1P6QCrhsCspz+gimqak1TKcvK4LTgaFGa9rM7oHnxdkGATxvUARvi5QOjm2eI5B4V7gOF3vdEG5YQmAc1llwqxTQFd92halFVWiSq2Bk6MUUTdRJbo9ZA4S9OribvfrNlIQRFrV2BNkum5ljZYhWz8zoas3BUHEvB4cGAKOA3Yl5SGhjVlFNyO1oAIVtY0fvCVVapzTBxu3iRwEAbqKwZP6BgIA/mfm3KArBZUor22As0yKnoHGZzSNiPIBxwGXcsuRrw9iclU1+FJf3LG1oTBAN8zXM0A32+lYmmFv0O+Xdb1ADvreN1vtCTqbqfv/6d3F3W5Xd7cUurqkVZF+pq8VlKuqQYOWQSaVwN9KFnkknVevLu6Yoe/yf+2nC6jXmG6m1I+J2Rj37u+Ie2M/5m45ie/PZGFXki4nJjrADf5WUn9lzgjdkgs/n8tp1ntiSif0vUADunq2+OHt7SpHP/1w17ensvDaTxcw6u3DOKCfwccHbK0ZEmE8L+gP/Yy8yf10x0i20Z6g89mmzQciLaMgiLSKrxVUUFFn8G33VvBDYcFeTpDYcX0KYjkvjO8BT2dHpORX4AsTLSfBGMNHh68C0K18vv9SPp7ddhavfJ8EwHJVotsjJsQDcaGeqNcwgwVNTY0fChsY1npxv1H6fKHVe1Kw+eh1qDVaDA73wjfzhwoFEVtjLC+ovLYeZ/RLbvCz0rJKa1BppgWgzalxlpx9Jy1bAgVBpFXuCkf4uskBwGQzLYSkaDufmkksx8NZhpcmRgMA3tt3GQUVt94bciytBMl5FVA4SrBjQTyeuj1KWGoGaCzWaC3mDNf1Bn11LMNsM+VO6JOiW8oH4o3r2Zg0PTDUE/83dwi2zR+KoRHe7ToPv47WlYJKFFfqFgU9mloMjZYhwtcFfYKUwvvWFRsbEqvXaHFRv7xF31YSxIlpUBBE2sS/sZsqOZqSookYHhoYgv7BSlTUNeCtXcm3fLzNf10HANw3IBiDwrzw3Pge2P/cKOx55jZ8/69hiAu9taUOTG1Cb38EeTihpEqNHxOzTX78nLIaZJfVQCrhENNCnR9ev2APbJo1CF/PG4odC+IxLMqnQwm6Xi4y9NBPvz+uzwvii1Pe1k3XAxetr7Jsa3lBl/MrUNeghZvCAWHeLm0/gNwSCoJIm4TV5E2UHJ2hnwZLQRCxJImEw/IpfcBxwM7T2cLQzc3IKq0WCjDOGhYmbOc4Dj0C3BDb1boCIABwkEowc5iuivTnf6aZfFmJk+m6XqDeXdzhIm97KZwx0X6Ij/S+6dlJjXlBJWCMCcuU3NZdt04bX6MoJc/0y/6Y03l9faC+QUpKF7AACoJIm/jlM0zWE1SsC6aoRhCxtJgQDzw0MAQA8J8fL6DhJpOkt/6dDi0Dhkd5NysIaM0eGtQVzjIpLudX4q9U086UE/KBQlsfCjMVfkjs72vFuF5cjazSGjhKOWFIje8pSskvb/EY1uisiYskktZREETa1LiQ6q33BOWqaoSZD9G3uCggITfjxQk9oHRyxKXc8ptaU6xGrcG2E5kAICwOaiuUTo6YFhcMAPj8z2smPXZjPpBlesH4ICglvwI/JeYA0AVg/ILM3QNssyeIXy7D1EUSiXEUBJE28cW60oqroLnFhVS/PpYBLdOVvg/zofFuYnnernLcP0AXCOxK6viaYj8mZqOsuh7Bnk4Y27PtmUzWZtbwcHAccCilsM3SF+0dMiuvrUdynq7HJc5CQZCfmwIRvi5gDPhMH9A1rcvUTV/eo6iyTkietna19RqhwCMFQZZBQRBpUxcPJ8gcJFA3aIUihzejXqPFN/pv0PwK14SIYVI/3cyt/RfzO7TIJmMMm49eB6DrBRJzSYybFe7jgrHRutlZ7+693GKgcyW/AiNWHcJz2xPbPObp9FIwBoR5O8PPgrW/+KnyFbW6afAju/kI97nIHYS8w8v5ttEbdCm3HA1aBi8XGYI87Ht1d0uhIIi0SSrhEKHvtbmVoon7LuajoKIOPq5yTOhtXdOHiX2JDfFEgLsCFXUNHVpT7O9rumnxTo5SPKjPLbJF/xoTBamEw6/nc7HuYGqz+wsr6jB78wlkl9Xg+zPZKGqjJ4VfL2xgG1PjTW1oROP5fFxl6BXobnA/n69lKzPEzjdZL8zel7OwFAqCSLsIeUG3EAR9qS/S9tCgYMgc6F+PiEci4TCxjy4Q/+18+4fENh/VLTtx34AgKJ1bXt/K2g3o6on/TukNAHh332WDKfM1ag3mbjmBLH2vL2PAoTZyp04I64VZdlYc3xME6JbjuHE2VY8A3ZBYio0EQfxyGZQUbTn0SUTahZ8hdvUmCyZeLazE0avF4Djg4cH2vWoxsQ536Zdn2HcxD+qGtmeJXcwpx96L+QAMp8XbqhlDQjFXv5zGi9+ew6n0Emi0DM9sO4OzWSp4ODvi3tggAMCBSy0HQeoGLRL1lZot3RMUoFQgXN9LbWydtsZp8rYRBJ3PLgMAYVkRYn5tF3MgBI09QXzyY0d99XcGAOD2Hn4IpkrRxAoMDPWEn5scBRV1+OtqEcb08GtxX8YY3vj1IhgDJvULRDcbmhbfmiV39cT14mrsv5SP+V+cwphoP+y5kA+ZVIJPHxsIhYMU35/Jxh9XClHXoIHcQdrsGEk5KtQ1aOHlIhOGzS1p5X198VdqEe7u36XZfT34gol5FWCMWfUQU1VdA1ILdD3tlBRtOdQTRNplYKgXpBIOZzLKhKJk7VWj1uDbU5QQTayLwZDYudxW991/qQBHrxZD5iDBYv3yG52BVMLhg4dj0LuLO4qr1Pj2VBYA4J0H+2NQmBf6BLnD312OKrUGf18zXlyysT6QpyhBxtAIbzw/vgccjSzYGuHjCgcJh4q6BuSqzLdwrCkkZaugZUCAuwJ+VrLwrj2gIIi0S4iXs1AT5bWfL7Rr+ID389kclNc2IMTLyWiXNSFi4YfE9l7Mb3F1eXWDFm/+dgmAbv2tzlbk01nmgM9nDkKA/oP3xQk9cI++V4XjONwerSsDcOBSvtHHH09r33phYpA5SIRebGvPC+IXf43t6iFqO+wNBUGk3Z65oxt8XGW4VlglJIi2x5fHdAnR/xgcapNTiknnNSjMCz6uMqhq6nH0qvEKylv/TkdaURV8XGVYOCbSwi20jAClAr89PRI7FsTjX6MNnyO/2OmBSwXNptPX1mtwMr19K8eLRZghZuV5QYkZZQDQ5rprxLQoCCLt5q5wFFbifn//FRSUt929fPJ6Cc5lqSCTSvDgwGBzN5GQDpFKOKFcw67zzYfEyqrV+ODAFQDAc3f0gJvCdmeEtcXLRYZBYV7NhrSGR/lA4ShBdlkNkm8IJLYcvY6y6noEKhXoY6XJvI3LZ1hvEMQYw+kMXY+aNa4715lREEQ65IEBwYgJ8UCVWtPmStyn0kvw+JaTAHTJpN6ucks0kZAOmaQfEttzIa/ZWmLv7b8CVU09ogPc8NAg260LdCsUjlKMiNIVIWw6JFZapcb6Q7oaQy3l5FiDxuUzrDcIylXVoqCiDlIJh75WGkx2Vtb5X0uslkTCYfk9vXUrcZ9peSXufRfz8Y9Pj0FVU4/Yrh5YOrmXhVtKSPsMDveCl4sMpdX1QvJvaZUa+y7mC7WtXp3Uy66HcvnlQfY3mSq/7mAqKmob0DPQXZhKb434nqArBZW3vOyPufAlBqID3OAkaz4Dj5gPTZEnHdZfvxL3NycysfTHC9j6+GB4uciEbvRvjmfg5e/PQ8uAsdF+WP+PAfTCJlbLQSrBhN4B+Pp4BhbvPIcGDUNek6HesdF+GNFkOQZ7xC+zcTarDAUVtahRa7D17+sAgJfvirbqALGrlzMUjhLU1muRXlyFCH3NM2tyRj8URvlAlkdBELkpL07ogd/O5+Jibjni3tgPd4UDwn1c4OUiw6EU3RT6BwcG4817+8LBSrvJCeFN7heIr49nCFWSAd2H54CuHnj5rp4itsw6+Lkr0C9YiXNZKhxKLsDvl4tQr2G4rbsvRnaz7hmfEgmH7v5uOJelwuX8CqsMghKFmWGUD2RpFASRm+LtKse7D8Zg+S8XkFlSg/LaBpzNUgn3LxoThefHd7fq4mSE8IZFeuOdaf1RrdYN70QHuHXqJOibMa6nP85lqfDx79dwrbAKHAcsudM2aiZ189MFQVfyKzGxj9itMVSv0eKc/r2TeoIsj4IgctPG9fLHuF7+qK3XIL24GmlFlbheXI3oADeMbqX6LiHWhuM4PBBHsxdbM7anH97ddxnX9EvnPDAgGD1vWLDUWgUqdTWQCipaXwhWDCl5Fahr0MJd4SBKxW17R0EQuWUKRyl6BLgJJeoJIZ1Pr0B3dFEqkKOqhcJRgufH9xC7Se3m4yoDABRVWl8QxOcD9Q/xaLYALDE/StYghBDSJo7jMFlfSXr+bZEIUNrO0g6+brq2WmUQRPlAoqKeIEIIIe3y3B3dMa6nPwZZaXXoljT2BKlFbklzfKXoWMoHEgUFQYQQQtpF4SjF4HDrWyOsLT5uukKtRVaWE1RWrca1Il2OFSVFi4OGwwghhHRqPvpq9RV1Dait14jcmkb81Pgwb2d4usjEbYydMmsQtGLFCgwbNgzOzs7w8PBodv/Zs2fx8MMPIyQkBE5OTujZsyfef//9ZvsdPnwYAwYMgFwuR1RUFDZv3tzmuc+dO4eRI0dCoVAgJCQEb7/9tgmeESGEEFvjrnCATF+vzJrygqg+kPjMGgSp1WpMmzYNTzzxhNH7T506BT8/P3z55Ze4cOECXnnlFSxZsgTr168X9klLS8OkSZMwZswYJCYm4plnnsHcuXOxZ8+eFs9bXl6O8ePHIzQ0FKdOncLq1avx2muv4ZNPPjH5cySEEGLdOI4T8oIKrWhI7AytHC86s+YELV++HABa7LmZM2eOwe8RERFISEjAzp07sWjRIgDAxo0bER4ejjVr1gAAevbsiT///BNr167FhAkTjB73q6++glqtxv/+9z/IZDL07t0biYmJePfddzF//nwTPTtCCCG2wsdNjhxVrdUkRzPGmvQEeYjaFntmdTlBKpUKXl6NiXcJCQkYN26cwT4TJkxAQkJCi8dISEjAbbfdBplMZvCYlJQUlJaWGn1MXV0dysvLDW6EEEI6B199XpAlhsO+OZ6Bj49cbXWftKIqqGrqIXOQIDrANopOdkZWFQQdPXoU27ZtM+itycvLg7+/v8F+/v7+KC8vR01NzY2HaPUx/H3GrFy5EkqlUriFhITcylMhhBBiRfjkaHPPECupUmPJ9+exclcyMoqrW9yP7wXqG6SEzMGqPortSoev/OLFi8FxXKu35OTkDjckKSkJU6ZMwbJlyzB+/PgOP/5WLVmyBCqVSrhlZmZavA2EEELMw8fNMlWjj14tAmO6n1MLK1rc7wzVB7IKHc4Jev755zFr1qxW94mIiOjQMS9evIixY8di/vz5ePXVVw3uCwgIQH5+vsG2/Px8uLu7w8nJyejxWnoMf58xcrkccrm8Q+0mhBBiG4SeIDPnBP15pUj4+WpBFW5vYY3Zc1llAHTLZRDxdDgI8vX1ha+vr8kacOHCBdx+++2YOXMmVqxY0ez++Ph4/Pbbbwbb9u3bh/j4+BaPGR8fj1deeQX19fVwdHQUHtOjRw94etJUREIIsTd8EGTO2WGMMfzRNAgqrDS6n1bLcDlfd5+tLELbWZl1IDIjIwOJiYnIyMiARqNBYmIiEhMTUVmp++MnJSVhzJgxGD9+PJ577jnk5eUhLy8PhYWFwjEWLFiAa9eu4aWXXkJycjI2bNiA7du349lnnxX2Wb9+PcaOHSv8/o9//AMymQyPP/44Lly4gG3btuH999/Hc889Z86nSwghxEr5WCAxOr24GtlljbmqLQVB2WU1qKnXQCaVIMzb2WztIW0z6xT5pUuXYsuWLcLvsbGxAIBDhw5h9OjR+Pbbb1FYWIgvv/wSX375pbBfaGgorl+/DgAIDw/Hr7/+imeffRbvv/8+goOD8dlnnxlMjy8qKsLVq42Z+EqlEnv37sXChQsRFxcHHx8fLF26lKbHE0KInfLV5wQVmjEI+jNV1wvk5SJDSZUaVwurjO53OV+XKxTh6wIHKSVFi4ljjE/hIk2Vl5dDqVRCpVLB3Z26KwkhxJapquvR/797AQDJr0+EwlFq8nMs2HoKuy/k4YnRkfjosO6L+Zn/3NFsSYyPDl/Fqt3JuLt/F6x7ONbk7bB3Hfn8phCUEEJIp+fu1Lh0RnGV6ZOjNVqGo1d1PUF39PJHF6UCAHCtqPmQ2BV9T1B3P1eTt4N0DAVBhBBCOj2O4+CtXzrDHLWCzmerUF7bADeFA/oFKRGpD3CuFjQfErtcoAuCuvm7mbwdpGMoCCKEEGIXzJkc/Zc+Hyg+whsOUgkiffVB0A3J0VotQ2qBblt3f+oJEhsFQYQQQuyCORdR/eOKblbzyG4+AIBIXxcAzYOgzNJq1NZrIXOQINTbxeTtIB1DQRAhhBC7YK6eoGp1A06nlwEARnTT1dFr7AkyHA7j6wNF+rpCKuFM2g7ScRQEEUIIsQs+buapGn08rQRqjRZBHk5C3Z8IfRCUUVINdYNW2JefHt+DhsKsAgVBhBBC7AK/krypawXxS2WMiPIBx+l6d/zd5XCRSaHRMmSUNPYG8UEQJUVbBwqCCCGE2AWhJ8jEOUF8kcTh+nwgQDcbjZ8hllrQNAjik6IpCLIGFAQRQgixC3xitClzggor6pCcp+vdGR7pbXDfjTPENFom/Ewzw6wDBUGEEELsgq8ZFlHlCyT2CnSHt/74vBtniKUXV0HdoIXCUYIQT1ozzBpQEEQIIcQu8LPDymsbUNegMckxhXygJkNhvBtniPFDYVF+rpDQzDCrQEEQIYQQu6B0coSDPvgoNsEMMcYYjl4tBgAMj2oeBPEzxK4VVoIx1mS5DMoHshYUBBFCCLELEkmTpTNuyAtSN2ix8chVXC8yvvK7MRkl1cguq4GjlMOgMM9m94d6O0PCARW1DSisrMNlfaVomhlmPSgIIoQQYjd83YwXTNx+MhNv7UrGP7eeglbL2nWsv1J1vUCxIZ5wljk0u1/hKEWIly7352pBVWNPECVFWw0KggghhNgNoWp0heFw2LG0EgBASn4Fdl/Ia9ex+KToYVHeLe7D5wVdzq/ANX1uEE2Ptx4UBBFCCLEbPi0UTDx1vUT4+YMDV9rsDdJqGRL0+UDDIpvnA/H4GWIHkgug1mjh5ChFkIfTTbWdmB4FQYQQQuyGj5Fp8jllNchR1UIq4eAmd0ByXgX2tNEbdLmgAsVVajg5ShET4tHifnxP0FF9QcVu/jQzzJpQEEQIIcRuGCuYeDK9FICu1s/sEeEAgPfb6A3i84EGhXtB5tDyRyk/Q6xBf6xuNDPMqlAQRAghxG4YS4zmh8LiQj3x+PBwoTdo78WWe4MS9PlAN1aJvhE/HMbrEUBJ0daEgiBCCCF2Q0iMblIn6FSGricoLtQTSmdHzB4eBgB4b7/x3qAGjRbHrukCJ2P1gZrycpHBw9lR+J2mx1sXCoIIIYTYjRt7gqrqGnApVzd1faC+1s+cEa33Bp3LVqGirgFKJ0f0DHRv9Xwcxwl5QQDNDLM2FAQRQgixG3xPUFl1PdQNWiRmlkGjZQjycEKgUjdry8NZhln63qD3D6Q26w3iZ4XFR3hD2o4kZ35IzFXugC5KhameCjEBCoIIIYTYDQ8nRyFwKa6qw8nrjUNhTT0+Ihyucgdcyi3H//5KM7jvr9S26wM1xSdHR/m5guNoZpg1oSCIEEKI3ZBIOHi76GeIVahxMl2X2zPwhmUvPJxleHFCDwDAm79dwsHkfABAbb1GmE3WWn2gpib0DkA3P1f8Y3BXkzwHYjoUBBFCCLEr/JBYQUUtzmSUAWjeEwQAj8WH4uHBIdAy4KmvE5GSV4HT6aVQN2jh7y5vNvOrJeE+Ltj33Cg8OCjEZM+BmAYFQYQQQuyKjz45+q/UYlTWNcBV7oDogOYJzhzHYfk9fTA0wguVdQ14fMsJ/HwuF4CuF4iGtmwfBUGEEELsCl8wka8KHdvVo8UEZ5mDBB/NiEOotzOySmvw9fEMAMCwNuoDEdtAQRAhhBC7wk+Tzy6rAQAM6Np8KKwpTxcZPp85CG6KxpXih7VRH4jYBgqCCCGE2BVffU4Q78akaGOi/Fzx4T8GwEHCoW+QkhZB7SQc2t6FEEII6Tx8mgRBEg6IbaMniHdbd18ceWmMQY8QsW30lySEEGJXmgZB0QHucJW3/6OQeoA6FxoOI4QQYld83GTCz+0ZCiOdFwVBhBBC7ErTniBj9YGI/aAgiBBCiF3xdJbBRSYFxwGDwrzEbg4REeUEEUIIsStSCYePHx2IKnUDulCOj12jIIgQQojdGdGN6vwQGg4jhBBCiJ2iIIgQQgghdomCIEIIIYTYJQqCCCGEEGKXKAgihBBCiF0yaxC0YsUKDBs2DM7OzvDw8Gh2/9mzZ/Hwww8jJCQETk5O6NmzJ95///1m+x0+fBgDBgyAXC5HVFQUNm/e3Op5Dx8+jClTpiAwMBAuLi6IiYnBV199ZaJnRQghhJDOwKxBkFqtxrRp0/DEE08Yvf/UqVPw8/PDl19+iQsXLuCVV17BkiVLsH79emGftLQ0TJo0CWPGjEFiYiKeeeYZzJ07F3v27GnxvEePHkW/fv3w3Xff4dy5c5g9ezYee+wx/PLLLyZ/joQQQgixTRxjjJn7JJs3b8YzzzyDsrKyNvdduHAhLl26hIMHDwIA/v3vf+PXX39FUlKSsM/06dNRVlaG3bt3t7sNkyZNgr+/P/73v/+1a//y8nIolUqoVCq4u7u3+zyEEEIIEU9HPr+tLidIpVLBy6uxjHlCQgLGjRtnsM+ECROQkJBwS8e9UV1dHcrLyw1uhBBCCOm8rCoIOnr0KLZt24b58+cL2/Ly8uDv72+wn7+/P8rLy1FTU9Ou427fvh0nTpzA7NmzW9xn5cqVUCqVwi0kJOTmngQhhBBCbEKHg6DFixeD47hWb8nJyR1uSFJSEqZMmYJly5Zh/PjxHX58Sw4dOoTZs2fj008/Re/evVvcb8mSJVCpVMItMzPTZG0ghBBCiPXp8Nphzz//PGbNmtXqPhERER065sWLFzF27FjMnz8fr776qsF9AQEByM/PN9iWn58Pd3d3ODm1vvDdkSNHcPfdd2Pt2rV47LHHWt1XLpdDLpd3qN2EEEIIsV0dDoJ8fX3h6+trsgZcuHABt99+O2bOnIkVK1Y0uz8+Ph6//fabwbZ9+/YhPj6+1eMePnwYkydPxqpVqwyG1wghhBBCADOvIp+RkYGSkhJkZGRAo9EgMTERABAVFQVXV1ckJSXh9ttvx4QJE/Dcc88hLy8PACCVSoVAa8GCBVi/fj1eeuklzJkzBwcPHsT27dvx66+/CudZv349vv/+exw4cACAbghs8uTJePrpp3H//fcLx5XJZK0mRzfFT5qjBGlCCCHEdvCf2+2a/M7MaObMmQxAs9uhQ4cYY4wtW7bM6P2hoaEGxzl06BCLiYlhMpmMRUREsE2bNhncv2zZMoPHtHTeUaNGtbvtmZmZRo9BN7rRjW50oxvdrP+WmZnZ5me9ReoE2SKtVoucnBy4ubmB4ziTHru8vBwhISHIzMykGkQWQNfbsuh6WxZdb8ui621ZN3O9GWOoqKhAly5dIJG0Pv/LrMNhtkwikSA4ONis53B3d6cXkQXR9bYsut6WRdfbsuh6W1ZHr7dSqWzXflZVJ4gQQgghxFIoCCKEEEKIXaIgSARyuRzLli2jukQWQtfbsuh6WxZdb8ui621Z5r7elBhNCCGEELtEPUGEEEIIsUsUBBFCCCHELlEQRAghhBC7REEQIYQQQuwSBUGEEEIIsUsUBFnYhx9+iLCwMCgUCgwZMgTHjx8Xu0mdwsqVKzFo0CC4ubnBz88PU6dORUpKisE+tbW1WLhwIby9veHq6or7778f+fn5IrW4c3nrrbfAcRyeeeYZYRtdb9PKzs7GI488Am9vbzg5OaFv3744efKkcD9jDEuXLkVgYCCcnJwwbtw4XLlyRcQW2y6NRoP//Oc/CA8Ph5OTEyIjI/H6668bLMhJ1/vm/f7777j77rvRpUsXcByHH374weD+9lzbkpISzJgxA+7u7vDw8MDjjz+OysrKDreFgiAL2rZtG5577jksW7YMp0+fRv/+/TFhwgQUFBSI3TSbd+TIESxcuBB///039u3bh/r6eowfPx5VVVXCPs8++yx+/vln7NixA0eOHEFOTg7uu+8+EVvdOZw4cQIff/wx+vXrZ7CdrrfplJaWYvjw4XB0dMSuXbtw8eJFrFmzBp6ensI+b7/9Nj744ANs3LgRx44dg4uLCyZMmIDa2loRW26bVq1ahY8++gjr16/HpUuXsGrVKrz99ttYt26dsA9d75tXVVWF/v3748MPPzR6f3uu7YwZM3DhwgXs27cPv/zyC37//XfMnz+/441p97Lq5JYNHjyYLVy4UPhdo9GwLl26sJUrV4rYqs6poKCAAWBHjhxhjDFWVlbGHB0d2Y4dO4R9Ll26xACwhIQEsZpp8yoqKli3bt3Yvn372KhRo9jTTz/NGKPrbWr//ve/2YgRI1q8X6vVsoCAALZ69WphW1lZGZPL5ezrr7+2RBM7lUmTJrE5c+YYbLvvvvvYjBkzGGN0vU0JAPv++++F39tzbS9evMgAsBMnTgj77Nq1i3Ecx7Kzszt0fuoJshC1Wo1Tp05h3LhxwjaJRIJx48YhISFBxJZ1TiqVCgDg5eUFADh16hTq6+sNrn90dDS6du1K1/8WLFy4EJMmTTK4rgBdb1P76aefMHDgQEybNg1+fn6IjY3Fp59+KtyflpaGvLw8g+utVCoxZMgQut43YdiwYThw4AAuX74MADh79iz+/PNP3HnnnQDoeptTe65tQkICPDw8MHDgQGGfcePGQSKR4NixYx06H60ibyFFRUXQaDTw9/c32O7v74/k5GSRWtU5abVaPPPMMxg+fDj69OkDAMjLy4NMJoOHh4fBvv7+/sjLyxOhlbbvm2++wenTp3HixIlm99H1Nq1r167ho48+wnPPPYeXX34ZJ06cwFNPPQWZTIaZM2cK19TY+wtd745bvHgxysvLER0dDalUCo1GgxUrVmDGjBkAQNfbjNpzbfPy8uDn52dwv4ODA7y8vDp8/SkIIp3OwoULkZSUhD///FPspnRamZmZePrpp7Fv3z4oFAqxm9PpabVaDBw4EG+++SYAIDY2FklJSdi4cSNmzpwpcus6n+3bt+Orr77C//3f/6F3795ITEzEM888gy5dutD17mRoOMxCfHx8IJVKm82Oyc/PR0BAgEit6nwWLVqEX375BYcOHUJwcLCwPSAgAGq1GmVlZQb70/W/OadOnUJBQQEGDBgABwcHODg44MiRI/jggw/g4OAAf39/ut4mFBgYiF69ehls69mzJzIyMgBAuKb0/mIaL774IhYvXozp06ejb9++ePTRR/Hss89i5cqVAOh6m1N7rm1AQECzCUUNDQ0oKSnp8PWnIMhCZDIZ4uLicODAAWGbVqvFgQMHEB8fL2LLOgfGGBYtWoTvv/8eBw8eRHh4uMH9cXFxcHR0NLj+KSkpyMjIoOt/E8aOHYvz588jMTFRuA0cOBAzZswQfqbrbTrDhw9vVvLh8uXLCA0NBQCEh4cjICDA4HqXl5fj2LFjdL1vQnV1NSQSw49HqVQKrVYLgK63ObXn2sbHx6OsrAynTp0S9jl48CC0Wi2GDBnSsRPeUlo36ZBvvvmGyeVytnnzZnbx4kU2f/585uHhwfLy8sRums174oknmFKpZIcPH2a5ubnCrbq6WthnwYIFrGvXruzgwYPs5MmTLD4+nsXHx4vY6s6l6ewwxuh6m9Lx48eZg4MDW7FiBbty5Qr76quvmLOzM/vyyy+Ffd566y3m4eHBfvzxR3bu3Dk2ZcoUFh4ezmpqakRsuW2aOXMmCwoKYr/88gtLS0tjO3fuZD4+Puyll14S9qHrffMqKirYmTNn2JkzZxgA9u6777IzZ86w9PR0xlj7ru3EiRNZbGwsO3bsGPvzzz9Zt27d2MMPP9zhtlAQZGHr1q1jXbt2ZTKZjA0ePJj9/fffYjepUwBg9LZp0yZhn5qaGvavf/2LeXp6MmdnZ3bvvfey3Nxc8RrdydwYBNH1Nq2ff/6Z9enTh8nlchYdHc0++eQTg/u1Wi37z3/+w/z9/ZlcLmdjx45lKSkpIrXWtpWXl7Onn36ade3alSkUChYREcFeeeUVVldXJ+xD1/vmHTp0yOj79cyZMxlj7bu2xcXF7OGHH2aurq7M3d2dzZ49m1VUVHS4LRxjTUpgEkIIIYTYCcoJIoQQQohdoiCIEEIIIXaJgiBCCCGE2CUKggghhBBilygIIoQQQohdoiCIEEIIIXaJgiBCCCGE2CUKggghhBBilygIIoQQQohdoiCIEEIIIXaJgiBCCCGE2KX/B2Vy3yv7CIOEAAAAAElFTkSuQmCC",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "pd.DataFrame(FDS.smoothed_mean_last_epoch.numpy()).iloc[:, 7].plot(\n",
-    "    title=\"Smoothed mean bina values for 'Longitude' feature\"\n",
-    ");"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "id": "7afc8719",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "smoothed_features = FDS._smooth(torch.clone(features).detach(), labels, epoch).numpy()\n",
-    "left_bin_edges_indices = find_bin(\n",
-    "    FDS.bin_edges, labels.squeeze(), ret_value=False\n",
-    ").numpy()\n",
-    "continuous_cols = df.drop(columns=[\"MedHouseVal\"]).columns.values.tolist()\n",
-    "\n",
-    "df_w_bins = df.copy()\n",
-    "df_w_bins[\"MedHouseVal_bins\"] = left_bin_edges_indices\n",
-    "df_smoothed_w_bins = df_w_bins.copy()\n",
-    "df_smoothed_w_bins[continuous_cols] = smoothed_features"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "id": "d3e732b1",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAGzCAYAAADaCpaHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABvIElEQVR4nO3dd3xUVfo/8M/MJDPpk95DINQAoSuCgEAQRFRsgOz+VljF7lrXgq4CrlhQV113bbsKlq/SxLKKItKbCEgChIQSEpIQ0ntPZs7vj8m9mUkmlUzmzuTzfr3yIty5c+fc3CnPnPOc56iEEAJEREREDkpt7wYQERERXQoGM0REROTQGMwQERGRQ2MwQ0RERA6NwQwRERE5NAYzRERE5NAYzBAREZFDYzBDREREDo3BDBERETk0BjOkaDt37oRKpcLOnTt75PGmTp2KqVOndusxP/vsMwwZMgSurq7w9fXt1mP3RmvWrIFKpUJ6erq9m9JCeno6VCoVXn/99W497pkzZzBz5kzo9XqoVCp888033Xp8JVCpVFi+fLn8f2vXuW/fvrjuuut6vnHN2OJ9gi4Ng5leTnrDOHz4sL2b0mFffPEF3nrrLXs3o0NSUlKwePFi9O/fH//5z3/w4Ycf2uRx9u/fj+XLl6OkpMQmxyf7WrRoEY4fP46VK1fis88+w7hx4+zdJKd38uRJLF++XJFBM7XkYu8GELVlypQpqK6uhlarlbd98cUXOHHiBB555BH7NayDdu7cCaPRiLfffhsDBgyw2ePs378fK1aswOLFi9n742Sqq6tx4MABPPvss3jwwQft3Zwe86c//Qm33XYbdDqdXR7/5MmTWLFiBaZOnYq+ffta3Pbzzz/bpU3UOvbMkKKp1Wq4ublBrXbMp2peXh4AOGyAUVlZae8m9Hr5+fkAuvc55AjXVaPRwM3NDSqVqluO153nrNVqLb5gkf055icE9bijR49i9uzZ8PHxgZeXF+Lj4/Hrr79a7CMNWe3btw+PPfYYgoKC4OnpiZtuukl+Q5YYjUYsX74c4eHh8PDwwLRp03Dy5En07dsXixcvlvdrnjMzdepU/PDDDzh//jxUKhVUKpX8ram1XIrW8m4+/PBD9O/fH+7u7rj88suxZ88eq+deW1uLZcuWYcCAAdDpdIiKisKTTz6J2traNv9mffv2xbJlywAAQUFBLXICfvzxR0yePBmenp7w9vbGnDlzkJSUZHGMY8eOYfHixYiJiYGbmxtCQ0Nxxx13oLCwUN5n+fLleOKJJwAA/fr1k/8u6enpcg7HmjVrWrSveXuWL18OlUqFkydP4g9/+AP8/PwwadIk+fbPP/8cY8eOhbu7O/z9/XHbbbchMzOzzb/Bxo0boVKpsGvXrha3ffDBB1CpVDhx4kSHz7U1zc9F0vz5BAAlJSV45JFHEBUVBZ1OhwEDBuDVV1+F0Wi02G/t2rUYO3YsvL294ePjg7i4OLz99tvttkXy5ptvIjo6Gu7u7rjqqqvk8zSXkpKCW2+9Ff7+/nBzc8O4cePw3XffybcvX74c0dHRAIAnnnjC4vkOdO51uWvXLtx///0IDg5GZGSkfHtHnoetKSkpwaOPPoq+fftCp9MhMjISt99+OwoKCgAAdXV1eP755zF27Fjo9Xp4enpi8uTJ2LFjR7vHbis36ueff8aoUaPg5uaGoUOHYtOmTR0+5/Pnz+P+++/H4MGD4e7ujoCAAMybN8/icdasWYN58+YBAKZNmya/pszfh5rnzOTl5eHOO+9ESEgI3NzcMHLkSHzyyScW+5jnVEnvPzqdDpdddhkOHTrU7t+EWsdhJmpXUlISJk+eDB8fHzz55JNwdXXFBx98gKlTp2LXrl0YP368xf5/+ctf4Ofnh2XLliE9PR1vvfUWHnzwQaxbt07eZ+nSpVi1ahWuv/56zJo1C4mJiZg1axZqamrabMuzzz6L0tJSZGVl4c033wQAeHl5dfqcPvroI9xzzz2YOHEiHnnkEZw7dw433HAD/P39ERUVJe9nNBpxww03YO/evbj77rsRGxuL48eP480338Tp06fbTMR866238Omnn+Lrr7/Ge++9By8vL4wYMQKAKSl40aJFmDVrFl599VVUVVXhvffew6RJk3D06FH5A2vr1q04d+4c/vznPyM0NBRJSUn48MMPkZSUhF9//RUqlQo333wzTp8+jS+//BJvvvkmAgMDAZgCqOZBZEfMmzcPAwcOxEsvvQQhBABg5cqVeO655zB//nwsWbIE+fn5eOeddzBlyhQcPXq01V6DOXPmwMvLC+vXr8dVV11lcdu6deswbNgwDB8+vMPneqmqqqpw1VVX4cKFC7jnnnvQp08f7N+/H0uXLsXFixflXKytW7di4cKFiI+Px6uvvgoASE5Oxr59+/Dwww+3+ziffvopysvL8cADD6CmpgZvv/02pk+fjuPHjyMkJASA6XV15ZVXIiIiAk8//TQ8PT2xfv163Hjjjfjqq69w00034eabb4avry8effRRLFy4ENdee638fO/s6/L+++9HUFAQnn/+ebmXoqPPQ2sqKiowefJkJCcn44477sCYMWNQUFCA7777DllZWQgMDERZWRn++9//YuHChbjrrrtQXl6Ojz76CLNmzcJvv/2GUaNGdfIKmpKhFyxYgHvvvReLFi3C6tWrMW/ePPz000+4+uqr2z3nQ4cOYf/+/bjtttsQGRmJ9PR0vPfee5g6dSpOnjwJDw8PTJkyBQ899BD++c9/4plnnkFsbCwAyP82V11djalTp+Ls2bN48MEH0a9fP2zYsAGLFy9GSUlJi+fMF198gfLyctxzzz1QqVRYtWoVbr75Zpw7dw6urq6d/psQAEG92urVqwUAcejQoVb3ufHGG4VWqxWpqanytuzsbOHt7S2mTJnS4lgzZswQRqNR3v7oo48KjUYjSkpKhBBC5OTkCBcXF3HjjTdaPM7y5csFALFo0SJ5244dOwQAsWPHDnnbnDlzRHR0dKvnkpaWZrG9+THq6upEcHCwGDVqlKitrZX3+/DDDwUAcdVVV8nbPvvsM6FWq8WePXssjvn+++8LAGLfvn1W/2aSZcuWCQAiPz9f3lZeXi58fX3FXXfdZbFvTk6O0Ov1FturqqpaHPPLL78UAMTu3bvlba+99prVc09LSxMAxOrVq1scB4BYtmxZi7YuXLjQYr/09HSh0WjEypUrLbYfP35cuLi4tNje3MKFC0VwcLBoaGiQt128eFGo1WrxwgsvdPpcrV3n5uciiY6Otng+/f3vfxeenp7i9OnTFvs9/fTTQqPRiIyMDCGEEA8//LDw8fGxaHNHSH9vd3d3kZWVJW8/ePCgACAeffRReVt8fLyIi4sTNTU18jaj0SgmTpwoBg4c2OKYr732msVjdfZ1OWnSJIvz6czz0Jrnn39eABCbNm1qcZv0+m9oaLB4jQkhRHFxsQgJCRF33HGHxfbm19DadY6OjhYAxFdffSVvKy0tFWFhYWL06NHtnrMQ1p9nBw4cEADEp59+Km/bsGFDi/ceyVVXXWXxPvHWW28JAOLzzz+Xt9XV1YkJEyYILy8vUVZWJoRoupYBAQGiqKhI3vfbb78VAMT//ve/Fo9FHcNhJmqTwWDAzz//jBtvvBExMTHy9rCwMPzhD3/A3r17UVZWZnGfu+++2+Jb9OTJk2EwGHD+/HkAwLZt29DQ0ID777/f4n5/+ctfbHgmTQ4fPoy8vDzce++9FuPeixcvhl6vt9h3w4YNiI2NxZAhQ1BQUCD/TJ8+HQA61F3e3NatW1FSUoKFCxdaHFOj0WD8+PEWx3R3d5d/r6mpQUFBAa644goAwO+//97px+6Ie++91+L/mzZtgtFoxPz58y3aGxoaioEDB7b7N1iwYAHy8vIshvk2btwIo9GIBQsWyNt64lw3bNiAyZMnw8/Pz+JcZsyYAYPBgN27dwMw5adUVlZi69atXXqcG2+8EREREfL/L7/8cowfPx6bN28GABQVFWH79u2YP38+ysvL5XYUFhZi1qxZOHPmDC5cuNDq8bvyurzrrrug0Wjk/3fmeWjNV199hZEjR+Kmm25qcZv0+tdoNPJrzGg0oqioCA0NDRg3blyXr2l4eLjFY/r4+OD222/H0aNHkZOT0+Y5A5bPs/r6ehQWFmLAgAHw9fXtcps2b96M0NBQLFy4UN7m6uqKhx56CBUVFS2GWRcsWAA/Pz/5/5MnTwYAnDt3rkuPTxxmonbk5+ejqqoKgwcPbnFbbGwsjEYjMjMzMWzYMHl7nz59LPaTXrTFxcUAIAc1zWf3+Pv7W7zAbUV6/IEDB1psd3V1tfhgAExd2snJyQgKCrJ6LCnBtzPOnDkDAHJA1JyPj4/8e1FREVasWIG1a9e2eKzS0tJOP3ZH9OvXz+L/Z86cgRCixd9L0l63+DXXXAO9Xo9169YhPj4egGmIadSoURg0aJC8X0+c65kzZ3Ds2LF2r+f999+P9evXY/bs2YiIiMDMmTMxf/58XHPNNR16HGt/q0GDBmH9+vUAgLNnz0IIgeeeew7PPfdcq20xD4jMdeV1ae26Ah17HlqTmpqKW265pc19AOCTTz7BG2+8gZSUFNTX17fano4aMGBAiyFH6XmUnp6O0NDQNh+juroaL7/8MlavXo0LFy7IQ6lA159n58+fx8CBA1tMVJCGpaT3HEl775HUeQxmqNs1/yYkMX/TsIXWcioMBkOXj2k0GhEXF4d//OMfVm83z6/pzDEBU76C+RuvxMWl6WU5f/587N+/H0888QRGjRoFLy8vGI1GXHPNNS0SVq3pyt/E/Jur1F6VSoUff/zR6rVtL2dJp9PhxhtvxNdff413330Xubm52LdvH1566SWL/S71XK1pfp5GoxFXX301nnzySav7Sx+KwcHBSEhIwJYtW/Djjz/ixx9/xOrVq3H77be3SOrsCul8/vrXv2LWrFlW9+nuqfzWrivQsedhV33++edYvHgxbrzxRjzxxBMIDg6GRqPByy+/jNTU1Es+fnuanzNg6gFevXo1HnnkEUyYMEEuRHjbbbd1+XnWWfZ6j3RmDGaoTUFBQfDw8MCpU6da3JaSkgK1Wt3pD3RpdsbZs2ctvjkVFhZ26JtJax/Q0reb5oXjmn8rkh7/zJkzFt9K6+vrkZaWhpEjR8rb+vfvj8TERMTHx3fbFNH+/fsDMH1gzpgxo9X9iouLsW3bNqxYsQLPP/+8vF36Rm3uUv8m7bVXCIF+/fpZ9KR0xoIFC/DJJ59g27ZtSE5OhhDCYoipM+dqjZ+fX4tzrKurw8WLF1ucS0VFRZt/d4lWq8X111+P66+/HkajEffffz8++OADPPfcc+0GGtbaffr0aTmhVuoBdHV17VBbmuuO12VHn4dt3d/aDC1zGzduRExMDDZt2mTxHJVm+XWF1KtlfrzTp08DQJsJy+ZtWrRoEd544w15W01NTYvnT2de79HR0Th27BiMRqNF70xKSop8O9kWc2aoTRqNBjNnzsS3335rMXUxNzcXX3zxBSZNmtRud3Rz8fHxcHFxwXvvvWex/V//+leH7u/p6Wm1O1h6c5byHgDTN/PmVXfHjRuHoKAgvP/++6irq5O3r1mzpsUb2vz583HhwgX85z//afF41dXVXapdMWvWLPj4+OCll16y6HaXSDOQpG9vzb+tWat+7OnpCaBl0OLj44PAwECLvwkAvPvuux1u78033wyNRoMVK1a0aIsQokNTp2fMmAF/f3+sW7cO69atw+WXX24RyHbmXK3p379/i3P88MMPW/TMzJ8/HwcOHMCWLVtaHKOkpAQNDQ0A0OKc1Gq1PBOtvSn5APDNN99Y5Lz89ttvOHjwIGbPng3AFEBMnToVH3zwQYuAC0C7s9C643XZ0edha2655RYkJibi66+/bnGbdB2tXdeDBw/iwIEDbR67LdnZ2RaPWVZWhk8//RSjRo2y2sPUnEajafE8e+edd1o8V1p7TVlz7bXXIicnx2LGZkNDA9555x14eXm1mMlH3Y89MwQA+Pjjj/HTTz+12P7www/jxRdfxNatWzFp0iTcf//9cHFxwQcffIDa2lqsWrWq048VEhKChx9+GG+88QZuuOEGXHPNNUhMTMSPP/6IwMDAdr8RjR07FuvWrcNjjz2Gyy67DF5eXrj++usxbNgwXHHFFVi6dCmKiorg7++PtWvXyh9QEldXV7z44ou45557MH36dCxYsABpaWlYvXp1i5yZP/3pT1i/fj3uvfde7NixA1deeSUMBgNSUlKwfv16bNmypdOl5X18fPDee+/hT3/6E8aMGYPbbrsNQUFByMjIwA8//IArr7wS//rXv+Dj44MpU6Zg1apVqK+vR0REBH7++WekpaVZ/ZsApqnrt912G1xdXXH99dfD09MTS5YswSuvvIIlS5Zg3Lhx2L17t/xNtiP69++PF198EUuXLkV6ejpuvPFGeHt7Iy0tDV9//TXuvvtu/PWvf23zGK6urrj55puxdu1aVFZWtli7qDPnas2SJUtw77334pZbbsHVV1+NxMREbNmyRZ6mLnniiSfw3Xff4brrrsPixYsxduxYVFZW4vjx49i4cSPS09MRGBiIJUuWoKioCNOnT0dkZCTOnz+Pd955B6NGjWp1eq65AQMGYNKkSbjvvvtQW1uLt956CwEBARbDW//+978xadIkxMXF4a677kJMTAxyc3Nx4MABZGVlITExsc3HuNTXZUefh6154oknsHHjRsybNw933HEHxo4di6KiInz33Xd4//33MXLkSFx33XXYtGkTbrrpJsyZMwdpaWl4//33MXToUFRUVLTbRmsGDRqEO++8E4cOHUJISAg+/vhj5ObmYvXq1R26/3XXXYfPPvsMer0eQ4cOxYEDB/DLL78gICDAYr9Ro0ZBo9Hg1VdfRWlpKXQ6HaZPn47g4OAWx7z77rvxwQcfYPHixThy5Aj69u2LjRs3Yt++fXjrrbfg7e3dpXOlTrDDDCpSEGkKY2s/mZmZQgghfv/9dzFr1izh5eUlPDw8xLRp08T+/futHqv5NG9r06sbGhrEc889J0JDQ4W7u7uYPn26SE5OFgEBAeLee+9t874VFRXiD3/4g/D19RUALKZpp6amihkzZgidTidCQkLEM888I7Zu3Wp1iuW7774r+vXrJ3Q6nRg3bpzYvXt3iymXQpimWL766qti2LBhQqfTCT8/PzF27FixYsUKUVpa2ubf19rUbPNzmzVrltDr9cLNzU30799fLF68WBw+fFjeJysrS9x0003C19dX6PV6MW/ePJGdnW11KvLf//53ERERIdRqtcWU1qqqKnHnnXcKvV4vvL29xfz580VeXl6rU7OttVUIIb766isxadIk4enpKTw9PcWQIUPEAw88IE6dOtXm30AiXQeVSiU/r8x19FytTdk1GAziqaeeEoGBgcLDw0PMmjVLnD17tsXUbCFMU5KXLl0qBgwYILRarQgMDBQTJ04Ur7/+uqirqxNCCLFx40Yxc+ZMERwcLLRarejTp4+45557xMWLF9s8R/Np1G+88YaIiooSOp1OTJ48WSQmJrbYPzU1Vdx+++0iNDRUuLq6ioiICHHdddeJjRs3Wj1mc5fyupR05HnYmsLCQvHggw+KiIgIodVqRWRkpFi0aJEoKCgQQpimaL/00ksiOjpa6HQ6MXr0aPH999+LRYsWtSiv0JHrHB0dLebMmSO2bNkiRowYIXQ6nRgyZIjYsGFDh8+5uLhY/PnPfxaBgYHCy8tLzJo1S6SkpFh9rvznP/8RMTExQqPRWLyHWHufyM3NlY+r1WpFXFxci5IIbV1La69p6jiVEMw4ImUoKSmBn58fXnzxRTz77LP2bg4RETkI5syQXVRXV7fYJuVHNC8TTkRE1BbmzJBdrFu3DmvWrJHLs+/duxdffvklZs6ciSuvvNLezSMiIgfCYIbsYsSIEXBxccGqVatQVlYmJwW/+OKL9m4aERE5GObMEBERkUNjzgwRERE5NAYzRERE5NB6Rc6M0WhEdnY2vL29u60kPREREdmWEALl5eUIDw9vsZCnuV4RzGRnZ3dpQUAiIiKyv8zMTERGRrZ6e68IZqRS0pmZmZ1eR4iIiIjso6ysDFFRUe0uCdErghlpaMnHx4fBDBERkYNpL0WECcBERETk0BjMEBERkUNjMENEREQOjcEMEREROTQGM0REROTQGMwQERGRQ2MwQ0RERA6NwQwRERE5NAYzRERE5NAYzBAREZFDYzBDREREDo3BDBERETk0BjNOav3hTOw7W2DvZhAREdlcr1g1u7c5mV2GJzceAwCkvzLHzq0hIiKyLfbMOKGcsmp7N4GIiKjHMJhxQiqVyt5NICIi6jEMZpyQmsEMERH1IgxmnJDaLJYRQtivIURERD2AwYwTMu+ZMTKWISIiJ8dgxgmZjzI1GI32awgREVEPYDDjhMx7ZhoM7JohIiLnxmDGCVkEMxxnIiIiJ8dgxgmZDzMZGMwQEZGTYzDj5JgzQ0REzo7BjBMymvXGsGeGiIicHYMZJ2QevzABmIiInB2DGSdkXiiPPTNEROTsGMw4IYueGQYzRETk5BjMOCEje2aIiKgXYTDjhMyDmXoDZzMREZFzYzDjhMzXlmTPDBEROTsGM07IPIBhzgwRETk7BjNOiDkzRETUmzCYcUKWs5mYM0NERM6NwYwTYp0ZIiLqTRjMOCHWmSEiot6EwYwTssiZ4XIGRETk5BjMOCHzYIY9M0RE5OwYzDghwQRgIiLqRRjMOCFOzSYiot6EwYwTsiiax5wZIiJycgxmnBCXMyAiot6EwYwTYgIwERH1Ji72bgB1H4NRoLKuwaLOjIEJwERE5OTYM+NEbn5vP0Ys/xl55TXyNvbMEBGRs2Mw40QSM0sAADtS8uRtzJkhIiJnx2DGSdTUG+Tffdxd5d/rOZuJiKw4mV2GKat2YM2+NHs3heiSMZhxEoWVdfLvdQ1NeTLMmSEia6795x5kFFVh+f9O2rspRJeMwYyTKCivlX+/UFIt/86cGSJqjsPP5GwYzDiJgoqmYOZiaVMCMN+0iKi52gZD+zsRORAGM07CPJixqADMYIaImqmptxx+FoLvE+TYGMw4iXyzYSZz7JkhoubMJwwAnChAjo/BjJMoqKizup1rMxFRc7UNlj0zdQZOFCDHxmDGSeRXtNYzwzcpIrLUvGemroHvE+TYGMw4iYJWhpmYM0NEzTUPZpgQTI6OwYyTaK1nhsNMRNRc8wRg9syQo2Mw4ySknpkIX3eL7eyZIaLmahqa98wwmCHHxmDGCdQ2GFBW0wAAGBGpt7iNOTNE1Fwtc2bIyTCYcQKFjTOZXDUqxIb5WNzGnhkiaq55TwxzZsjRMZhxAlLBvABPXYthJtaZIaLmWiYAs2eGHBuDGScgFcwL9NYiwo85M0TUNiYAk7NhMOMEpJ6ZIC/2zBBR+9gzQ87GZsHMypUrMXHiRHh4eMDX19fqPg899BDGjh0LnU6HUaNGWd1HCIHXX38dgwYNgk6nQ0REBFauXGmrZjskqfpvoJcOIT5uUKmabmPPDBE1x54ZcjYutjpwXV0d5s2bhwkTJuCjjz5qdb877rgDBw8exLFjx6ze/vDDD+Pnn3/G66+/jri4OBQVFaGoqMhWzXZITcNMOmhd1Aj21iG3zLSNs5mIqDlOzSZnY7NgZsWKFQCANWvWtLrPP//5TwBAfn6+1WAmOTkZ7733Hk6cOIHBgwcDAPr169f9jXVw0jBToJcOABDu6y4HM1xAjoiaq2XPDDkZRefM/O9//0NMTAy+//579OvXD3379sWSJUva7Zmpra1FWVmZxY8zk3tmvLQATMGMhDkzRNRc856ZOk7NJgen6GDm3LlzOH/+PDZs2IBPP/0Ua9aswZEjR3Drrbe2eb+XX34Zer1e/omKiuqhFtuHnADsbeqZmTwgUL6NOTNE1BwTgMnZdCqYefrpp6FSqdr8SUlJ6bbGGY1G1NbW4tNPP8XkyZMxdepUfPTRR9ixYwdOnTrV6v2WLl2K0tJS+SczM7Pb2qREUgJwUOMw022X98H7/28MAObMEFFLHGYiZ9OpnJnHH38cixcvbnOfmJiYS2mPhbCwMLi4uGDQoEHyttjYWABARkaGnEfTnE6ng06n67Z2KFldgxGl1fUAmnJmAMBL5wqAC00SUUvsmSFn06lgJigoCEFBQbZqSwtXXnklGhoakJqaiv79+wMATp8+DQCIjo7usXYoWWGlaYjJRa2C3t1V3q5Rm+ZnM2eGiJprkTNjYDBDjs1ms5kyMjJQVFSEjIwMGAwGJCQkAAAGDBgALy8vAMDZs2dRUVGBnJwcVFdXy/sMHToUWq0WM2bMwJgxY3DHHXfgrbfegtFoxAMPPICrr77aoremN5OSfwO8tFCrmwrMuGgYzBCRdc3rzDRfeJLI0dgsmHn++efxySefyP8fPXo0AGDHjh2YOnUqAGDJkiXYtWtXi33S0tLQt29fqNVq/O9//8Nf/vIXTJkyBZ6enpg9ezbeeOMNWzXb4TRP/pVIPTNMACai5povLMmeGXJ0Ngtm1qxZ02aNGQDYuXNnu8cJDw/HV1991T2NckIF5U3Vf825SMEM36SIqJkWPTPMmSEHp+ip2dS+/GYF8yTsmSGi1jABmJwNgxkH17z6r8RFbbq0zJkhoua4NhM5GwYzDq559V+JlADMnhkiaq55wi97ZsjRMZhxcK0lALtwajYRtUKamv3MtUMAcDkDcnwMZhxc8+q/kqacGX7jIqImBqOQF6D1cTPVpuIwEzk6BjMOTs6ZadEzw5wZImrJfFq2VGiTw0zk6BjMOLC6BiNKqlouZQBwNhMRWWee/OvNnhlyEgxmHJi0lIFGrYKv2VIGQFPOjBCAkQENETWSpmW7alRw15o+AtgzQ46OwYwDkwrmBXhaLmUAABpN0//rmTdDRI2kYMbNRQOtRgOAPTPk+BjMOLDWZjIBTT0zAPNmiKiJNMykc9VA5yr1zHA2Ezk2BjMOrLXqv0BTzgzAvBkiaiJNy3ZzVUOrMX0EsGeGHB2DGQfWWvVfAHBVN11ag4HBDBGZ1Db2zLiZ9cxwoUlydAxmHJhc/ddb2+I2tVoFVWPnDHtmiEhirWem3iA4UYAcGoMZB9ZawTwJqwATUXO15gnALk0fAeydIUfGYMaBFZS3ngAMsAowEbXUlACshs5FI2+vref7BDkuBjMOrK2cGYBVgImoJfOp2a5mJRxqDZzRRI6LwYwDay+YkXpm6pkATESN5GDGVQOVSgWdC2c0keNjMOOg6g1GFMtLGbRMAAaYM0NELUnVfqWZTFLeDKsAkyNjMOOgChuTfzVqFfw8rAczzJkhouZqzKZmA2DPDDkFBjMOShpisraUgYQ9M0TUnDw120UKZkz/smeGHBmDGQfVVvVfiUtjDQnWmSEiiZQz03yYiT0z5MgYzDioArlgXhvBDHtmiKgZeZjJxXKYieszkSNjMOOgmnpmrOfLAGY5M5zNRESN5KJ5zXpmqusYzJDjYjDjoArK267+CzQFM+yZISJJ03IGpp6ZfoGeAIDfM0rs1SSiS8ZgxkFJCcCtVf8FABcNZzMRkaWmhSZNb//ThwQDALan5NqtTUSXisGMg2qvYB4AaBorAHOYiYgkzXtmpg4KhkatwuncCmQWVdmzaURdxmDGQXUkmHGR68wwmCEiE3ltpsYEYL2HK8ZF+wEAtiWzd4YcE4MZB5Uvz2ZqPwGYOTNEJGk+NRsAZsSGAAC2peTZpU1El4rBjAOyXMqgIz0zzJkhIhPzhSYl02NNeTO/nitERW2DXdpFdCkYzDigosr2lzIAmormsWeGiCQ1zRKAAaB/kBf6BXqi3iCw53S+vZpG1GUMZhyQNMTk76mVh5KsYc4Mkf0IIRRZiE5atkBKAJZIs5o41ESOiMGMA+pI8i/AnBkie7rv898R+9xPuO3DA/h4bxqyipUxU6ipaJ5lMBPfONS0IyWP7xnkcFzs3QDqPDn5t43qvwB7ZojsJb2gEj8l5QAAfj1XhF/PFeGF709iWLgPZg4NxazhIRgc4g2VqvWeVVtpmppt+V32sr7+8HZzQWFlHRKzSjCmj1+Pt42oqxjMOKCCivar/wJmPTMGJgAT9aRNv2cBAC7v649Zw0OxJSkHh9OLkJRdhqTsMrz5y2n08ffArGEhmDksFGP6+LU5ZNxdDEaB+sa6UzoXy54ZV40aVw0KwvfHLmJbci6DGXIoDGYcUEeq/wLsmSGyB6NR4KvfLwAA/t+EaNwwMhx3TuqHwopabEvOw88nc7D7TAEyiqrwnz1p+M+eNAR6aTEjNgQzh4VgYv/AFkNA3UWayQS07JkBTENNpmAmD0/MGmKTNhDZAoMZB9TxnJnGCsAMZoh6zK9phbhQUg1vNxfMHBoibw/w0mH+ZVGYf1kUKmsbsPt0Pn4+mYttybkoqKjD2kOZWHsoE55aDaYODsbMYSGYNiQYPm6u3dY2i2DGpWXANHVQMNQqICWnHFnFVYj08+i2xyayJQYzDkgOZtoomAc09cwwmY+o52w8Yhpium5EeKs9LJ46F8yOC8PsuDDUG4w4eK4IW5Jy8PPJHOSW1eKH4xfxw/GLcNWocEVMAGYNC8XVQ0MQ4uN2SW2raZzJpNWoobYyrOXnqcXYaD8cSi/GjpQ8/GlC30t6PKKewmDGATUlALczzCQtNMm1mYh6RGVtA346YUr8vXVsZIfu46pRY9LAQEwaGIgVNwzD8QuljYFNLs7mVWDPmQLsOVOAv31zAqOifDFrWChmDQtBTJBXp9tXa6X6b3PTh4TgUHoxfklmMEOOg8GMA5ISgNsNZuSeGSYAE/WEzccvoqrOgH6BnhjTx7fT91erVRgZ5YuRUb548pohSM2vwM9Jufj5ZA6OZpQgIdP08+pPKRgQ7IWZQ0Mwa1goRkTqOzQzqqlgXus5OTNig/HqTyk4kFqIytoGeOr4MUHKx2epg2kwGFFc1TibqZ0EYObMEPWsrxpnMd06NrJbpl33D/LCfVO9cN/U/sgtq8HWk7nYkpSDA6mFOJtXgbN5FXh3ZypCfdwwc1gIZg4NxfgYf7hqrPe8tDYt29yAYC9E+bsjs6gae88WYNaw0Es+DyJbYzDjYIoq6yAEoFahzaUMgKZhJubMENleZlEVfj1XBJUKuGl0RLcfP8THDf/vimj8vyuiUVpdj52n8vBzUi52nMpDTlkNPj1wHp8eOA8fNxfEx4Zg5tAQXDU4CB7aprd5eZFJK8m/EpVKhfghIVizPx3bk/MYzJBDYDDjYPIrpKUMdO3WpdBwajZRj9nUOB37yv6BCPd1t+lj6d1dMXdUBOaOikBNvQH7Uwuw5UQufknORWFlHb4+egFfH70AnYsakwcGYuawUMQPCUatlXWZrImPDcaa/enYlpIHo1FYTRYmUhIGMw6mo9V/AbM6MyyaR2RTQgh5iOmWsd3fK9MWN1cNpg8JwfQhITAYBX7PKMaWEznYcjIHmUXV+CU5D78k50GtgjzV2tq0bHPj+wXAU6tBQUUtjl8oxcgo3x44E6KuYzDjYOTqv+3kywDsmSHqKYfSi5FRVAUvnYtdh2U0ahUu6+uPy/r649k5sUjJKZcTiJOyy5BRZFofqr2kXq2LGlMGBeHHEznYlpzLYIYUj8GMg5Gr/7YzkwlgnRminrLxSCYA4Nq4UIscFXtSqVSIDfNBbJgPHp4xEJlFVdh6Mhe/ZxTjj+Oj271/fGyIKZhJycNjMwf3QIuJuk4ZrzrqsAJpmKlDPTOczURka1V1Dfjh2EUAwK1jo+zcmtZF+Xvgjkn9cAf6dWj/qYODoFIBSdlluFhajTC9bfOAiC5F21lgpDhNSxm0nzPjytlMRDa3JSkHlXUG9PH3wGV9nWdxxkAvHUY3Di9tT8mzb2OI2sFgxsHkd3BdJoA5M0Q9QVq+4JYx3VNbRkniY01rS21LZjBDysZgxsEUlHes+i/ACsBEtnahpBr7UwsBADeP6dlZTD0hPjYYALDvbAGq6wzt7E1kPwxmHIycANyZnBmuzURkE1//ngUhgCti/BHl73wrTA8O8UaErztqG4zYn1pg7+YQtYrBjANpMBhRVNWVnhkGM0TdzVRbxlQoT8mJv5dCpVLJvTO/cKiJFIzBjAMpqmpaysDfs/0EYClnpp7BDFG3+z2jGGkFlfDQajB7uPOW/J8+xBTMbE/JhRB8LyFlYjDjQKTqv/6e2naXMgDM12ZizgxRd9t4xNQrM3t4mFOvLH1FTAA8tBrkltUiKbvM3s0hsorBjAORqv92ZIgJMJvNxJwZom5VU2/A94nZAHp++YKe5uaqwaQBgQA4q4mUi8GMA5EK5nUk+RdgzgyRrfx8MhfltQ2I8HXHFf0C7N0cm5shTdFOybVzS4isYzDjQAo6UWMGAFxYAZjIJppqy0T0ihWlpw4JAgAcyypFXlmNnVtD1BKDGQfSmeq/AKBpowJwXYMR2SXV3dc4ol4ip7QGe8/kAwBuGRtp59b0jGBvN3mxSVYDJiViMONApATgjvfMWK8AvPdMAca/9AsmvrIdu0/nd28jiZzc10cvwCiAy/r6ITrA097N6THxjbOatjGYIQViMONAupoA3Hw207rDmSiuqgcApBVUdmMLiZybqbaMaYjp1l7SKyOR6s3sPVOAmnpWAyZlYTDjQDpT/RdoPWemuLJO/p35NEQdl5hVirN5FXBzVePauDB7N6dHDQ3zQZjeDdX1Bhw4V2jv5hBZYDDjQDqbANza1OziqqZghjVoiDpu45FMAMA1w0Lh7eZq59b0LJVKJRfQ25bMWU2kLAxmHITBKFDU2KMS6N2xBODWpmaXNA4xAUA9a9AQdUhNvQH/S7wIoPck/jYnDTVtT85jNWBSFAYzDqKwshZGAahUgL9HB2czyQnAlr0vlj0zfEMi6ohtyXkora5HmN4NE/sH2rs5djGxfyDcXNXILq1B8sVyezeHSMZgxkEUlJsCEH8PLVw0HbtsLlamZtc2GFBV15S8x5wZoo6REn9vGh3RoeVEnJF5NeDtLKBHCsJgxkF0NvkXsJ4AbD7EBJhW4iaituWV12DX6d5VW6Y18Y3VgLmKNikJgxkH0dnkX8AsZ8YsL8Z8iAngMBNRR3x7NBsGo8CYPr7oH+Rl7+bYlZQEnJhVIte+IrI3BjMOorPVfwHznBmzYKayWc8MgxmiNgkhmpYv6OW9MgAQ4uOGuAg9hAB2nGLvDCkDgxkH0dnqv4D1nJnmPTMcZiJqW1J2GU7llkProsZ1I8Lt3RxFkHpntnOoiRTCZsHMypUrMXHiRHh4eMDX19fqPg899BDGjh0LnU6HUaNGWd1ny5YtuOKKK+Dt7Y2goCDccsstSE9Pt1WzFUuu/tuJnBmpZ6bebDZTi2CGPTNEbZJ6ZWYODYHevXfVlmmNtIr2njP5qG1gNWCyP5sFM3V1dZg3bx7uu+++Nve74447sGDBAqu3paWlYe7cuZg+fToSEhKwZcsWFBQU4Oabb7ZFkxVNTgDuVM6M6fIKARgbg5bmCcDMmSFqXV2DEd8mXADQ+5YvaMuwcB8Ee+tQWWfAwXNF9m4OEVxsdeAVK1YAANasWdPqPv/85z8BAPn5+Th27FiL248cOQKDwYAXX3wR6sYP5r/+9a+YO3cu6uvr4erae74lycNMXeiZAUw9MFq1ymIpA2k7EVm3PSUPxVX1CPbWYfLAIHs3RzHUahXiY4Px5W+Z2JaciymD+Lch+1J0zszYsWOhVquxevVqGAwGlJaW4rPPPsOMGTPaDGRqa2tRVlZm8ePomhaZ7HgCsItZMCP1wEgLTErHYc4MUevk2jJjem9tmdZMH2IaatqWwmrAZH+KDmb69euHn3/+Gc888wx0Oh18fX2RlZWF9evXt3m/l19+GXq9Xv6JiorqoRbbhmkpgy4MM2nMe2ZMQUtJleXK2+yZIbKusKIWO1JMCa63juEQU3OTBgRC56JGVnE1TudW2Ls51Mt1Kph5+umnoVKp2vxJSUnptsbl5OTgrrvuwqJFi3Do0CHs2rULWq0Wt956a5vfBJYuXYrS0lL5JzMzs9vaZA9FlXVNSxl4dqZnpunyNvXMWAYzzJkhsu7bhGw0GAVGRuoxMMTb3s1RHHetBhP7BwAAtrEaMNlZp3JmHn/8cSxevLjNfWJiYi6lPRb+/e9/Q6/XY9WqVfK2zz//HFFRUTh48CCuuOIKq/fT6XTQ6Treg6F0UvJvZ5YyAADzXvGGZgnAUiVhLjRJZB1ry7QvPjYEO07lY1tyHu6fOsDezaFerFPBTFBQEIKCei7Rq6qqSk78lWg0GgCA0dh7cj26Uv0XAFQqFVzUKjQYhdwDU9TYMyMFM4Ze9Hck6qiT2WU4ebEMWo0a17O2TKukejO/ZxSjqLKuUz3HRN3JZjkzGRkZSEhIQEZGBgwGAxISEpCQkICKiqax1bNnzyIhIQE5OTmorq6W96mrM33gzpkzB4cOHcILL7yAM2fO4Pfff8ef//xnREdHY/To0bZquuI0zWTq/BuFeRVgg1GgtLpZAjCHmYhakBJ/42OD4ccP6FaF+7pjaJiPqRpwCgvokf3YbGr2888/j08++UT+vxR87NixA1OnTgUALFmyBLt27WqxT1paGvr27Yvp06fjiy++wKpVq7Bq1Sp4eHhgwoQJ+Omnn+Du7m6rpitOV3tmANOMplqYZi2VVddDSjWSE4A5zERkod5gxDdHWVumo+Jjg3HyYhm2p+RxSI7sxmbBzJo1a9qsMQMAO3fubPc4t912G2677bbuaZSDapqW3flgxrxnRkr+9dK5wN3VNFzHBGAiS7tO5aOwsg6BXjrWT+mA+NgQvLP9LHadzkddgxFaF0VPkiUnxWedAyhoHGYK6kTBPImUMGwwCrnGjK+Hq1mQw5wZInNS4u+No8Lh2omE+95qRIQegV5aVNQ24FA6qwGTffCV6gDyL2GYSQ5aDEKuMePnoZVr0DBnhqhJcWWdPM2YQyYdo1arMG2wKRH4l2RO0Sb7YDDjAJpWzO58IqKrumnlbPOeGakGDXNmiJp8l5iNeoPAsHAfxIb52Ls5DiO+ceHJbcmsBkz2wWDGAVxSzoymaTjJomfGLMghIhNpFhMTfztn8sBAaDVqZBRVITWf1YCp5zGYUTiLpQy6kjOjbsqZKaqUgpmmnJl65swQAQBO55bjWFYpXNQq3DCStWU6w1PngiukasDJnKJNPY/BjMIVV3VtKQOJ5Wwm0zCTn6fWIjGYiICvGhN/pw8JRkAXekF7u/jGAnoMZsgeGMwonFRjxs9D26WZFebDSdaGmZgzQ2Sqw7SpsbYME3+7RqoGfPh8kfxeQ9RTGMwo3KUk/wJNPTP1BqNcZ4ZTs4ks7TlbgPzyWvh7auWZOdQ5Uf4eGBziDaMAdp7Kt3dzqJdhMKNwl1L9F2jeM9M4zGTWy8NhJqKm2jI3jAxn0bdLEB/bONTEpQ2oh/FVq3AF5V2fyQRYrwDs56G12E7Um5VW1WNrkqk+CmcxXRopmNl5Kg/1Bvb6Us9hMKNwUs9MV2YyAbCoJ2NZZ4Y5M0QA8L9j2agzGDEk1BvDwllb5lKMivKDv6cW5TUNOJxebO/mUC/CYEbhLqX6LwC50m9FbT3qGkzflPw8tcyZIWokDTHdOjYSKpXKzq1xbBq1ClMHm9az2sZqwNSDGMwoXHclAEvHcdWo4KnVMGeGCMDZvAokZJZAo1Zh7qgIezfHKcxorAa8nXkz1IMYzCicXP23y8NMKovj+HpooVKpzGY5CZYfp15Lqvg7dVBQl4dyydLkgYFw1ahwrqAS51gNmHoIgxmFk3NmupwAbLrE0nCVv4eph0cKcgCAnTPUGxmMAl//bqotw8Tf7uPt5orx/UzVgNk7Qz2FwYyCGc2WIOh6AnBjz0zjMJOvh6tpu6YpmGHeDPVG+84WIKesBnp3V0yPZW2Z7iQV0OMq2tRTGMwoWHFVnZzT0pWlDICmhSbzzSoJA02znADOaKLeSRpimjsqHDoXjZ1b41ykKdqH0otRWl1v59ZQb8BgRsGaAhDXLi1lALTsmfHzNPXMaNTmPTMMZqh3Kaupx08ncgAAt4zhEFN3iw7wxIBgLxiMArtOsxow2R6DGQW71IJ5QFPQUlbTAMCUAAxY5sxwRhP1NpuPXURtgxEDg70wIlJv7+Y4Jal3ZjuHmqgHMJhRsEtdygCwDFoAUy8PAKjVKkg3NbBSJ/UyUm2ZW1hbxmbih5imaO84lc/3GLI5BjMKdqnVfwHApdnwlNQzA5hVB2bPDPUi6QWVOHy+GGoVcNNo1paxlTF9fOHr4YrS6nr8nlFi7+aQk2Mwo2CXWv0XsNYzYxbMaJoWoSTqLaTE38kDgxDi42bn1jgvF40aUwexGjD1DAYzCiZX//Xu2kwmwDLRF2gaZjK/jQvCUW9hNApsYm2ZHhPfWA2Yq2iTrTGYUTC5+m939sx4alvcxp4Z6i1+PVeICyXV8HZzwdVDQ+zdHKc3ZVAQXNQqnM2rwPnCSns3h5wYgxkFk6ZTd7X6L9BUAVhiOczEnBnqXaTE3+tHhsPNlbVlbE3v7orL+voDALYls3eGbIfBjIJ1SwKwWc+MSmV6c2l+G4vmUW9QUduAHxtry3CIqedIU7S3pTBvhmyHwYxCGY0ChZXdV2cGAHzcXC3+L/3O5QyoN9h8/CKq6w2ICfTE6Chfezen15DyZg6eK0J5DasBk20wmFEo86UMAry6ngBs3jNjnvwLQK4qzJwZ6g2+Ym0Zu+gX6ImYQE80GAV2ny6wd3PISTGYUSgp+df3EpYyACzrzJjXmAHMe2YYzJBzyyyqwsG0IqhUwM1jWFump3GoiWyNwYxCdUf1X6DtnhnmzFBvIdWWmTQgEGF6dzu3pveZ3lgNeOepfPYEk00wmFEoOfn3EoMZnWvTJTZP/gWaiuYxZ4acmdEo5GCGi0rax7i+fvBxc0FRZR0SMovt3RzqZt8lZsvDuPbCYEahmgrmXVowIyXfAYC/p+WxpGnb/KZEzuxQehEyi6rhpXPBrGGh9m5Or+SqUeOqwY1DTZyi7VSq6hrw0JdH8fiGRJRU1dmtHQxmFKppKYOuJ/8CQISvO96+bRRGROox/zLLb6UucgVgBjPkvKTaMnPiwuCuZW0Ze5kRy2DGGV0orpZ/L66y32w1F7s9MrWpoPzSp2VL5o6KwNxRLZMeWQGYnF1VXQM2H78IALh1HIeY7OmqQUHQqFU4lVuOzKIqRPl72LtJ1A2yzIIZey6Nw54ZhequnJm2MGeGnN1PJ3JQWWdAdIAHxkX72bs5vZqvhxZjG6/Bdq7V5DQyi6vk343Cfl+MGcwoVHdU/22PlDPD2UzkrMwTf1lbxv7ih0hTtBnMOIuMwqZgxp69/AxmFKq7pma3xZXDTOTELpRUY39qIQDgptGsLaME0oSEX1MLUVHbYOfWUHf47940+Xd7dvIzmFEgo1E0rZjtfWkJwG1h0TxyZl//ngUhgAkxAczPUIj+QZ6IDvBAncGIvWdYDdjZGDjMROZKquubljLwZM4MUWcJIeRZTLdwUUnFUKlUiG8soLctmdWAnYFZXVa79vJzNpMCSUNMendXaF1sF2+6MGeGnNSR88VIL6yCh1aD2cNZW0ZJ4mOD8fG+NOw4lQejUUCtZi6To9l9Oh8PrT2K6joDzOMXJgCThYJy2yf/ApyaTc5LSvy9Ni4Mnjp+Z1OSy/r6w1vngoKKOiRmldi7OdQFb2w9jZKqetQ2WPbqG5kATOa6q2Bee6ScmXoOM5ETqak34PtEU20ZLl+gPFoXNaYMCgLAKdqOKCGzBImZJdBq1PjpkcnY9/R0+TZ7Xk8GMwokL2Vgw5lMQNOK2gYOM5ET2ZKUg/LaBkT6uWN8P397N4eskFbR/oXVgB3OpwfSAQDXjQjDkFAfRPg2LdyalF1mp1YxmFEkeSaTrYMZzmYiJyQl/t48JpL5GAo1dXAwVCog+WIZskuq278DKcZvaUUAgFutJNbbs5QTgxkF6omCeYD51GwOM5FzyCmtwb6zpim/t4xhbRml8vfUYkwfUzVgFtBzLEWVpi/bEX7u7ezZsxjMKFBPLGUAAK4a9syQc9l0NAtGAVze1x/RAZ72bg61QRpq2s4p2g6jpt6AqjoDAMDPs2VOpz2rbDOYUSC5+q8NC+YBTcsZMGeGnIEQAl81DjFZ6wInZZHqzexLLURVHasBO4LCxl4ZV40K3lZmCdpzVJfBjAL1WAIwc2bIiSRkliA1vxJurmrMjmNtGaUbFOKFSD931DUYse9sob2bQx1Q1JjP6e+ptdoLo2bPDEmMRoHCnkoAZgVgciJSbZnZw8Pg7eZq59ZQe0zVgBsXnuRQk0MoqpKCGeufTfZMt2cwozCl1fVyT0mAjevMsGgeOYuaegO+S8gGwNoyjkRaeHJ7Sp5dC65Rx1Q1Lg7qpdNYvd3DjgUqGcwojPlSBjoX60+Y7iLlzNQzZ4Yc3LbkPJTVNCBc74YJ/QPs3RzqoPEx/vDUapBXXosT2aX2bg61Q/qkUDXrg3nl5jjEhvngmWuH9HyjGjGYUZieqv4LsGeGnMfGI5kAgJvGRMglB0j5dC4aTB5oqga8jQX0HEezl9htl/fBjw9PRpjeftO1GcwoTE8l/wJNOTMZRVWoqTfY/PGIbCGvrAa7TucD4BCTI5reOEV7WwrzZpTOjutItovBjMLI1X9tXDAPaOqZOXK+GLd/9JvNH4/IFr5JuACjAMZG+yEmyMvezaFOmtZYDfjEhTLklNbYuznUBtE40KTEvk8GMwrTUwXzgKacGQD4Lb3I5o9H1N2EEPLyBeyVcUxB3jqMjPQFwIUnHYU9ly1oDYMZhSko75mlDICmYSYiR3XiQhlO51ZA56LGnBFh9m4OddEMqRowh5oUjcNM1GEFdkgAJnJUUuLvzGGh0Luztoyjmt5YDXjv2QLm7ylYa7OZlIDBjMI0zWbqiWEm5T0hiTqqtsGAbxNNtWW4fIFjiw3zRrjeDTX1RuxPLbB3c6gdHGaidhWU90z1XwBypWEiR7QjJQ8lVfUI8dFh0oBAezeHLoFKpZJnNf3CKdqKJRQ8zsRgRkGEECislBaZtH0wc1lff/l3rYZPBXIsG49cAADcNDqSvYxOQK4GnJyn6A9NYs8MtaO0ul6uxtsTOTNxkXr85/ZxAAAD3zzIgRRU1GLnKdM3+FvHRti5NdQdJsQEwN1Vg5yyGiRll9m7OdQG5sxQm74+avqm6ePmYvOlDCSjonwBmKoA89sQOYpvE7LRYBQYGeWLAcHe9m4OdQM3Vw0mDTQNF3KKtjIp+SOCwYxC1DUYseJ/JwEAZTUNPfa45jOauKoBOQqptsytY9gr40y4irayyUXzlNcxw2BGKQ6ft0/ROrVZMNNgNNqlDUSdkZRdiuSLZdBq1Lh+ZLi9m0PdaHpjMJOYVYq8clYDpo5jMKMQ0toyALD+ngk99rgWPTOMZcgBfNWY+DtjaDB8PWyfW0Y9J9jHDSMi9QBMs9VIWTjMRO3adcoUzLx92yhc3s+/nb27j4Y9M+RA6g1GfJtgCmZYW8Y5xTcW0OMq2sojBTMqBY4zMZhRgNyyGqTklEOlAiYPDOrRx9awZ4YcyM5T+SisrEOglw5Tevi1Qj0jvrHezJ4zrAasVMoLZWwczKxcuRITJ06Eh4cHfH19W9yemJiIhQsXIioqCu7u7oiNjcXbb7/dYr+dO3dizJgx0Ol0GDBgANasWWPLZve43Y1DTCMi9PD37Nluc42KPTPkOKTlC24aHQ4X1kZySsPCfRDio0N1vQG/niu0d3PIjIJHmWwbzNTV1WHevHm47777rN5+5MgRBAcH4/PPP0dSUhKeffZZLF26FP/617/kfdLS0jBnzhxMmzYNCQkJeOSRR7BkyRJs2bLFlk3vUVK+zFWDev6bplqtkjPTWWuGlKyosk6esnsLh5iclkqlktdq4lCTskjlOxQ4ygQXWx58xYoVANBqT8odd9xh8f+YmBgcOHAAmzZtwoMPPggAeP/999GvXz+88cYbAIDY2Fjs3bsXb775JmbNmmW7xvcQg1FgzxnTWiRXDbZPt7lGpUKDEDBwbjYp2HcJF1BvEBge4YMhoT72bg7Z0IzYYHz5Wwa2p+ThBSEUmaPRmynxaiiun7a0tBT+/k0JsAcOHMCMGTMs9pk1axYOHDjQ6jFqa2tRVlZm8aNUx7JKUFpdDx83F4yM9LVLG6S8GQYzpGRf/d6Y+DuGvTLObmL/QOhc1LhQUo2UnHJ7N4caKfkTQlHBzP79+7Fu3Trcfffd8racnByEhIRY7BcSEoKysjJUV1dbPc7LL78MvV4v/0RFRdm03ZdCGmKaNDDQbjkAUjDDlBlSqlM55Th+oRSuGhVuGMVCec7OXauRFw9lNWAFcabZTE8//TRUKlWbPykpKZ1uyIkTJzB37lwsW7YMM2fO7PT9zS1duhSlpaXyT2Zm5iUdz5bsmS8jkYIZJgCTUn31u6ni7/QhwT2eJE/20bSKNqsBK43yQpku5Mw8/vjjWLx4cZv7xMTEdOqYJ0+eRHx8PO6++2787W9/s7gtNDQUubmWT+bc3Fz4+PjA3d3d6vF0Oh10OtuvOn2pSqrqkJhZAgCYooBgxsgEYFKgBoMRmxqHmG7hEFOvET8kBM/iBBIyS1BQUYtAL+W/pzs7oeCBpk4HM0FBQQgK6r4P3qSkJEyfPh2LFi3CypUrW9w+YcIEbN682WLb1q1bMWFCz1XJtZW9ZwtgFMCgEC+E6a0HZj3BRe6ZUe4TlXqvPWcKUFBRiwBPLaY1lrsn5xeqd8OwcB8kZZdhR0oe5o1TbrpAb9FUNM++7bDGpkkaGRkZSEhIQEZGBgwGAxISEpCQkICKigoApqGladOmYebMmXjssceQk5ODnJwc5Oc3lfa/9957ce7cOTz55JNISUnBu+++i/Xr1+PRRx+1ZdN7hFT1155DTACgVjEBmJRLWlTyhlHhcGVtmV4lPtaUL8m8GaVRXjRj03eG559/HqNHj8ayZctQUVGB0aNHY/To0Th8+DAAYOPGjcjPz8fnn3+OsLAw+eeyyy6Tj9GvXz/88MMP2Lp1K0aOHIk33ngD//3vfx1+WrYQArvPSMGMfb9tal1MT4PKWlbbJGUprarH1pOmYWYuX9D7SKto7z6dj9oGvj/Zm5K/7to0mFmzZg2EEC1+pk6dCgBYvny51dvT09MtjjN16lQcPXoUtbW1SE1NbTdnxxGcyi1Hblkt3F01GNfXz65tGRLqDQBIyCy2azuImvvuWDbqDEYMCfXGsHC9vZtDPSwuQo8gbx0q6wz4La3I3s3p9XrtMBO1ThpiuiLGH26uGru2RVrY8rc0BjOkLF81DjGxV6Z3UqtVmD7Y1DvDasDKocBYhsGMvShhSrbksr6mYObw+SIYmTdDCnE2rwIJmSVwUatw42jWlumtpIUnt6XkyuX0yT6UPJuJwYwdVNY24HC6qRfkqsH2n50xPEIPd1cNSqrqcTa/wt7NIQLQVFtm6uAgTsvtxSYNDITWRY3MomqcyeP7kz1xmIks/HquEHUGI6L83dE3wMPezYGrRo3RfXwBgOPSpAgGo8Cm3znERICH1gUT+wcA4FCTUqgUONDEYMYOzIeYlFIWWhpqOpTOYIbsb+/ZAuSW1cLXw5W1ZUie1bSN1YDtSrmDTAxm7KIpmFHOm7SUBHyIPTOkAFLi79yR4dC52DdBnuxvemO9md8zilFUWWfn1vRijeNMCvkOboHBTA9LL6jE+cIquGpUmNDYdaoEo/v4wkWtQnZpDbKKq+zdHOrFymrqsSUpBwBwC4eYCECErzuGhHrDKICdpzjUZG8MZkgulDc22g9euk6vJmEzHloXDIsw1fH4w38O4tF1Cfh4bxoOpxehqq7Bzq2j3uSHYxdR22DEoBAvxEWwtgyZzGjsndnGasB2o+RhJuV8mvYSTUsYKGeISXLLmAgkZpYgo6gKGUVV+PqoaXE/tQoYGOyNuEg9RkTqERehR2yYj93r45BzkpYvuGVMpGJyysj+pscG4187zmL3qXzUNRjlyuXUc+TZTApMAGYw04NqGwzYn1oIQBn1ZZq7fUJfXBsXhuMXSnE8qxTHskpx/EIJcstqcSq3HKdyy+UPGhe1CoNCvE3BTaQeIyJ8MTjUm28wdEnSCipx5Hwx1CrgJtaWITMjI30R4KlFYWUdDqcXYeKAQHs3qfdSXizDYKYnHUkvRnW9AUHeOsSGedu7OVYFeukwbXAwppnVv8ktqzEFNxdKcTyrBMeySlFYWYeTF8tw8mIZ1h7KBABoNWoMCfNGXITUg+OLgSFeXByQOkxK/J0yKAjBPm52bg0piUatwrQhwdh4JAvbUvIYzNiBkosWMpjpQdIspikDlTMluyNCfNwQMtQNM4aaxqyFELhYWiP33Jj+LUVJVT2ONfbo/N9B0311LmoMDffBiAg94iJ9MSJSj/5BXtCoHef8qWcYWVuG2hEvBTPJufjbnFiHeh91BlIoo8S/OoOZHiRPyR6svCGmzlCpVAj3dUe4rzuuGR4KwBTgZBVXm4KZCyU4nmUaqiqvbcDRjBIczSgBcB4A4O6qwfAIH8RF+MrDVP0CPKFmgNOrHThXiOzSGvi4ucjJnkTmJg8KgqtGhfTCKpwrqET/IC97N4kUgsFMD8ktq0FKTjlUKmCyE3aPqlQqRPl7IMrfA3NGhAEwfdNOL6zE8QuN+TdZpTiRXYqqOgMOpRfjUHrTwpZeOhcMj/DBiEhfeZiqj78Hv3n1ItIQ0/Ujw5lcTlZ56VxwRUwA9pwpwLbkXAYzPaxpOQPlvS8zmOkhUq/MiEhf+Hlq7dyanqFWqxAT5IWYIC/MHWVK5jQYBc7lV8hDU8eySpCUXYaK2gb8eq4Iv55rKtqnd3dFXISUYGz6N8LXXZEvJLo0FbUN+PEEa8tQ++KHBDcGM3m4e0p/ezenV+EwEylqlWx70qhVGBjijYEh3vKHVoPBiDN5FY1JxqYhquSL5SitrsfeswXYe7ZAvr+/p9YswViPEZG+CPHRMcBxcJuPX0R1vQExQZ4YHeVr7+aQgsXHhmD5/07i8PlilFbVQ+/hau8mkQIwmOkBBqPA3jOmD+TeHsxY46JRIzbMB7FhPph/WRQAoK7BiNO55RZJxqdyylFUWYddp/Pl4BAAgrx1cs+NNIsqyJurLDsSacr/rWNZW4baFuXvgUEhXjidW4Gdp/PkXl+yPaHg5QwYzPSAxKwSlFbXw8fNBSMjWdG0I7QuagyP0GN4hB5AHwBATb0BKTnl8vTw4xdKcTq3HPnltdiWkmdRGTRM79bUg9OYh+PfS4b3HE1GYRV+SyuCirVlqIOmDwnB6dwKbEtmMGMPCoxlGMz0BKnq7+SBQXBhzZUuc3PVYFSUL0aZDUNU1xlw8mJTgvGxC6VIza/AxdIaXCytwc8nm1bZjfRzl3tuRkSaAiW9O7uo7e2rxunYkwYEIkzvbufWkCOYERuM93elYuepPDQYjHxfJQYzPYH5MrbjrtVgbLQ/xkb7y9sqahuQdKG0aRbVhVKkFVQiq7gaWcXV2Hw8R963b4AHRjTWv4mL0GNYhF5Ra2Y5O6NRyMEMa8tQR43u4wc/D1cUV9Xj8PliXBHT9UV7k7JL8cym43hg2gDMHBbaja10PpzN1IsVV9bhWFYJAGDyIOebkq1EXjoXjI8JwHizN7jS6nokXZCqGJsSjTOLqpFeWIX0wip8l5gNwDQW3D/IyyIHZ2iYHu5aThW2hd/Si5BVXA1vnQtmDuUHCXWMRq3CtMHB2HT0Aran5F1SMPPKjylIzCrFY+sT8dMjPoj08+jGljoX0TifSXmhDIMZm9t7tgBGAQwO8WYXuh3p3V0xcUCgRQn04so60zpUjVPEj2eVIru0BmfzKnA2rwKbzBbaHBTibZGDMyTUm7VQuoGU+DtnRBgDRuqU6bGmYOaX5Fw8c21sl46RfLEMexonZ1TUNuCJDcfwf0vGs4CnA2IwY2POUvXXGfl5ajFlUBCmmA3/5ZfX4sSFpkU2E7NKkV9ei5SccqTklGOD2UKbg0O9LXJwBoVwoc3OqKxtwObjFwFwiIk6b8qgILioVTiXX4m0gkr0C/Ts9DH+uycNAHBZXz+cuFCGA+cK8emBdCy+sl93N9cpCAUXmmEwY0NCCOxmvoxDCfLWYdqQYEwbYrnQpinBuATHGgOdoso6JGWXISm7DF+iaaHN2DBveRXxuEg9BgZ7MTmxFT+dyEFVnQF9AzwwNtrP3s0hB+Pj5orL+/ljf2ohtiXnYsnkmE7dP7esBt8lmnpfn7k2FiculOK5b5Pwyk8pmDIoCDGsLtwqlQKjGQYzNpSSU4688lq4u2owri/frB1ViI8brh7qhqvNFtrMLq2xmCJ+LKsUpdX1SMwqRWJWKYAMAKaFNoeFWy7TEMOFNgE0zWK6ZQxry1DXxMeGYH9qIban5HU6mPlkfzrqDQLjov0wuo8fRkX54ueTudhzpgCPrU/Exnsn8ItIM8pdM5vBjE1JQ0wT+gdA58J8AGehUqkQ4euOCF93XDPctA6VEAKZRdVyBeNjWaU4ccG00ObvGSX4PaNEvr+HVoPh4eZF/vTo28sW2swqrsL+1EIAwE1jWCeEuiZ+SDD+/v1J/JZWhLKaevi4dazUQlVdA/7voOkLhxQEqVQqrLp1BGa+uRsJmSX4YPc5PDBtgM3a7oiaZjPZtx3WMJixIam+DIeYnJ9KpUKfAA/0CfDAdSPCAbS90OZv6UX4Lb1pHSpvnQuGywnGpmGqKH/nXYfq699N3fsT+wdw9gh1Wd9AT/QP8kRqfiV2n86XX3vt2XA4C6XV9YgO8JB7XAEgTO+OFTcMw2PrE/HWL6cxbXAwhob72Kr5DkuJ70oMZmyksrYBh8+bPqymMJjplTqz0GZ5bQMOnCvEgXOF8v317q5ma1CZZlGF690cPsARQlgMMRFdivjYEKTmn8O25LwOBTMGo8BHe02Jv3dO6tdiyPem0RHYkpSDLUm5eGx9Ar598Er2rDcSCh5oYjBjIwdSC1FvEOjj74G+AfzmSSadXWhzz5kCeeooAAR4as1WETfNogrxcbPX6XTJkfPFSC+sgqdWg9lxrC1DlyZ+SDA+3H0OO07lwWAU7eajbT2Zg4yiKvh6uFqdRadSqbDypjgcTi9GSk453vrlDJ66Zoitmu9QOMzUC5lX/XX0b9JkW51ZaLOwsg47T+Vj56mmhTaDvXUWU8TjIvUI9FLuQptSbZlr48LgoeVbEF2asdF+0Lu7oqSqHr9nFOOyvv5t7v/h7nMAgP83PrrV51+glw4rb4rDvZ8fwQe7UjEjNtiiynhvx9lMvQiXMKBL0ZmFNvPKa/FLch5+SW5aaDNc79aYYGyaRRUXoYefAhbarK4z4Idjptoyt7C2DHUDF40aUwcH4duEbGxLzmszmDlyvhi/Z5RAq1Hj9onRbR73muGhuHl0BDYdvYDH1ydi88OTGXwrGK+MDaQXVCKjqAquGhUm9O96mW0ic9YW2qyqa0DyxbIWC21ml9Ygu7QGW5KaFtqM8neX69+MaFyHqqcX2vz5ZA7KaxsQ5e+Oy9v5Bk3UUdOHBDcGM7l4enbrQ0L/3WPqlZk7KhzB3u0Pzy67YRgOnCtEemEVXvkxBS/MHd5tbXZEonGcSYmDDQxmbEDqlRkX7Q9PLlpINuShdenwQpuZRdXILKrGD41VdwGgX6BnU4JxDyy0KQ0x3Tw6sldNRSfbmjooGBq1CmfyKpBRWIU+VvIUMwqrsCXJtMjsXVM6VpNG7+6KVbeOwJ8++g2fHjiPmUNDMWkg19hjMNNLcAkDsqfOLLSZVmAqBd8TC21eLK3G3rOmZGbOYqLupPdwxbhoPxxMK8K2lFz82cpyBB/vS4NRmIb+B4V4d/jYkwcG4U9XROOzX8/jiY2J+OmRKT3eo6kUQrmTmRjMdLfaBgMONBYDmzKQwQwpw6UstKlRqzAw2OuSF9rc9PsFCAFc3s/f6jdnoksxIzYEB9OKsD0lr0UwU1JVh/WHTcuO3NXJSsEAsPTaIdhzJh/phVVY8b8k/GP+qO5ossNpimWU1zXDYKabHU4vRnW9AUHeOsSGdTz6J+pp7S20eaxxLaq2F9r0lYeo2lpo07y2DBeVJFuYHhuMlZuT8eu5QpTX1MPbrBrw/x3MQFWdAUNCvXHlgM7nMXpoXfDG/JGY9/4BbPr9AmYNC8WsYb23rACHmXoBTskmR9Z8oU0hBHLLak09N2Y5OBYLbf5muq/WxTTF3HyIakCQaaHNo5klOJdfCXdXDa6NC7PjGZKz6h/khX6BnkgrqMTeMwWY3fg8q2sw4pP96QBMvTJdfV8eG+2Pu6f0x/u7UvHMpuMYG+2n6BIItsBhpl6ESxiQM1GpVAjVuyFUH4qZjd9EhRC4UFItz5463tiLU1bTgMTMEiRmlsj3d3NVY1i4HrUNBgDA7OGhNk0wpt5t+pBgfLQ3Db8k58nBzHeJ2cgrr0WIjw7Xj+zYcgetefTqgdh5Kg8pOeV49uvjeP//je1VX1qlCsBKPGO+q3SjnNIanMoth0oFTBrAjHdyTiqVCpF+Hoj085A/MIQQyCiqslim4cSFMlTUNuDI+WL5vqwtQ7YUH2sKZnY2VgNWq5qmYy+e2K/VYdCO0rlo8Mb8kbjx3/uwJSkXXx+9gJt7YTK7EuM3BjPdaHfjENPISF9FFCgj6ikqlQrRAZ6IDvCUv/0ajQJphZXyKuIBXlpMZN0lsqHL+vrD280FhZV1SMgsQVVdA1JyyuGh1eAPl/fplscYFq7Hw/ED8frPp7HsuyRcEROAcF/3bjm20il5mOnSwlSywKq/RE3UahX6B3nhxtEReP76oXhg2oBe1SVPPc9Vo5bff7en5OI/e0wLSs4fFwW9R/dNp773qv4YFeWL8poGPPXVMbmYnLOTzlKJyxkwmOkmDQYj9pwxBTNcJZuIyD7iY03J6+/uTMXu0/lQq0yrY3cnF40ab8wfCTdXNfacKcDnBzO69fhKp8TvJAxmukliVinKahqgd3fFyEi9vZtDRNQrTR0UDLWqaUhk9vAwRPl3f12j/kFe8mraL/2QjPSCym5/DMVRcA8Ug5luIg0xTRoYCBcN/6xERPbg56m1CF6WTO7eXhlziyb0xYSYAFTXG/DXDYkwGJX7Yd8dmoaZlIefut2E+TJERMqgM5u1NLqPn80eR61W4bV5I+Clc8Hh88X4T+PMKWenxNw3BjPdoLiyDseySgBwCQMiInt7ds5QDAn1xqb7J9r8sSL9PPD8dUMBAP/4+TRScsps/pj2ouBRJgYz3WHP2QIIAQwJ9Uaovv1l5YmIyHauGhSEnx6ZgjE27JUxN29cJOKHBKPOYMRj6xJR12DskcftaQLKjWYYzHQDVv0lIuq9VCoVXr4lDn4erjh5sQzvbD9j7ybZlAJHmRjMXCohBHZzSjYRUa8W7O2GF2+MA2CaFp5gtqyHs+AwkxNLvliO/PJauLtqMK5vz3RpEhGR8swZEYYbRobDYBR4bH0CauoN9m5St2LRPCcmzWKa2D8AOheNnVtDRET29MLcYQj21uFcfiVe/SnF3s2xCQ4zOaFdp/MAAFcN5hATEVFv5+uhxau3jgAArN6Xjv2pBXZuUffhMJOTqqhtwOF004rAnJJNREQAMG1wMBY2Lmz5xIZjKK+pt3OLuoc0m0mBHTMMZi7Fr6mFaDAKRAd4oG+gp72bQ0RECvHsnFhE+bvjQkk1/v79SXs3p1txmMnJSIXyLu/rb9+GEBGRonjpXPDGvFFQqYD1h7Pwy8lcezfp0nGYyTmF6N0wLtoPY6I5i4mIiCxd3s8fSxpX7H5603EUVdbZ/DGTL5bhfKFtFr2UZzMpsGuGwcwl+OP4aGy8b6I8NkpERGTu8ZmDMTDYCwUVtXjumxMQNsyi3Zaci2v/uQcLP/zVZo8BMGeGiIioV3Fz1eAf80fBRa3CD8cv4rvEbJs8zsnsMvzly6MQAsivqLXJY9gyELtUDGaIiIhsKC5SjwenDwAAPP9tEnLLarr1+HllNVjyySFU1ZmK9Nkq5hBNVfMUh8EMERGRjT0wbQDiIvQora7HkxuPdVsvR3WdAXd9ehjZpTUI8tZ1yzHbwwrAREREvZCrRo1/zB8JrYsau07nY+2hzEs+ptEo8PiGBCRmlcLXwxX/WjgagO0mHSl3kInBDBERUY8YGOKNJ2YOBgC8+P1JZBZVXdLx/rH1NDYfz4GrRoUP/t9Ym9c7kzqTFDiZicEMERFRT7ljUj9c3tcflXUGPL4hEUZj1/o7vjqShX/tOAsAePnmERgfEyAP/tg6UVeBsQyDGSIiop6iUavw+ryR8NBq8FtaET7el9bpY/yWVoSnNx0DANw/tT9uHRvZ3c20Sih4oInBDBERUQ/qE+CBv80ZCgBYteUUzuSWd/i+5wsrcc9nh1FvEJg9PBR/bRy2AiB3mdgsZ4bDTERERCRZeHkUpg4OQl2DEY+tT0S9wdjufUqr63HHmkMorqrHiEg9/jF/FNTqno8sOJuJiIiIoFKp8OotI6B3d8XxC6X4d2P+S2vqDUY88H+/IzW/EmF6N/z39nFw12osj9kYZCi4tp3NMJghIiKygxAfN7wwdxgA4F/bz+J4VqnV/YQQWPZdEvaeLYCHVoP/LhqHYB+3FvvZevhHSizudcNMK1euxMSJE+Hh4QFfX98WtycmJmLhwoWIioqCu7s7YmNj8fbbb1vss2nTJlx99dUICgqCj48PJkyYgC1bttiy2URERD3ihpHhmBMXhgajwGPrE1BTb2ixz0d70/DFwQyoVMA/bxuNYeF6q8cyjzGsHceZ2TSYqaurw7x583DfffdZvf3IkSMIDg7G559/jqSkJDz77LNYunQp/vWvf8n77N69G1dffTU2b96MI0eOYNq0abj++utx9OhRWzadiIjI5lQqFf5+43AEeulwJq8Cb/x8yuL2X07mYuXmZADAs9fGYsbQkFaP5eehRaSfOwBg45Gsbm+rglczgEr0wMpRa9aswSOPPIKSkpJ2933ggQeQnJyM7du3t7rPsGHDsGDBAjz//PMdevyysjLo9XqUlpbCx8eno80mIiLqEduSc3HnJ4ehUgFr77oC42MCcDK7DLe+vx9VdQYsvDwKL90UB1U7Yzyf7E/Hsu+SEB3gge2PT4WmGxOEn/vmBD779Tweih+Ix64e1G3HbUtHP78VlzNTWloKf3//Vm83Go0oLy9vc5/a2lqUlZVZ/BARESlVfGwI5o+LhBDAXzcmIq2gUl488soBAXhh7vB2AxkAmDcuEn4erjhfWIWfTuT0QMuVQVHBzP79+7Fu3Trcfffdre7z+uuvo6KiAvPnz291n5dffhl6vV7+iYqKskVziYiIus1z1w1FhK87MouqMfvt3cgurUFMkCfe/cNYuGo69nHtoXXB7RP6AgDe35XardWApaJ5Shxm6nQw8/TTT0OlUrX5k5KS0umGnDhxAnPnzsWyZcswc+ZMq/t88cUXWLFiBdavX4/g4OBWj7V06VKUlpbKP5mZl76gFxERkS15u7nitXkjAAA19Ub4erji40WXQe/h2qnjLJrYF26uahy/UIoD5wq7rX1KLprn0tk7PP7441i8eHGb+8TExHTqmCdPnkR8fDzuvvtu/O1vf7O6z9q1a7FkyRJs2LABM2bMaPN4Op0OOl3PLIVORETUXSb2D8RT1wzBhsOZeOWWEV1aPNLfU4sF46LwyYHz+GDXOUzsH2iDlipLp4OZoKAgBAUFdVsDkpKSMH36dCxatAgrV660us+XX36JO+64A2vXrsWcOXO67bGJiIiU5r6p/XHf1P6XdIwlk2Pw2a/nset0Pk5ml2Fo+KVPfmmazaS8rhmb5sxkZGQgISEBGRkZMBgMSEhIQEJCAioqKgCYhpamTZuGmTNn4rHHHkNOTg5ycnKQn58vH+OLL77A7bffjjfeeAPjx4+X9ykttV5ciIiIqLeL8vfAnBHhAIAPd6d2yzGVPMxk02Dm+eefx+jRo7Fs2TJUVFRg9OjRGD16NA4fPgwA2LhxI/Lz8/H5558jLCxM/rnsssvkY3z44YdoaGjAAw88YLHPww8/bMumExERObR7pphSPv537CKyiqvs3Brbsmkws2bNGgghWvxMnToVALB8+XKrt6enp8vH2Llzp9V91qxZY8umExERObThEXpMGhAIg1Hgo71p3XBEJ5rNRERERI7hnqtMvTNrf8tEcWXdJR2r1w4zERERkf1MGhCIYeE+qK434LNfz9u7OTbDYIaIiMhJqVQq3HOVaWbUmv3pl7QAZVPPjPK6ZhjMEBERObFrh4ci0s8dRZV12HAJC1AK2Hwpxy5jMENEROTEXDRq3DXZlDvzn93nYDAqNyjpKgYzRERETk5agDKjqAo/nrjYpWMwAZiIiIjsxkPrgkUT+wIAPth17pIWoOx1FYCJiIhIGW6fYLYAZWrnF6BU8uAUgxkiIqJeQFqAEgDe332u0/fnMBMRERHZ3ZLJMVCrgN2NC1B2hQJjGQYzREREvYX5ApQfdHIBSk7NJiIiIkWQFqD8/thFZBZ1YgFKDjMRERGREgyP0GPywK4vQMnZTERERGR390wxLXGw7lDHF6BU7iATgxkiIqJe58oBAZ1egFKqTcNhJiIiIrK75gtQVtd1fQFKJWAwQ0RE1AuZL0C58Uhmu/tzmImIiIgUxWIByj1paDAY29y/qWie8saZGMwQERH1UvPHRckLUP6UlNOh+ygvlGEwQ0RE1Gu5azUdXoCSw0xERESkSOYLUO5vYwFKzmYiIiIiRbJYgHJX+0scKDCWYTBDRETU2y2ZHAONWoU9ZwqQlF1qdR8OMxEREZFiRfl7YE5cGADgw93nrO/E2UxERESkZHd3cAFKBcYyDGaIiIio/QUohYIHmhjMEBEREYCmBSjXHspAUbMFKOWieT3dqA5gMENEREQAmhagrKk34rMDrSxAqcBxJgYzREREBMByAcpPDlguQNlGPT27YzBDREREsmuHhyLKv+UClFLOjPL6ZRjMEBERkRnzBSg/3HOuxQKUChxlYjBDREREluaNNS1AmVlUjR9PmBag5DATEREROQyLBSh3p0KIponZKgUONDGYISIiohakBShPXCizWICSw0xERETkEJovQMlhJiIiInI45gtQpuSUAeBsJiIiInIg5gtQZhVXA+AwExERETkYaQFKJWMwQ0RERK2SFqCUcDYTERERORxpAUoAikyaYTBDREREbbpyQACGR/gAAHzcXO3cmpZc7N0AIiIiUjaVSoX/3n4ZtqfkIT422N7NaYHBDBEREbUrVO+GP4zvY+9mWMVhJiIiInJoDGaIiIjIoTGYISIiIofGYIaIiIgcGoMZIiIicmgMZoiIiMihMZghIiIih8ZghoiIiBwagxkiIiJyaAxmiIiIyKExmCEiIiKHxmCGiIiIHBqDGSIiInJovWLVbCEEAKCsrMzOLSEiIqKOkj63pc/x1vSKYKa8vBwAEBUVZeeWEBERUWeVl5dDr9e3ertKtBfuOAGj0Yjs7Gx4e3tDpVLZuzndpqysDFFRUcjMzISPj4+9m2NzPF/n1ZvOFeD5OrPedK6A7c9XCIHy8nKEh4dDrW49M6ZX9Myo1WpERkbauxk24+Pj0yteNBKer/PqTecK8HydWW86V8C259tWj4yECcBERETk0BjMEBERkUNjMOPAdDodli1bBp1OZ++m9Aier/PqTecK8HydWW86V0A559srEoCJiIjIebFnhoiIiBwagxkiIiJyaAxmiIiIyKExmCEiIiKHxmCGiIiIHBqDGTt7+eWXcdlll8Hb2xvBwcG48cYbcerUKYt9pk6dCpVKZfFz7733WuyTkZGBOXPmwMPDA8HBwXjiiSfQ0NBgsc/OnTsxZswY6HQ6DBgwAGvWrLH16VlYvnx5i/MYMmSIfHtNTQ0eeOABBAQEwMvLC7fccgtyc3MtjuEI5ynp27dvi/NVqVR44IEHADj+dd29ezeuv/56hIeHQ6VS4ZtvvrG4XQiB559/HmFhYXB3d8eMGTNw5swZi32Kiorwxz/+ET4+PvD19cWdd96JiooKi32OHTuGyZMnw83NDVFRUVi1alWLtmzYsAFDhgyBm5sb4uLisHnz5h493/r6ejz11FOIi4uDp6cnwsPDcfvttyM7O9viGNaeE6+88orizre9a7t48eIW53HNNddY7OMs1xaA1dexSqXCa6+9Ju/jKNe2I585Pfle/O9//xt9+/aFm5sbxo8fj99++61rJybIrmbNmiVWr14tTpw4IRISEsS1114r+vTpIyoqKuR9rrrqKnHXXXeJixcvyj+lpaXy7Q0NDWL48OFixowZ4ujRo2Lz5s0iMDBQLF26VN7n3LlzwsPDQzz22GPi5MmT4p133hEajUb89NNPPXauy5YtE8OGDbM4j/z8fPn2e++9V0RFRYlt27aJw4cPiyuuuEJMnDjR4c5TkpeXZ3GuW7duFQDEjh07hBCOf103b94snn32WbFp0yYBQHz99dcWt7/yyitCr9eLb775RiQmJoobbrhB9OvXT1RXV8v7XHPNNWLkyJHi119/FXv27BEDBgwQCxculG8vLS0VISEh4o9//KM4ceKE+PLLL4W7u7v44IMP5H327dsnNBqNWLVqlTh58qT429/+JlxdXcXx48d77HxLSkrEjBkzxLp160RKSoo4cOCAuPzyy8XYsWMtjhEdHS1eeOEFi2tu/lpXyvm2d20XLVokrrnmGovzKCoqstjHWa6tEMLiPC9evCg+/vhjoVKpRGpqqryPo1zbjnzm9NR78dq1a4VWqxUff/yxSEpKEnfddZfw9fUVubm5nT4vBjMKk5eXJwCIXbt2yduuuuoq8fDDD7d6n82bNwu1Wi1ycnLkbe+9957w8fERtbW1QgghnnzySTFs2DCL+y1YsEDMmjWre0+gDcuWLRMjR460eltJSYlwdXUVGzZskLclJycLAOLAgQNCCMc5z9Y8/PDDon///sJoNAohnOe6CiFafAAYjUYRGhoqXnvtNXlbSUmJ0Ol04ssvvxRCCHHy5EkBQBw6dEje58cffxQqlUpcuHBBCCHEu+++K/z8/OTzFUKIp556SgwePFj+//z588WcOXMs2jN+/Hhxzz33dOs5mrP2gdfcb7/9JgCI8+fPy9uio6PFm2++2ep9lHi+rQUzc+fObfU+zn5t586dK6ZPn26xzRGvrRAtP3N68r348ssvFw888ID8f4PBIMLDw8XLL7/c6fPgMJPClJaWAgD8/f0ttv/f//0fAgMDMXz4cCxduhRVVVXybQcOHEBcXBxCQkLkbbNmzUJZWRmSkpLkfWbMmGFxzFmzZuHAgQO2OhWrzpw5g/DwcMTExOCPf/wjMjIyAABHjhxBfX29RRuHDBmCPn36yG10pPNsrq6uDp9//jnuuOMOi5XbneW6NpeWloacnByLtun1eowfP97ievr6+mLcuHHyPjNmzIBarcbBgwflfaZMmQKtVivvM2vWLJw6dQrFxcXyPkr8G5SWlkKlUsHX19di+yuvvIKAgACMHj0ar732mkXXvCOd786dOxEcHIzBgwfjvvvuQ2FhoXybM1/b3Nxc/PDDD7jzzjtb3OaI17b5Z05PvRfX1dXhyJEjFvuo1WrMmDGjS+fbK1bNdhRGoxGPPPIIrrzySgwfPlze/oc//AHR0dEIDw/HsWPH8NRTT+HUqVPYtGkTACAnJ8fiSQVA/n9OTk6b+5SVlaG6uhru7u62PDUAwPjx47FmzRoMHjwYFy9exIoVKzB58mScOHECOTk50Gq1Ld74Q0JC2j0H6ba29unJ87Tmm2++QUlJCRYvXixvc5brao3UPmttM297cHCwxe0uLi7w9/e32Kdfv34tjiHd5ufn1+rfQDqGPdTU1OCpp57CwoULLVYSfuihhzBmzBj4+/tj//79WLp0KS5evIh//OMfABznfK+55hrcfPPN6NevH1JTU/HMM89g9uzZOHDgADQajVNf208++QTe3t64+eabLbY74rW19pnTU+/FxcXFMBgMVvdJSUnp9LkwmFGQBx54ACdOnMDevXsttt99993y73FxcQgLC0N8fDxSU1PRv3//nm5ml82ePVv+fcSIERg/fjyio6Oxfv16u33o9pSPPvoIs2fPRnh4uLzNWa4rWaqvr8f8+fMhhMB7771ncdtjjz0m/z5ixAhotVrcc889ePnll+2+tk1n3HbbbfLvcXFxGDFiBPr374+dO3ciPj7eji2zvY8//hh//OMf4ebmZrHdEa9ta585jojDTArx4IMP4vvvv8eOHTsQGRnZ5r7jx48HAJw9exYAEBoa2iLTXPp/aGhom/v4+PjYLZDw9fXFoEGDcPbsWYSGhqKurg4lJSUt2tjeOUi3tbWPPc/z/Pnz+OWXX7BkyZI293OW6wo0tc9a28zbnpeXZ3F7Q0MDioqKuuWaS7f3JCmQOX/+PLZu3WrRK2PN+PHj0dDQgPT0dACOd76SmJgYBAYGWjx3ne3aAsCePXtw6tSpdl/LgPKvbWufOT31XhwYGAiNRtNt58tgxs6EEHjwwQfx9ddfY/v27S26Ia1JSEgAAISFhQEAJkyYgOPHj1u8eUhvpEOHDpX32bZtm8Vxtm7digkTJnTTmXReRUUFUlNTERYWhrFjx8LV1dWijadOnUJGRobcRkc9z9WrVyM4OBhz5sxpcz9nua4A0K9fP4SGhlq0raysDAcPHrS4niUlJThy5Ii8z/bt22E0GuXAbsKECdi9ezfq6+vlfbZu3YrBgwfDz89P3kcJfwMpkDlz5gx++eUXBAQEtHufhIQEqNVqeUjGkc7XXFZWFgoLCy2eu850bSUfffQRxo4di5EjR7a7r1KvbXufOT31XqzVajF27FiLfYxGI7Zt29a18+10yjB1q/vuu0/o9Xqxc+dOiyl9VVVVQgghzp49K1544QVx+PBhkZaWJr799lsRExMjpkyZIh9DmiY3c+ZMkZCQIH766ScRFBRkdZrcE088IZKTk8W///3vHp+y/Pjjj4udO3eKtLQ0sW/fPjFjxgwRGBgo8vLyhBCm6YB9+vQR27dvF4cPHxYTJkwQEyZMcLjzNGcwGESfPn3EU089ZbHdGa5reXm5OHr0qDh69KgAIP7xj3+Io0ePyrN3XnnlFeHr6yu+/fZbcezYMTF37lyrU7NHjx4tDh48KPbu3SsGDhxoMX23pKREhISEiD/96U/ixIkTYu3atcLDw6PFdFYXFxfx+uuvi+TkZLFs2TKbTN9t63zr6urEDTfcICIjI0VCQoLFa1ma3bF//37x5ptvioSEBJGamio+//xzERQUJG6//XbFnW9b51peXi7++te/igMHDoi0tDTxyy+/iDFjxoiBAweKmpoa+RjOcm0lpaWlwsPDQ7z33nst7u9I17a9zxwheu69eO3atUKn04k1a9aIkydPirvvvlv4+vpazJLqKAYzdgbA6s/q1auFEEJkZGSIKVOmCH9/f6HT6cSAAQPEE088YVGPRAgh0tPTxezZs4W7u7sIDAwUjz/+uKivr7fYZ8eOHWLUqFFCq9WKmJgY+TF6yoIFC0RYWJjQarUiIiJCLFiwQJw9e1a+vbq6Wtx///3Cz89PeHh4iJtuuklcvHjR4hiOcJ7mtmzZIgCIU6dOWWx3huu6Y8cOq8/dRYsWCSFM07Ofe+45ERISInQ6nYiPj2/xdygsLBQLFy4UXl5ewsfHR/z5z38W5eXlFvskJiaKSZMmCZ1OJyIiIsQrr7zSoi3r168XgwYNElqtVgwbNkz88MMPPXq+aWlprb6WpbpCR44cEePHjxd6vV64ubmJ2NhY8dJLL1kEAEo537bOtaqqSsycOVMEBQUJV1dXER0dLe66664WH0DOcm0lH3zwgXB3dxclJSUt7u9I17a9zxwheva9+J133hF9+vQRWq1WXH755eLXX3/t0nmpGk+OiIiIyCExZ4aIiIgcGoMZIiIicmgMZoiIiMihMZghIiIih8ZghoiIiBwagxkiIiJyaAxmiIiIyKExmCEiIiKHxmCGiIiIHBqDGSIiInJoDGaIiIjIof1/tZeLyNQSjWMAAAAASUVORK5CYII=",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "df_w_bins[df_w_bins[\"MedHouseVal_bins\"] == 5][\"Longitude\"].plot(\n",
-    "    title=\"Longitude feature values before calibration\"\n",
-    ");"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "144a8779",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAHICAYAAAC772uFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB/DUlEQVR4nO3dd3iTVfsH8G+SNunem5bSltECZZUhexQZouBi6U9BRNyKOHExFAeu19etr4KKyhLBzd4gslqgpay2tIxuundyfn+kedq06aRpRr+f68oFffLkyXky75xzn/vIhBACRERERBZKbuoGEBEREV0PBjNERERk0RjMEBERkUVjMENEREQWjcEMERERWTQGM0RERGTRGMwQERGRRWMwQ0RERBaNwQwRERFZNAYzZLF27doFmUyGXbt2tcn9jRo1CqNGjWrVY37//fcIDw+Hra0t3NzcWvXY7dHKlSshk8mQnJxs6qZcl8rKSjz33HMICgqCXC7HrbfeauomXbfk5GTIZDKsXLlS2rZ48WLIZDK9/WQyGR577LE2bl1dnTp1wuzZs03dDGoiBjNUL90Xw5EjR0zdlCb78ccf8Z///MfUzWiShIQEzJ49G2FhYfjqq6/w5ZdfGuV+Dhw4gMWLFyM3N9cox6fW98033+Cdd97BnXfeiW+//RZPPfUU4uPjsXjxYosP1MwJ3xvWw8bUDSBqqREjRqCkpARKpVLa9uOPP+LUqVOYP3++6RrWRLt27YJGo8GHH36Izp07G+1+Dhw4gCVLlmD27Nns/bEQO3bsQIcOHfDBBx9I29avX48lS5Zg1KhR6NSpk+ka14pefvllvPDCCya7/4beG2fOnIFczt/7loLPFFksuVwOOzs7i/3AycjIAACLDTCKiopM3QSrlZGR0WavC1M+jzY2NrCzs2u147XmuahUKtja2rba8ci4LPNbgMzK8ePHMXHiRLi4uMDJyQnR0dH4559/9PbRDVnt378fCxYsgLe3NxwdHXHbbbchMzNTb1+NRoPFixcjICAADg4OGD16NOLj4+uMYdfOmRk1ahT++OMPXLx4ETKZDDKZTPoFW18uRX15N19++SXCwsJgb2+PgQMHYu/evQbPvaysDIsWLULnzp2hUqkQFBSE5557DmVlZQ0+Zp06dcKiRYsAAN7e3pDJZFi8eLF0/V9//YXhw4fD0dERzs7OmDRpEuLi4vSOceLECcyePRuhoaGws7ODn58f5syZg+zsbGmfxYsX49lnnwUAhISESI9LcnKywRwGndrt0eU2xMfH46677oK7uzuGDRsmXb9q1SpERUXB3t4eHh4emDFjBlJTUxt8DNavXw+ZTIbdu3fXue6LL76ATCbDqVOnmnyu9al9LjqGciJyc3Mxf/58BAUFQaVSoXPnznj77beh0Wj09lu9ejWioqLg7OwMFxcXREZG4sMPP2y0Le+++y6GDBkCT09P2NvbIyoqCuvXr5eu1z0nO3fuRFxcnPR8rVy5ElOnTgUAjB49Wtpe83XblNfM7Nmz4eTkhAsXLuCmm26Cs7Mz7r777gbbfPnyZdx///0ICAiASqVCSEgIHn74YZSXlwMAcnJy8MwzzyAyMhJOTk5wcXHBxIkTERsb2+jjYShnRueHH35At27dYGdnh6ioKOzZs8fgbQ29Jq/3vQEYfn0kJiZi6tSp8PDwgIODA2644Qb88ccfevvoPlPWrl2LZcuWITAwEHZ2doiOjsb58+cbfUyoZTjMRNclLi4Ow4cPh4uLC5577jnY2triiy++wKhRo7B7924MGjRIb//HH38c7u7uWLRoEZKTk/Gf//wHjz32GNasWSPts3DhQixfvhy33HILxo8fj9jYWIwfPx6lpaUNtuWll15CXl4eLl26JHXPOzk5Nfucvv76azz44IMYMmQI5s+fj8TEREyePBkeHh4ICgqS9tNoNJg8eTL27duHefPmISIiAidPnsQHH3yAs2fPYuPGjfXex3/+8x989913+OWXX/DZZ5/ByckJvXr1AqBNCp41axbGjx+Pt99+G8XFxfjss88wbNgwHD9+XArQtm7disTERNx3333w8/NDXFwcvvzyS8TFxeGff/6BTCbD7bffjrNnz+Knn37CBx98AC8vLwDaAKp2ENkUU6dORZcuXfDGG29ACAEAWLZsGV555RVMmzYNc+fORWZmJj766COMGDECx48fr7eHYdKkSXBycsLatWsxcuRIvevWrFmDHj16oGfPnk0+1+tVXFyMkSNH4vLly3jwwQfRsWNHHDhwAAsXLsTVq1elXKytW7di5syZiI6Oxttvvw0AOH36NPbv348nn3yywfv48MMPMXnyZNx9990oLy/H6tWrMXXqVPz++++YNGkSvL298f3332PZsmUoLCzEm2++CQDo0qULnnjiCfz3v//Fiy++iIiICACQ/m3qawbQJhePHz8ew4YNw7vvvgsHB4d623vlyhUMHDgQubm5mDdvHsLDw3H58mWsX78excXFUCqVSExMxMaNGzF16lSEhIQgPT0dX3zxBUaOHIn4+HgEBAQ0+7nYvXs31qxZgyeeeAIqlQqffvopJkyYgH///Vd6TegYek1e73vDkPT0dAwZMgTFxcV44okn4OnpiW+//RaTJ0/G+vXrcdttt+nt/9Zbb0Eul+OZZ55BXl4eli9fjrvvvhuHDh1q9uNBTSCI6rFixQoBQBw+fLjefW699VahVCrFhQsXpG1XrlwRzs7OYsSIEXWONXbsWKHRaKTtTz31lFAoFCI3N1cIIURaWpqwsbERt956q979LF68WAAQs2bNkrbt3LlTABA7d+6Utk2aNEkEBwfXey5JSUl622sfo7y8XPj4+Ig+ffqIsrIyab8vv/xSABAjR46Utn3//fdCLpeLvXv36h3z888/FwDE/v37DT5mOosWLRIARGZmprStoKBAuLm5iQceeEBv37S0NOHq6qq3vbi4uM4xf/rpJwFA7NmzR9r2zjvvGDz3pKQkAUCsWLGiznEAiEWLFtVp68yZM/X2S05OFgqFQixbtkxv+8mTJ4WNjU2d7bXNnDlT+Pj4iMrKSmnb1atXhVwuF0uXLm32uRp6nmufi05wcLDe6+m1114Tjo6O4uzZs3r7vfDCC0KhUIiUlBQhhBBPPvmkcHFx0WtzU9U+j/LyctGzZ08xZswYve0jR44UPXr00Nu2bt26Oq93IZr3mpk1a5YAIF544YUmtffee+8Vcrnc4GeA7n1cWloq1Gq13nVJSUlCpVLpPYeGXm+611VNAAQAceTIEWnbxYsXhZ2dnbjtttvq3Lb2a1KI639vCFH39TF//nwBQO/9XlBQIEJCQkSnTp2kx0D3mRIREaH3GfLhhx8KAOLkyZN17ouuH4eZqMXUajW2bNmCW2+9FaGhodJ2f39/3HXXXdi3bx/y8/P1bjNv3jy9X9HDhw+HWq3GxYsXAQDbt29HZWUlHnnkEb3bPf7440Y8k2pHjhxBRkYGHnroIb3E4tmzZ8PV1VVv33Xr1iEiIgLh4eHIysqSLmPGjAEA7Ny5s9n3v3XrVuTm5mLmzJl6x1QoFBg0aJDeMe3t7aX/l5aWIisrCzfccAMA4NixY82+76Z46KGH9P7esGEDNBoNpk2bptdePz8/dOnSpdHHYPr06cjIyNAbLlm/fj00Gg2mT58ubWuLc123bh2GDx8Od3d3vXMZO3Ys1Gq1NMzh5uaGoqIibN26tdn3UfM8rl27hry8PAwfPvy6zqE5rxmdhx9+uNHjajQabNy4Ebfccgv69+9f53rd+1ilUkl5a2q1GtnZ2XByckK3bt1afF6DBw9GVFSU9HfHjh0xZcoUbN68GWq1Wm/f2q9JwDivlz///BMDBw7UG151cnLCvHnzkJycjPj4eL3977vvPr3PkOHDhwPQDlVR6+MwE7VYZmYmiouL0a1btzrXRUREQKPRIDU1FT169JC2d+zYUW8/d3d3ANoPdgBSUFN7do+Hh4e0rzHp7r9Lly56221tbfUCNgA4d+4cTp8+XW+3tC7BtznOnTsHAFJAVJuLi4v0/5ycHCxZsgSrV6+uc195eXnNvu+mCAkJ0fv73LlzEELUebx0GkugnDBhAlxdXbFmzRpER0cD0A4x9enTB127dpX2a4tzPXfuHE6cONHo8/nII49g7dq1mDhxIjp06IBx48Zh2rRpmDBhQqP38fvvv+P1119HTEyMXl7V9QyTNec1A2iTbgMDAxs9bmZmJvLz8+sM69Smm5H36aefIikpSS/Y8PT0bPR+DDH0euratSuKi4uRmZkJPz8/aXvt1yRgnNfLxYsX6wybA9VDfRcvXtR7rBr7rKPWxWCG2pRCoTC4XVSNdRtLfV8WtX/lNYdGo0FkZCTef/99g9fXzK9pzjEBbQ5EzQ9sHRub6rfstGnTcODAATz77LPo06cPnJycoNFoMGHChDoJq4a05DGp+YtX116ZTIa//vrL4HPbWM6SSqXCrbfeil9++QWffvop0tPTsX//frzxxht6+13vuRpS+zw1Gg1uvPFGPPfccwb31wVXPj4+iImJwebNm/HXX3/hr7/+wooVK3Dvvffi22+/rff+9u7di8mTJ2PEiBH49NNP4e/vD1tbW6xYsQI//vhji85B126gaa8ZQL8npTW88cYbeOWVVzBnzhy89tpr8PDwgFwux/z581v83DRH7dckYJzXS3OZ6rOuvWIwQy3m7e0NBwcHnDlzps51CQkJkMvlzf5CDw4OBgCcP39e7xdXdnZ2k37R1PcFrftVVLs4lq4npvb9nzt3Tu+XbkVFBZKSktC7d29pW1hYGGJjYxEdHd0qCai6YwLaL8yxY8fWu9+1a9ewfft2LFmyBK+++qq0XfcrvabrfUwaa68QAiEhIXo9Kc0xffp0fPvtt9i+fTtOnz4NIYTeEFNzztUQd3f3OudYXl6Oq1ev1jmXwsLCBh93HaVSiVtuuQW33HILNBoNHnnkEXzxxRd45ZVX6q0Z9PPPP8POzg6bN2+GSqWStq9YsaJJ51Hf89jU10xzeXt7w8XFRZpRVp/169dj9OjR+Prrr/W25+bmSkm1zWXouT179iwcHBzq7TnTaY33hiHBwcH1ftbprifTYc4MtZhCocC4ceOwadMmvSnP6enp+PHHHzFs2LA6XdyNiY6Oho2NDT777DO97R9//HGTbu/o6GiwG1n3gV9zeqdara5Tdbd///7w9vbG559/Lk09BbRTu2t/IU6bNg2XL1/GV199Vef+SkpKWlTzYvz48XBxccEbb7yBioqKOtfrZiDpfvXV/pVnqPqxo6MjgLpBi4uLC7y8vOpMef3000+b3N7bb78dCoUCS5YsqdMWIUSTpk6PHTsWHh4eWLNmDdasWYOBAwfqBbLNOVdDwsLC6pzjl19+WadnZtq0aTh48CA2b95c5xi5ubmorKwEgDrnJJfLpZloDU3JVygUkMlkevebnJzc4Ky3mup7Hpv6mmku3TIKv/32m8Eq4LrnQ6FQ1Hlu1q1bh8uXL7fofgHg4MGDerktqamp2LRpE8aNG1dvj4dOa7w3DLnpppvw77//4uDBg9K2oqIifPnll+jUqRO6d+/e6DHIeNgzQ4365ptv8Pfff9fZ/uSTT+L111/H1q1bMWzYMDzyyCOwsbHBF198gbKyMixfvrzZ9+Xr64snn3wS7733HiZPnowJEyYgNjYWf/31F7y8vBr9JRUVFYU1a9ZgwYIFGDBgAJycnHDLLbegR48euOGGG7Bw4ULk5OTAw8MDq1evlr6gdGxtbfH666/jwQcfxJgxYzB9+nQkJSVhxYoVdXJm7rnnHqxduxYPPfQQdu7ciaFDh0KtViMhIQFr167F5s2bDSZONsTFxQWfffYZ7rnnHvTr1w8zZsyAt7c3UlJS8Mcff2Do0KH4+OOP4eLighEjRmD58uWoqKhAhw4dsGXLFiQlJRl8TADt1PUZM2bA1tYWt9xyCxwdHTF37ly89dZbmDt3Lvr37489e/bg7NmzTW5vWFgYXn/9dSxcuBDJycm49dZb4ezsjKSkJPzyyy+YN28ennnmmQaPYWtri9tvvx2rV69GUVER3n333TqPSVPP1ZC5c+fioYcewh133IEbb7wRsbGx2Lx5c51eg2effRa//vorbr75ZsyePRtRUVEoKirCyZMnsX79eiQnJ8PLywtz585FTk4OxowZg8DAQFy8eBEfffQR+vTpI+VPGDJp0iS8//77mDBhAu666y5kZGTgk08+QefOnXHixIlGz6NPnz5QKBR4++23kZeXB5VKhTFjxsDHx6dJr5mWeOONN7BlyxaMHDlSKj9w9epVrFu3Dvv27YObmxtuvvlmLF26FPfddx+GDBmCkydP4ocffqjzfmmOnj17Yvz48XpTswFgyZIljd62td4btb3wwgv46aefMHHiRDzxxBPw8PDAt99+i6SkJPz8888WW7zTaphmEhVZAt001/ouqampQgghjh07JsaPHy+cnJyEg4ODGD16tDhw4IDBY9We4mloenVlZaV45ZVXhJ+fn7C3txdjxowRp0+fFp6enuKhhx5q8LaFhYXirrvuEm5ubgKA3jTtCxcuiLFjxwqVSiV8fX3Fiy++KLZu3Wpwuuunn34qQkJChEqlEv379xd79uwRI0eO1JuaLYR2au3bb78tevToIVQqlXB3dxdRUVFiyZIlIi8vr8HH19DU7JrnNn78eOHq6irs7OxEWFiYmD17tt501UuXLonbbrtNuLm5CVdXVzF16lRx5coVg1ORX3vtNdGhQwchl8v1pqIWFxeL+++/X7i6ugpnZ2cxbdo0kZGRUe/UbENtFUKIn3/+WQwbNkw4OjoKR0dHER4eLh599FFx5syZBh8DHd3zIJPJpNdVTU09V0NTs9VqtXj++eeFl5eXcHBwEOPHjxfnz5+vM/VWCO1U24ULF4rOnTsLpVIpvLy8xJAhQ8S7774rysvLhRBCrF+/XowbN074+PgIpVIpOnbsKB588EFx9erVRs/z66+/Fl26dBEqlUqEh4eLFStWGJyebGhqthBCfPXVVyI0NFQoFIo6r9umvGZmzZolHB0dG21nTRcvXhT33nuv8Pb2FiqVSoSGhopHH31UmnZcWloqnn76aeHv7y/s7e3F0KFDxcGDB+u8X5ozNfvRRx8Vq1atkh6rvn371nmPNvSabI33hqHXx4ULF8Sdd94p3NzchJ2dnRg4cKD4/fff9fbRfS6tW7dOb3tDpRDo+smEYDYSmb/c3Fy4u7vj9ddfx0svvWTq5hARkRlhvxiZnZKSkjrbdOPdo0aNatvGEBGR2WPODJmdNWvWYOXKlbjpppvg5OSEffv24aeffsK4ceMwdOhQUzePiIjMDIMZMju9evWCjY0Nli9fjvz8fCkp+PXXXzd104iIyAwxZ4aIiIgsGnNmiIiIyKIxmCEiIiKLxmCG2o3ly5cjPDzcqGuzrFy5EjKZTK8isrF16tQJs2fPlv7etWsXZDKZ3krUTaW77fr161uvgc3QqVMn3HzzzSa577Yye/ZsdOrUydTNqNf1vH5aU3x8PGxsbBpdToEIYDBD7UR+fj7efvttPP/886zU2Up+/PHHJi8pQNRc3bt3x6RJk/TWVyKqD2czUbvwzTffoLKyEjNnzjR1U4xuxIgRKCkpgVKpNOr9/Pjjjzh16hTmz59v1PuxNl999VWbrdxs6R566CHcdNNNuHDhgrS+GpEh/IlK7cKKFSswefJk2NnZmbopRieXy2FnZ8ceKDNla2urt2o21W/s2LFwd3fHt99+a+qmkJnjpx1ZvaSkJJw4cQJjx46tc11RURGefvppBAUFQaVSoVu3bnj33XfrrLgrk8nw2GOPYePGjejZsydUKhV69OhhcAHOmmbNmgUvLy+DqxmPGzcO3bp1a/D2586dwx133AE/Pz/Y2dkhMDAQM2bMMLgyuE59OQ+ffPIJQkNDYW9vj4EDB2Lv3r0YNWqUwarKGo0Gy5YtQ2BgIOzs7BAdHY3z589L148aNQp//PEHLl68CJlMBplM1qQ8kFWrVmHgwIFwcHCAu7s7RowYgS1bttTZb9++fRg4cCDs7OwQGhqK7777Tu/6nJwcPPPMM4iMjISTkxNcXFwwceJExMbGGnws1q5d2+D5NPcxKisrw6JFi9C5c2eoVCoEBQXhueeea3DVbJ3aOTPJycmQyWR499138eWXXyIsLAwqlQoDBgzA4cOHGzzWkSNHIJPJDH7Zb968GTKZDL///jsA4OLFi3jkkUfQrVs32Nvbw9PTE1OnTm1SflftvCyd63lsdAvUurm5wcnJCd26dcOLL76ot4+trS1GjRqFTZs2NdpGat84zERW78CBAwCAfv366W0XQmDy5MnYuXMn7r//fvTp0webN2/Gs88+i8uXL+ODDz7Q23/fvn3YsGEDHnnkETg7O+O///0v7rjjDqSkpMDT09Pgfd9zzz347rvvsHnzZr3E1rS0NOzYsQOLFi2qt93l5eUYP348ysrK8Pjjj8PPzw+XL1/G77//jtzcXLi6ujb5Mfjss8/w2GOPYfjw4XjqqaekFa7d3d0RGBhYZ/+33noLcrkczzzzDPLy8rB8+XLcfffdOHToEADtKsN5eXm4dOmS9Dg5OTk12IYlS5Zg8eLFGDJkCJYuXQqlUolDhw5hx44dGDdunLTf+fPnceedd+L+++/HrFmz8M0330irWPfo0QMAkJiYiI0bN2Lq1KkICQlBeno6vvjiC4wcORLx8fEICAho1vk05zHSaDSYPHky9u3bJ60kffLkSXzwwQc4e/YsNm7c2OTnpaYff/wRBQUFePDBByGTybB8+XLcfvvtSExMhK2trcHb9O/fH6GhoVi7di1mzZqld92aNWvg7u6O8ePHAwAOHz6MAwcOYMaMGQgMDERycjI+++wzjBo1CvHx8XBwcGhRu2tq6mMTFxeHm2++Gb169cLSpUuhUqlw/vx57N+/v84xo6KisGnTJuTn58PFxeW620hWypSrXBK1hZdfflkAEAUFBXrbN27cKACI119/XW/7nXfeKWQymTh//ry0DYBQKpV622JjYwUA8dFHH0nbaq/arFarRWBgoJg+fbrefbz//vtCJpOJxMTEett9/Phxg6vv1lZ7dd/aq4mXlZUJT09PMWDAAFFRUSHtt3LlSgFAb2Vj3W0jIiKkVZGFEOLDDz8UAMTJkyelbZMmTdJblbwh586dE3K5XNx2221CrVbrXafRaPTOBYDYs2ePtC0jI0OoVCrx9NNPS9tKS0vrHCcpKUmoVCqxdOnSZp9Pcx6j77//XsjlcrF37169+//8888FALF///4GH4tZs2bpPW661ZQ9PT1FTk6OtH3Tpk0CgPjtt98aPN7ChQuFra2t3m3LysqEm5ubmDNnjrStuLi4zm0PHjwoAIjvvvtO2mZoNXpDK0gLIeqsjN3Ux+aDDz5ocBX2mn788UcBQBw6dKjRfan94jATWb3s7GzY2NjU6Tn4888/oVAo8MQTT+htf/rppyGEwF9//aW3fezYsXpJiL169YKLiwsSExPrvW+5XI67774bv/76KwoKCqTtP/zwA4YMGYKQkJB6b6vredm8eTOKi4sbP9F6HDlyBNnZ2XjggQdgY1PdGXv33XfD3d3d4G3uu+8+vQTi4cOHA0CD59qQjRs3QqPR4NVXX62TyyOTyfT+7t69u3R/AODt7Y1u3brp3bdKpZKOo1arkZ2dLQ1VHDt2rNnn05zHaN26dYiIiEB4eDiysrKky5gxYwAAO3fubPoDU8P06dP17qupj/n06dNRUVGBDRs2SNu2bNmC3NxcTJ8+Xdpmb28v/b+iogLZ2dno3Lkz3NzcDD5mLdHUx8bNzQ0AsGnTpkaToXWPSVZWVqu0kawTgxlqty5evIiAgAA4OzvrbY+IiJCur6ljx451juHu7o5r1641eD/33nsvSkpK8MsvvwAAzpw5g6NHj+Kee+5p8HYhISFYsGAB/ve//8HLywvjx4/HJ5980mC+jCG68+jcubPedhsbm3rzXGqfq+4LpbFzrc+FCxcgl8vRvXv3RvdtyuOs0WjwwQcfoEuXLlCpVPDy8oK3tzdOnDhh8PFp7Hya8xidO3cOcXFx8Pb21rt07doVAJCRkdHoORrS0se8d+/eCA8Px5o1a6Rta9asgZeXlxREANrV6F999VUpP0z3mOXm5jb7NVWfpj4206dPx9ChQzF37lz4+vpixowZWLt2rcHARlTlr9UOeolqYs4MWT1PT09UVlaioKCgTuDSHAqFwuB20cjyZt27d0dUVBRWrVqFe++9F6tWrYJSqcS0adMavc/33nsPs2fPxqZNm7BlyxY88cQTePPNN/HPP/8YzHVpLS0917a67zfeeAOvvPIK5syZg9deew0eHh6Qy+WYP3++wS/E1jwfjUaDyMhIvP/++wavDwoKavYxgetr4/Tp07Fs2TJkZWXB2dkZv/76K2bOnKnXy/T4449jxYoVmD9/PgYPHgxXV1fIZDLMmDGj0d6R+gIJtVqt1+6mPjb29vbYs2cPdu7ciT/++AN///031qxZgzFjxmDLli16x9QFc15eXo0+DtR+MZghqxceHg5AO6upV69e0vbg4GBs27atTpCTkJAgXd9a7r33XixYsABXr17Fjz/+iEmTJtU7xFNbZGQkIiMj8fLLL+PAgQMYOnQoPv/88yavIq47j/Pnz2P06NHS9srKSiQnJ+s9Js3RnF/KYWFh0Gg0iI+PR58+fVp0fzWtX78eo0ePxtdff623PTc3t0Vfes15jMLCwhAbG4vo6Giz6S2YPn06lixZgp9//hm+vr7Iz8/HjBkz9PZZv349Zs2ahffee0/aVlpaitzc3EaP7+7ubnC/ixcvIjQ0VPq7OY+NXC5HdHQ0oqOj8f777+ONN97ASy+9hJ07d+rNPExKSoJcLpd6d4gM4TATWb3BgwcD0OZF1HTTTTdBrVbj448/1tv+wQcfQCaTYeLEia3WhpkzZ0Imk+HJJ59EYmIi/u///q/R2+Tn56OyslJvW2RkJORyeZOmAOv0798fnp6e+Oqrr/SO98MPP7R42AgAHB0dmzw8ceutt0Iul2Pp0qV1egFa0juiUCjq3G7dunW4fPlys48FNO8xmjZtGi5fvoyvvvqqznFKSkpQVFTUojZcj4iICERGRmLNmjVYs2YN/P39MWLECL19DD1mH330EdRqdaPHDwsLwz///IPy8nJp2++//47U1FS9/Zr62OTk5NS5Xhfk1n5tHz16FD169GjW7D1qf9gzQ1YvNDQUPXv2xLZt2zBnzhxp+y233ILRo0fjpZdeQnJyMnr37o0tW7Zg06ZNmD9/fqtWHPX29saECROwbt06uLm5YdKkSY3eZseOHXjssccwdepUdO3aFZWVlfj++++hUChwxx13NPm+lUolFi9ejMcffxxjxozBtGnTkJycjJUrVyIsLKzFvQtRUVFYs2YNFixYgAEDBsDJyQm33HKLwX07d+6Ml156Ca+99hqGDx+O22+/HSqVCocPH0ZAQADefPPNZt33zTffjKVLl+K+++7DkCFDcPLkSfzwww96vQTN0ZzH6J577sHatWvx0EMPYefOnRg6dCjUajUSEhKwdu1abN68Gf37929RO67H9OnT8eqrr8LOzg73339/nUTrm2++Gd9//z1cXV3RvXt3HDx4ENu2bau3rEBNc+fOxfr16zFhwgRMmzYNFy5cwKpVq+q8R5r62CxduhR79uzBpEmTEBwcjIyMDHz66acIDAzEsGHDpONVVFRg9+7deOSRR1rnQSLrZappVERt6f333xdOTk51pqcWFBSIp556SgQEBAhbW1vRpUsX8c477+hNFxZCOzX70UcfrXPc2lNWa0/Nrmnt2rUCgJg3b16T2pyYmCjmzJkjwsLChJ2dnfDw8BCjR48W27Zta7ANhqbWCiHEf//7XxEcHCxUKpUYOHCg2L9/v4iKihITJkyoc9va08F104dXrFghbSssLBR33XWXcHNzEwCaNE37m2++EX379hUqlUq4u7uLkSNHiq1bt+qdy6RJk+rcrvYU4NLSUvH0008Lf39/YW9vL4YOHSoOHjxYZ7/mnE9THyMhhCgvLxdvv/226NGjh3QuUVFRYsmSJSIvL6/Bx6C+qdnvvPNOnX0BiEWLFjV4PJ1z584JAAKA2LdvX53rr127Ju677z7h5eUlnJycxPjx40VCQkKTXz/vvfee6NChg1CpVGLo0KHiyJEjdR5vIZr22Gzfvl1MmTJFBAQECKVSKQICAsTMmTPF2bNn9Y71119/CQDi3LlzTXoMqP2SCdEGGX1EJpaXl4fQ0FAsX74c999/v0nasGnTJtx6663Ys2eP3tRjU9FoNPD29sbtt99ucFiA+BiZ2q233gqZTCbNBCSqD3NmqF1wdXXFc889h3feecdki/x99dVXCA0N1etGbyulpaV18iW+++475OTkGFzOoD3iY2ReTp8+jd9//x2vvfaaqZtCFoA9M0RGtnr1apw4cQJvvvkmPvzwwzpF+trCrl278NRTT2Hq1Knw9PTEsWPH8PXXXyMiIgJHjx41+grbloCPEZHlYjBDZGQymQxOTk6YPn06Pv/8c73aH20lOTkZTzzxBP7991/k5OTAw8MDN910E9566y34+Pi0eXvMER8jIsvFYIaIiIgsGnNmiIiIyKIxmCEiIiKL1i6K5mk0Gly5cgXOzs5mU36ciIiIGiaEQEFBAQICAuoUgqypXQQzV65cafHib0RERGRaqampDS6u2y6CGd0igqmpqXBxcTFxa4iIiKgp8vPzERQUpLcYsCHtIpjRDS25uLgwmCEiIrIwja7C3kbtICIiIjIKBjNERERk0RjMEBERkUVjMENEREQWjcEMERERWTQGM0RERGTRGMwQERGRRWMwQ0RERBaNwQwRERFZNAYzREREZNEYzBAREZFFYzBDREREFo3BjJVaeyQV+89nmboZRERERtcuVs1ub+Kv5OO59ScAAMlvTTJxa4iIiIyLPTNWKC2/xNRNICIiajMMZqyQTCYzdROIiIjaDIMZKyRnMENERO0IgxkrJK8RywghTNcQIiKiNsBgxgrV7JnRMJYhIiIrx2DGCtUcZarUaEzXECIiojbAYMYK1eyZqVSza4aIiKwbgxkrpBfMcJyJiIisHIMZK1RzmEnNYIaIiKwcgxkrx5wZIiKydgxmrJCmRm8Me2aIiMjaMZixQjXjFyYAExGRtWMwY4VqFspjzwwREVk7BjNWSK9nhsEMERFZOQYzVkjDnhkiImpHGMxYoZrBTIWas5mIiMi6MZixQjXXlmTPDBERWTsGM1aoZgDDnBkiIrJ2DGasEHNmiIioPWEwY4X0ZzMxZ4aIiKwbgxkrxDozRETUnjCYsUKsM0NERO0JgxkrpJczw+UMiIjIyjGYsUI1gxn2zBARkbUzWjCzbNkyDBkyBA4ODnBzczO4zxNPPIGoqCioVCr06dPH4D5CCLz77rvo2rUrVCoVOnTogGXLlhmr2VZBMAGYiIjaERtjHbi8vBxTp07F4MGD8fXXX9e735w5c3Do0CGcOHHC4PVPPvkktmzZgnfffReRkZHIyclBTk6OsZptFTg1m4iI2hOjBTNLliwBAKxcubLeff773/8CADIzMw0GM6dPn8Znn32GU6dOoVu3bgCAkJCQ1m+sldErmsecGSIisnJmnTPz22+/ITQ0FL///jtCQkLQqVMnzJ07t9GembKyMuTn5+td2hMuZ0BERO2JWQcziYmJuHjxItatW4fvvvsOK1euxNGjR3HnnXc2eLs333wTrq6u0iUoKKiNWmwemABMRETtSbOCmRdeeAEymazBS0JCQqs1TqPRoKysDN999x2GDx+OUaNG4euvv8bOnTtx5syZem+3cOFC5OXlSZfU1NRWa5M5U2sE8ksr9OrMqJkATEREVq5ZOTNPP/00Zs+e3eA+oaGh19MePf7+/rCxsUHXrl2lbREREQCAlJQUKY+mNpVKBZVK1WrtsBS3f3YAsam5mD+2i7SNPTNERGTtmhXMeHt7w9vb21htqWPo0KGorKzEhQsXEBYWBgA4e/YsACA4OLjN2mEpYlNzAQA7EzKkbcyZISIia2e02UwpKSnIyclBSkoK1Go1YmJiAACdO3eGk5MTAOD8+fMoLCxEWloaSkpKpH26d+8OpVKJsWPHol+/fpgzZw7+85//QKPR4NFHH8WNN96o11tDQGmFWvq/i72t9P8KzmYiIgPir+TjoVVHMWdoJ8weylmiZNmMFsy8+uqr+Pbbb6W/+/btCwDYuXMnRo0aBQCYO3cudu/eXWefpKQkdOrUCXK5HL/99hsef/xxjBgxAo6Ojpg4cSLee+89YzXbYmUXlUv/L6+szpNhzgwRGXLTf/cCABb/Fs9ghiye0YKZlStXNlhjBgB27drV6HECAgLw888/t06jrFhWQZn0/8u5JdL/mTNDRLVx+JmsjVlPzaamyyqsDmau5pVK/+eHFhHVVlapbnwnIgvCYMZK1Axm9CoAM5gholpKK/SHn4Xg5wRZNgYzViKzxjBTTeyZIaLaak4YADhRgCwfgxkrkVVYbnA712YiotrKKvV7ZsrVnChAlo3BjJXILKyvZ4YfUkSkr3bPTHklPyfIsjGYsRJZ9QwzMWeGiGqrHcwwIZgsHYMZK1FfzwyHmYiottoJwOyZIUvHYMZK6HpmOrjZ621nzwwR1VZaWbtnhsEMWTYGM1agrFKN/NJKAECvQFe965gzQ0S1lTFnhqwMgxkrkF01k8lWIUOEv4vedeyZIaLaavfEMGeGLB2DGSugK5jn6aiqM8zEOjNEVFvdBGD2zJBlYzBjBXQF87yclejgzpwZImoYE4DJ2jCYsQK6nhlvJ/bMEFHj2DND1obBjBXQVf/1clLB18UOMln1deyZIaLa2DND1obBjBWoHmZSQWkjh4+zSrqOs5mIqDZOzSZrw2DGCuiGmbyctEFMQI2hJi4gR0S1lbFnhqwMgxkrIPXMOCkB6AczzJkhotpq98yUc2o2WTgGM1ZASgCuGl4a3tlLuo45M0RUGxOAydowmLECugRg76phphkDO+Lz/+sHgDkzRFQXh5nI2jCYsXDllRrklVQAqM6ZAQAnlS0ALjRJRHWxZ4asDYMZC5ddpB1ispHL4GpvK21XyLXzs5kzQ0S11cmZUTOYIcvGYMbC6ZJ/PZ2UkMurC8zYKBjMEJFhtevM1F54ksjSMJixcLWTf3V0PTNMACai2movLMmeGbJ0DGYsXFZBdfXfmmx0wQw/pIioljo9M8yZIQvHYMbCZdYqmKfDnhkiqg8TgMnaMJixcLWr/+rYyLVPLXNmiKg2rs1E1obBjIWrXf1XR5cAzJ4ZIqqtdsIve2bI0jGYsXD1JQDbcGo2EdVDNzX7xZvCAXA5A7J8DGYsXO3qvzrVOTP8xUVE1dQaIS1A62KnrU3FYSaydAxmLJyUM1OnZ4Y5M0RUV81p2bpCmxxmIkvHYMaClVdqkFtcdykDgLOZiMiwmsm/zuyZISvBYMaC6ZYyUMhlcKuxlAFQnTMjBKBhQENEVXTTsm0VMtgrtV8B7JkhS8dgxoLpCuZ5OuovZQAACkX13xXMmyGiKrpgxs5GAaVCAYA9M2T5GMxYsPpmMgHVPTMA82aIqJpumEllq4DKVtczw9lMZNkYzFiw+qr/AtU5MwDzZoiomm5atp2tHEqF9iuAPTNk6RjMWLD6qv8CgK28+qlVqxnMEJFWWVXPjF2NnhkuNEmWjsGMBZOq/zor61wnl8sgq+qcYc8MEekY6pmpUAtOFCCLxmDGgtVXME+HVYCJqLaymgnANtVfAeydIUvGYMaCZRXUnwAMsAowEdVVnQAsh8pGIW0vq+DnBFkuBjMWrKGcGYBVgImorppTs21rlHAoU3NGE1kuBjMWrLFgRtczU8EEYCKqIgUztgrIZDKobDijiSwfgxkLVaHW4Jq0lEHdBGCAOTNEVJeu2q9uJpMub4ZVgMmSMZixUNlVyb8KuQzuDoaDGebMEFFtpTWmZgNgzwxZBQYzFko3xGRoKQMd9swQUW3S1GwbXTCj/Zc9M2TJGMxYqIaq/+rYVNWQYJ0ZItLR5czUHmZizwxZMgYzFipLKpjXQDDDnhkiqkUaZrLRH2bi+kxkyRjMWKjqnhnD+TJAjZwZzmYioipS0bxaPTMl5QxmyHIxmLFQWQUNV/8FqoMZ9swQkU71cgbanpkQL0cAwLGUXFM1iei6MZixULoE4Pqq/wKAjYKzmYhIX/VCk9qP/zHhPgCAHQnpJmsT0fViMGOhGiuYBwCKqgrAHGYiIp3aPTOjuvpAIZfhbHohUnOKTdk0ohZjMGOhmhLM2Eh1ZhjMEJGWtDZTVQKwq4Mt+ge7AwC2n2bvDFkmBjMWKlOazdR4AjBzZohIp/bUbAAYG+ELANiekGGSNhFdLwYzFkh/KYOm9MwwZ4aItGouNKkzJkKbN/NPYjYKyypN0i6i68FgxgLlFDW+lAFQXTSPPTNEpFNaKwEYAMK8nRDi5YgKtcDes5mmahpRizGYsUC6ISYPR6U0lGQIc2aITEcIYZaF6HTLFugSgHV0s5o41ESWiMGMBWpK8i/AnBkiU3p41TFEvPI3Znx5EN/sS8Kla+YxU6i6aJ5+MBNdNdS0MyGDnxlkcWxM3QBqPin5t4HqvwB7ZohMJTmrCH/HpQEA/knMwT+JOVj6ezx6BLhgXHc/jO/pi26+zpDJ6u9ZNZbqqdn6v2UHdPKAs50NsovKEXspF/06urd524haisGMBcoqbLz6L1CjZ0bNBGCitrTh2CUAwMBOHhjf0w+b49JwJDkHcVfyEXclHx9sO4uOHg4Y38MX43r4oV9H9waHjFuLWiNQUVV3SmWj3zNjq5BjZFdv/H7iKrafTmcwQxaFwYwFakr1X4A9M0SmoNEI/HzsMgDg/wYHY3LvANw/LATZhWXYfjoDW+LTsOdcFlJyivHV3iR8tTcJXk5KjI3wxbgevhgS5lVnCKi16GYyAXV7ZgDtUJM2mMnAs+PDjdIGImNgMGOBmp4zU1UBmMEMUZv5Jykbl3NL4Gxng3HdfaXtnk4qTBsQhGkDglBUVok9ZzOxJT4d20+nI6uwHKsPp2L14VQ4KhUY1c0H43r4YnS4D1zsbFutbXrBjE3dgGlUVx/IZUBCWgEuXStGoLtDq903kTExmLFAUjDTQME8oLpnhsl8RG1n/VHtENPNvQLq7WFxVNlgYqQ/Jkb6o0KtwaHEHGyOS8OW+DSk55fhj5NX8cfJq7BVyHBDqCfG9/DDjd194etid11tK62ayaRUyCE3MKzl7qhEVLA7Didfw86EDNwzuNN13R9RW2EwY4GqE4AbGWbSLTTJtZmI2kRRWSX+PqVN/L0zKrBJt7FVyDGsixeGdfHCksk9cPJyXlVgk47zGYXYey4Le89l4eWNp9AnyA3je/hhfA9fhHo7Nbt9ZQaq/9Y2JtwXh5OvYdtpBjNkORjMWCBdAnCjwYzUM8MEYKK28OfJqyguVyPEyxH9Oro1+/ZyuQy9g9zQO8gNz00Ix4XMQmyJS8eW+DQcT8lFTKr28vbfCejs44Rx3X0xvocfegW6NmlmVHXBvPpzcsZG+ODtvxNw8EI2isoq4aji1wSZP75KLUylWoNrxVWzmRpJAGbODFHb+rlqFtOdUYGtMu06zNsJD49ywsOjwpCeX4qt8enYHJeGgxeycT6jEOczCvHprgvwc7HDuB6+GNfdD4NCPWCrMNzzUt+07Jo6+zghyMMeqTkl2Hc+C+N7+F33eRAZG4MZC5NTVA4hALkMDS5lAFQPMzFnhsj4UnOK8U9iDmQy4La+HVr9+L4udvi/G4LxfzcEI6+kArvOZGBLXDp2nslAWn4pvjt4Ed8dvAgXOxtER/hiXHdfjOzmDQdl9ce8tMikgeRfHZlMhuhwX6w8kIwdpzMYzJBFYDBjYTILdUsZqBqtS6Hg1GyiNrOhajr20DAvBLjZG/W+XO1tMaVPB0zp0wGlFWocuJCFzafSse10OrKLyvHL8cv45fhlqGzkGN7FC+N6+CE63AdlBtZlMiQ6wgcrDyRje0IGNBphMFmYyJwwmLEwTa3+C9SoM8OieURGJYSQhpjuiGr9XpmG2NkqMCbcF2PCfaHWCBxLuYbNp9KwOT4NqTkl2HY6A9tOZ0AugzTV2tC07JoGhXjCUalAVmEZTl7OQ+8gtzY4E6KWYzBjYaTqv43kywDsmSFqK4eTryElpxhOKhuTDsso5DIM6OSBAZ088NKkCCSkFUgJxHFX8pGSo10fqrGkXqWNHCO6euOvU2nYfjqdwQyZPQYzFkaq/tvITCaAdWaI2sr6o6kAgJsi/fRyVExJJpMhwt8FEf4ueHJsF6TmFGNrfDqOpVzD3YOCG719dISvNphJyMCCcd3aoMVELWce7zpqsizdMFOTemY4m4nI2IrLK/HHiasAgDujgkzcmvoFeThgzrAQzEFIk/Yf1c0bMhkQdyUfV/NK4O9q3DwgouvRcBbYdVi2bBmGDBkCBwcHuLm5GdzniSeeQFRUFFQqFfr06WNwn82bN+OGG26As7MzvL29cccddyA5OdlYzTZ71UsZNJ4zY8vZTERGtzkuDUXlanT0cMCATtazOKOXkwp9q4aXdiRkmLYxRI0wWjBTXl6OqVOn4uGHH25wvzlz5mD69OkGr0tKSsKUKVMwZswYxMTEYPPmzcjKysLtt99ujCZbhMwmrssEMGeGqC3oli+4o1/r1JYxJ9ER2rWltp9mMEPmzWjDTEuWLAEArFy5st59/vvf/wIAMjMzceLEiTrXHz16FGq1Gq+//jrkVUMmzzzzDKZMmYKKigrY2rbeAmyWIqugadV/AVYAJjK2y7klOHAhGwBwe7+2ncXUFqIjfPDO5jPYfz4LJeVq2CuNs5o30fUyWs9Ma4iKioJcLseKFSugVquRl5eH77//HmPHjm0wkCkrK0N+fr7exVpICcDNyZnh2kxERvHLsUsQArgh1ANBHta3wnQ3X2d0cLNHWaUGBy5kmbo5RPUy62AmJCQEW7ZswYsvvgiVSgU3NzdcunQJa9eubfB2b775JlxdXaVLUJD5JuU1R6Vag5zilvTMMJgham3a2jLaQnnmnPh7PWQyGaIjfAAA2zjURGasWcHMCy+8AJlM1uAlISGh1RqXlpaGBx54ALNmzcLhw4exe/duKJVK3HnnnRCi/i/ohQsXIi8vT7qkpqa2WptMKae4eikDD8fGE4B1OTMVDGaIWt2xlGtIyiqCg1KBiT2tt+T/mHBtMLMjIb3Bz10iU2pWzszTTz+N2bNnN7hPaGjo9bRHzyeffAJXV1csX75c2rZq1SoEBQXh0KFDuOGGGwzeTqVSQaVqvOfC0uiq/3o4KhtdygCouTYTc2aIWtv6o9pemYk9/a16ZekbQj3hoFQgPb8McVfy0bODq6mbRFRHs96B3t7e8Pb2NlZb6iguLpYSf3UUCm0CmqYdfkHrqv82ZYgJqDGbiTkzRK2qtEKN32OvAGj75Qvamp2tAsM6e2FLfDq2n85gMENmyWg5MykpKYiJiUFKSgrUajViYmIQExODwsJCaZ/z588jJiYGaWlpKCkpkfYpL9d+aU+aNAmHDx/G0qVLce7cORw7dgz33XcfgoOD0bdvX2M13WzpCuY1JfkXYM4MkbFsiU9HQVklOrjZ44YQT1M3x+jG6qZoJ6SbuCVEhhmtb/TVV1/Ft99+K/2tCz527tyJUaNGAQDmzp2L3bt319knKSkJnTp1wpgxY/Djjz9i+fLlWL58ORwcHDB48GD8/fffsLdvf9Uos5pRYwYAbFgBmMgoqmvLdGgXK0qPCtf2yJ+4lIeM/FL4uNiZuEVE+owWzKxcubLBGjMAsGvXrkaPM2PGDMyYMaN1GmXhmlP9FwAUDVQALq/UIKuwDAFu7S8oJLoeaXml2HcuEwBwR1SgiVvTNnyc7dA7yA2xqbnYkZCBGQM7mrpJRHrMemo26dMlADe9Z8ZwBeB957Iw6I1tGPLWDuw5m9m6jSSycr8cvwyNAAZ0ckewp6Opm9NmoqtmNW3n0gZkhhjMWJCWJgDXns205kgqrhVXAACSsopasYVE1k1bW0Y7xHRnO+mV0dHVm9l3LgulFWoTt4ZIH4MZC9Kc6r9A/Tkz14rKpf8zn4ao6WIv5eF8RiHsbOW4KdLf1M1pU939XeDvaoeSCjUOJmabujlEehjMWJDmJgDXNzX7WnF1MMMaNERNt/6otgDnhB5+cLZrX2vDyWQyqYDe9tOc1UTmhcGMhVBrBHKqelS8nJuWAFzf1OzcqiEmAKhgDRqiJimtUOO32KsA2k/ib226oaYdpzNYDZjMCoMZC5FdVAaNAGQywMOhibOZpARg/d4X/Z4ZfiARNcX20xnIK6mAv6sdhoR5mbo5JjEkzAt2tnJcySvF6asFpm4OkYTBjIXIKtAGIB4OStgomva02RiYml1WqUZxeXXyHnNmiJpGl/h7W98OTVpOxBrpqgED2rWaiMwFgxkL0dzkX8BwAnDNISZAuxI3ETUso6AUu8+2r9oy9YmuqgbMVbTJnDCYsRDNTf4FauTM1MiLqTnEBHCYiagpNh2/ArVGoF9HN4R5O5m6OSalSwKOvZQr1b4iMjUGMxaiudV/gZo5MzWCmaJaPTMMZogaJISoXr6gnffKAICvix0iO7hCCGDnGfbOkHlgMGMhmlv9FzCcM1O7Z4bDTEQNi7uSjzPpBVDayHFzrwBTN8cs6HpndnCoicwEgxkLIVX/bUbOjK5npqLGbKY6wQx7ZogapOuVGdfdF6727au2TH10q2jvPZeJskpWAybTYzBjIaQE4GblzGifXiEATVXQUjsBmDkzRPUrr9RgU8xlAO1v+YKG9AhwgY+zCkXlahxKzDF1c4gYzFgKaZipBT0zQHUPTM2lDGpuJ6K6diRk4FpxBXycVRjexdvUzTEbcrlMKqDHasBkDhjMWIjqRSabngBsUyOY0fXA6BaY1B2HOTNE9ZNqy/Rrv7Vl6jMmXDvUtD2B1YDJ9BjMWADtUgYtGGZS1OyZ0QYtucX6K2+zZ4bIsOzCMuxM0Ca43tmPQ0y1DevsBZWNHJeuleBseqGpm0PtHIMZC5BTVF69lIFjc3pmqp/e6p4Z/WCGOTNEhm2KuYJKjUDvQFd08XU2dXPMjr1SgSFhngCA7awGTCbGYMYC6JJ/m7OUAQDU7BWvrJUArKskzIUmiQxjbZnG6aoBb+cUbTIxBjMWoCXVfwFAJpPVWTk7p6pnRhfMqDXMmSGqLf5KPuKv5kOpkOMW1papl67ezLGUa8ipNbmAqC0xmLEA1TOZmj7EpFOzCrBaI5BXUisBmMNMRHXoEn+jI3zg3oyh3fYmwM0e3f1dtNWAE9g7Q6bDYMYCtLRnBqie0VSp1iC/pAK6SQdSAjCHmYj0VKg12HictWWaSjdFeweDGTIhBjMWoHpadvODmZo9M7rkXyeVDextFQCYAExU2+4zmcguKoeXkwojurK2TGN0eTO7z2aivJLD1mQaDGYsQFbVMJN3Mwrm6egShtUaIdWYcXOwrRHk8MOHqCZd4u+tfQJg24yE+/aqVwdXeDkpUVhWicPJrAZMpsF3qgXIvI5hJiloUQupxoy7g1KqQcOcGaJq14rKpWnGnMXUNHK5DKO7aYeatrEaMJkIgxkLUL1idvMTEW1rzGaq2TOjq0HDnBmiar/GXkGFWqBHgAsi/F1M3RyLUXOKNqsBkykwmLEA15Uzo6geTtLrmak1ZZuIqmcxMfG3eYZ38YJSIUdKTjEuZLIaMLU9BjNmTm8pg5bkzMirc2Z0dSDca+TMVDBnhggAcDa9ACcu5cFGLsPk3qwt0xyOKhvcoKsGzAJ6ZAIMZszcteKWLWWgoz+bSTvM5O6o1EsMJiLg56rE3zHhPvBsQS9oexcdrltFm8EMtT0GM2ZOV2PG3UHZopkVNYeTDA0zMWeGSFuHaUNVbRkm/raMrhrwkYs50mcNUVthMGPmrif5F6jumalQa6Q6M5yaTaRv7/ksZBaUwcNRKc3MoeYJ8nBAN19naASw60ymqZtD7QyDGTN3PdV/gdo9M1XDTDV6eTjMRFRdW2Zy7wAobfix2FK6asDbWQ2Y2hjftWYuq6DlM5kAwxWA3R2UetuJ2rO84gpsjdPWR+EspuujC2Z2nclAhZq9vtR2GMyYOV3PTEtmMgHQqyejX2eGOTNEAPDbiSsoV2sQ7ueMHgGsLXM9+gS5w8NRiYLSShxJvmbq5lA7wmDGzF1P9V8AUqXfwrIKad0Ud0clc2aIquiGmO6MCoRMJjNxayybQi7DqG7a9ay2sxowtSEGM2autRKAdcexVcjgqFQwZ4YIwPmMQsSk5kIhl2FKnw6mbo5VGFtVDZiraFNbYjBj5qTqvy0eZpLpHcfNQQmZTFZjlpNg+XFqt3QVf0d19W7xUC7pG97FC7YKGRKzipDIasDURhjMmDkpZ6bFCcDap1g3XOXhoO3h0QU5AMDOGWqP1BqBX45pa8sw8bf1ONvZYlCIthowe2eorTCYMWOaGksQtDwBuKpnpmqYyc3BVrtdUR3MMG+G2qP957OQll8KV3tbjIlgbZnWpCugx1W0qa0wmDFj14rLpZyWlixlAFQvNJlZo5IwUD3LCeCMJmqfdENMU/oEQGWjMHFrrItuivbh5GvIK6kwcWuoPWAwY8aqAxDbFi1lANTtmXF31PbMKOQ1e2YYzFD7kl9agb9PpQEA7ujHIabWFuzpiM4+TlBrBHafZTVgMj4GM2bsegvmAdVBS35pJQBtAjCgnzPDGU3U3vx54irKKjXo4uOEXoGupm6OVdL1zuzgUBO1AQYzZux6lzIA9IMWQNvLAwByuQy6qypZqZPaGV1tmTtYW8ZoosO1U7R3nsnkZwwZHYMZM3a91X8BwKbW8JSuZwaoUR2YPTPUjiRnFeHIxWuQy4Db+rK2jLH06+gGNwdb5JVU4FhKrqmbQ1aOwYwZu97qv4ChnpkawYyiehFKovZCl/g7vIs3fF3sTNwa62WjkGNUV1YDprbBYMaMSdV/nVs2kwnQT/QFqoeZal7HBeGovdBoBDawtkybia6qBsxVtMnYGMyYMan6b2v2zDgq61zHnhlqL/5JzMbl3BI429ngxu6+pm6O1RvR1Rs2chnOZxTiYnaRqZtDVozBjBnTTaduafVfoLoCsI7+MBNzZqh90SX+3tI7AHa2rC1jbK72thjQyQMAsP00e2fIeBjMmLFWSQCu0TMjk2k/XGpfx6J51B4UllXir6raMhxiaju6KdrbE5g3Q8bDYMZMaTQC2UWtV2cGAFzsbPX+1v2fyxlQe/DnyasoqVAj1MsRfYPcTN2cdkOXN3MoMQcFpawGTMbBYMZM1VzKwNOp5QnANXtmaib/ApCqCjNnhtqDn1lbxiRCvBwR6uWISo3AnrNZpm4OWSkGM2ZKl/zrdh1LGQD6dWZq1pgBavbMMJgh65aaU4xDSTmQyYDb+7G2TFvjUBMZG4MZM9Ua1X+BhntmmDND7YWutsywzl7wd7U3cWvanzFV1YB3nclkTzAZBYMZMyUl/15nMKOyrX6Kayb/AtVF85gzQ9ZMoxFSMMNFJU2jfyd3uNjZIKeoHDGp10zdHGplv8ZekYZxTYXBjJmqLph3fcGMLvkOADwc9Y+lm7bNX0pkzQ4n5yA1pwROKhuM7+Fn6ua0S7YKOUZ2qxpq4hRtq1JcXoknfjqOp9fFIre43GTtYDBjpqqXMmh58i8AdHCzx4cz+qBXoCumDdD/VWojVQBmMEPWS1dbZlKkP+yVrC1jKmMjGMxYo8vXSqT/Xys23Ww1G5PdMzUoq+D6p2XrTOnTAVP61E16ZAVgsnbF5ZX48+RVAMCd/TnEZEoju3pDIZfhTHoBUnOKEeThYOomUSu4VCOYMeXSOOyZMVOtlTPTEObMkLX7+1QaisrVCPZ0QP9gd1M3p11zc1Aiquo52MG1mqxG6rVi6f8aYbofxgxmzFRrVP9tjC5nhrOZyFrVTPxlbRnTiw7XTdFmMGMtUrKrgxlT9vIzmDFTrTU1uyG2HGYiK3Y5twQHLmQDAG7ry9oy5kA3IeGfC9koLKs0cWuoNfxvX5L0f1N28jOYMUMajaheMdv5+hKAG8KieWTNfjl2CUIAg0M9mZ9hJsK8HRHs6YBytQb7zrEasLVRc5iJasotqaheysCROTNEzSWEkGYx3cFFJc2GTCZDdFUBve2nWQ3YGtSoy2rSXn7OZjJDuiEmV3tbKG2MF2/aMGeGrNTRi9eQnF0MB6UCE3uytow5iY7wwTf7k7DzTAY0GgG5nLlMlmbP2Uw8sfo4SsrVqBm/MAGY9GQVGD/5F+DUbLJeusTfmyL94ajibzZzMqCTB5xVNsgqLEfspVxTN4da4L2tZ5FbXIGySv1efQ0TgKmm1iqY1xhdzkwFh5nIipRWqPF7rLa2DJcvMD9KGzlGdPUGwCnaligmNRexqblQKuT4e/5w7H9hjHSdKZ9PBjNmSFrKwIgzmYDqFbXVHGYiK7I5Lg0FZZUIdLfHoBAPUzeHDNCtor2N1YAtzncHkwEAN/fyR7ifCzq4VS/cGncl30StYjBjlqSZTMYOZjibiayQLvH39n6BzMcwU6O6+UAmA05fzceV3JLGb0Bm49+kHADAnQYS601ZyonBjBlqi4J5QM2p2RxmIuuQlleK/ee1U37v6MfaMubKw1GJfh211YBZQM+y5BRpf2x3cLdvZM+2xWDGDLXFUgYAYKtgzwxZlw3HL0EjgIGdPBDs6Wjq5lADdENNOzhF22KUVqhRXK4GALg71s3pNGWVbQYzZkiq/mvEgnlA9XIGzJkhayCEwM9VQ0yGusDJvOjqzey/kI3iclYDtgTZVb0ytgoZnA3MEjTlqC6DGTPUZgnAzJkhKxKTmosLmUWws5VjYiRry5i7rr5OCHS3R3mlBvvPZ5u6OdQEOVX5nB6OSoO9MHJr7ZlZtmwZhgwZAgcHB7i5udW5PjY2FjNnzkRQUBDs7e0RERGBDz/8sM5+u3btQr9+/aBSqdC5c2esXLnSmM02KY1GILutEoBZAZisiK62zMSe/nC2szVxa6gx2mrAVQtPcqjJIuQU64IZw99Npky3N2owU15ejqlTp+Lhhx82eP3Ro0fh4+ODVatWIS4uDi+99BIWLlyIjz/+WNonKSkJkyZNwujRoxETE4P58+dj7ty52Lx5szGbbjJ5JRVST4mnkevMsGgeWYvSCjV+jbkCgLVlLIlu4ckdCRkmLbhGTVNctTiok0ph8HoHExaoNOo9L1myBADq7UmZM2eO3t+hoaE4ePAgNmzYgMceewwA8PnnnyMkJATvvfceACAiIgL79u3DBx98gPHjxxuv8SZScykDlY3hF0xr0eXMVDBnhizc9tMZyC+tRICrHQaHeZq6OdREg0I94KhUIKOgDKeu5KFXoJupm0QN0H1TyGr1wbx1eyS+PXgRL94U3vaNqmJ2OTN5eXnw8KgudHXw4EGMHTtWb5/x48fj4MGD9R6jrKwM+fn5ehdL0VbVfwH2zJD1WH80FQBwW78OUskBMn8qGwWGd9FWA97OAnqWo9ZbbMbAjvjryeHwdzXddG2zCmYOHDiANWvWYN68edK2tLQ0+Pr66u3n6+uL/Px8lJQYLrb05ptvwtXVVboEBQUZtd2tqa2Sf4HqnJmUnGKUVqiNfn9ExpCRX4rdZzMBcIjJEo2pmqK9PYF5M+bOhOtINqrZwcwLL7wAmUzW4CUhIaHZDTl16hSmTJmCRYsWYdy4cc2+fU0LFy5EXl6edElNTb2u47UlqfqvkQvmAdU9M0cvXsO9X/9r9PsjMoaNMZehEUBUsDtCvZ1M3RxqptFV1YBPXc5HWl6pqZtDDRBVA03m2PfZ7JyZp59+GrNnz25wn9DQ0GYdMz4+HtHR0Zg3bx5efvllvev8/PyQnq4fsaenp8PFxQX29oa7tFQqFVQq4wcDxtBWBfOA6pwZAPg3Ocfo90fU2oQQ0vIF7JWxTN7OKvQOdENMai52JGTgrkEdTd0kaoQply2oT7ODGW9vb3h7e7daA+Li4jBmzBjMmjULy5Ytq3P94MGD8eeff+pt27p1KwYPHtxqbTAnWQVts5QBUD3MRGSpTl3Ox9n0Qqhs5JjUy9/UzaEWGhvhUxXMpDOYMWNWNczUHCkpKYiJiUFKSgrUajViYmIQExODwsJCANqhpdGjR2PcuHFYsGAB0tLSkJaWhszMTOkYDz30EBITE/Hcc88hISEBn376KdauXYunnnrKmE03mSwTJAATWSpd4u+4Hn5wtWdtGUs1pqoa8L7zWczfM2P1zWYyB0YNZl599VX07dsXixYtQmFhIfr27Yu+ffviyJEjAID169cjMzMTq1atgr+/v3QZMGCAdIyQkBD88ccf2Lp1K3r37o333nsP//vf/6xyWjZQczZTWwwzmd8LkqipyirV2BSrrS3D5QssW4S/MwJc7VBaocGBC1mmbg41whyHmYwazKxcuRJCiDqXUaNGAQAWL15s8Prk5GS944waNQrHjx9HWVkZLly40GjOjiXLKmib6r8ApErDRJZoZ0IGcosr4OuiwrDOXqZuDl0HmUwmzWraxinaZkuY8TiTWU3Nbu+EEMgu0i0yafxgZkCn6no+SgVfCmRZ1h+9DAC4rW8gexmtgFQN+HSGWX9pUjvsmaHmySupkKrxtkXOTGSgK766tz8AQM0PD7IgWYVl2HVG+wv+zqgOJm4NtYbBoZ6wt1UgLb8UcVcsp9Bpe9TucmaoeX45rv2l6WJnY/SlDHT6BLkB0FYB5q8hshSbYq6gUiPQO8gNnX2cTd0cagV2tgoM66IdLtyRwKEmc2TOXxEMZsxEeaUGS36LBwDkl1a22f3WnNHEVQ3IUuhqy9zZj70y1oSraJs3qWie+XXMMJgxF0cumqZonbxGMFOp0ZikDUTNEXclD6ev5kOpkOOW3gGmbg61ojFVwUzspTxkFLAaMDUdgxkzoVtbBgDWPth2BQH1emYYy5AF+Lkq8Xdsdx+4ORg/t4zajo+LHXoFugLQzlYj88JhJmrU7jPaYObDGX0wMMSjkb1bj4I9M2RBKtQabIrRBjOsLWOdoqsK6HEVbfOjC2ZkZjjOxGDGDKTnlyIhrQAyGTC8S+stFdEUCvbMkAXZdSYT2UXl8HJSYUQbv1eobURX1ZvZe47VgM2V+YUyDGbMwp6qIaZeHVzh4di23eYKGXtmyHLoli+4rW8AbFgbySr1CHCBr4sKJRVq/JOYbermUA1mPMrEYMYc6PJlRnZt+1+acrlMykxnrRkyZzlF5dKU3Ts4xGS1ZDKZtFYTh5rMi658hxmOMjGYMTW1RmDvOe1aJCO7mabbXNc7o+bcbDJjv8ZcRoVaoGcHF4T7uZi6OWREY6uGmnYksBqwOTLDWIbBjKmduJSLvJIKuNjZoHegm0naoMubYTBD5uznY1WJv/3YK2PthoR5QWUjx+XcEiSkFZi6OVTFnL8hGMyYmG6IaVgXL5PlAOiCGabMkLk6k1aAk5fzYKuQYXIfFsqzdvZKhbR4KKsBmxHOZqL6mDJfRkcXzDABmMzVz8e0FX/HhPu0eZI8mUb1KtqsBmxuzC+UYTBjUrnF5YhNzQUAjDCDYEbDsWkyQ5VqDTZUDTHdwSGmdkNXbyYmNRdZhWUmbg0B1csZmCMGMya073wWNALo6usEf1d7k7XDRuqZMd8XKrVfe89lIauwDJ6OSoyuKndP1s/P1Q49AlwgBKsBm4vqonmmbYchDGZMSFf115RDTAAg52wmMmO6RSUn9wmALWvLtCvREdreGebNmBvzi2b4yWAiQgjsOacLZkz7a1Npo30ZFJWx2iaZl7ziCmyN1+ZMcPmC9ke3ivaes5koq+Tnk6mZ889dBjMmcia9AOn5ZbC3VaB/J3eTtiXczxkAEJN6zaTtIKrt1xNXUK7WINzPGT0CXE3dHGpjkR1c4e2sQlG5Gv8m5Zi6Oe0eh5moDt0Q0w2hHrCzVZi0LbqFLf9NYjBD5uXnqiEm9sq0T3K5DGO6aXtnWA3YfJhhLMNgxlTMYUq2zoBO2mDmyMUcaJg3Q2bifEYhYlJzYSOX4da+rC3TXukWntyekM5qwCbG2Uykp6isEkeStb0gI7uZfnZGzw6usLdVILe4AuczC03dHCIA1bVlRnXzhpeTysStIVMZ1sULShs5UnNKcC6Dn0+mxGEm0vNPYjbK1RoEedijk6eDqZsDW4UcfTu6AQDHpcksqDUCG45xiIkAB6UNhoR5AuBQk7mQmeFAE4MZE6g5xGQuZaF1Q02HkxnMkOntO5+F9PwyuDnYsrYMSbOatrMasEmZ7yATgxmTqA5mzOdDWpcEfJg9M2QGdIm/U3oHQGVj2gR5Mr0xVfVmjqVcQ05RuYlb045VjTOZyW9wPQxm2lhyVhEuZhfDViHD4KquU3PQt6MbbOQyXMkrxaVrxaZuDrVj+aUV2ByXBgC4g0NMBKCDmz3C/ZyhEcCuMxxqMjUGMyQVyosKdoeTysbEranmoLRBjw7aOh53fXUIT62JwTf7knAkOQfF5ZUmbh21J3+cuIqySg26+johsgNry5DW2Kreme2sBmwy5jzMZD7fpu1E9RIG5jPEpHNHvw6ITc1FSk4xUnKK8ctx7eJ+chnQxccZkYGu6BXoisgOrojwdzF5fRyyTrrlC+7oF2g2OWVkemMifPDxzvPYcyYT5ZUaqXI5tR1pNpMZJgAzmGlDZZVqHLiQDcA86svUdu/gTrgp0h8nL+fh5KU8nLiUh5OXc5GeX4Yz6QU4k14gfdHYyGXo6uusDW4CXdGrgxu6+TnzA4auS1JWEY5evAa5DLiNtWWoht6BbvB0VCK7qBxHknMwpLOXqZvUfplfLMNgpi0dTb6Gkgo1vJ1ViPB3NnVzDPJyUmF0Nx+MrlH/Jj2/VBvcXM7DyUu5OHEpD9lF5Yi/mo/4q/lYfTgVAKBUyBHu74zIDroeHDd08XXi4oDUZLrE3xFdveHjYmfi1pA5UchlGB3ug/VHL2F7QgaDGRMw56KFDGbakG4W04gu5jMluyl8Xezg290OY7trx6yFELiaVyr13Gj/zUNucQVOVPXo/HBIe1uVjRzdA1zQq4MrIgPd0CvQFWHeTlDILef8qW1oWFuGGhGtC2ZOp+PlSREW9TlqDXShjDk+6gxm2pA0Jbub+Q0xNYdMJkOAmz0C3OwxoacfAG2Ac+laiTaYuZyLk5e0Q1UFZZU4npKL4ym5AC4CAOxtFejZwQWRHdykYaoQT0fIGeC0awcTs3ElrxQudjZSsidRTcO7esNWIUNydjESs4oQ5u1k6iaRmWAw00bS80uRkFYAmQwYboXdozKZDEEeDgjycMCkXv4AtL+0k7OLcPJyVf7NpTycupKH4nI1Didfw+Hk6oUtnVQ26NnBBb0C3aRhqo4eDvzl1Y7ohphu6R3A5HIyyEllgxtCPbH3XBa2n05nMNPGqpczML/PZQYzbUTXK9Mr0A3ujkoTt6ZtyOUyhHo7IdTbCVP6aJM51RqBxMxCaWjqxKVcxF3JR2FZJf5JzME/idVF+1ztbRHZQZdgrP23g5u9Wb6R6PoUllXir1OsLUONiw73qQpmMjBvRJipm9OucJiJzGqVbFNSyGXo4uuMLr7O0pdWpVqDcxmFVUnG2iGq01cLkFdSgX3ns7DvfJZ0ew9HZY0EY1f0CnSDr4uKAY6F+/PkVZRUqBHq7Yi+QW6mbg6ZsegIXyz+LR5HLl5DXnEFXB1sTd0kMgMMZtqAWiOw75z2C7m9BzOG2CjkiPB3QYS/C6YNCAIAlFdqcDa9QC/J+ExaAXKKyrH7bKYUHAKAt7NK6rnRzaLyduYqy5ZEN+X/zijWlqGGBXk4oKuvE86mF2LX2Qyp15eMT5jxcgYMZtpA7KVc5JVUwMXOBr0DWdG0KZQ2cvTs4IqeHVwBdAQAlFaokZBWIE0PP3k5D2fTC5BZUIbtCRl6lUH9Xe2qe3Cq8nA82snwnqVJyS7Gv0k5kLG2DDXRmHBfnE0vxPbTDGZMwQxjGQYzbUFX9Xd4F2/YsOZKi9nZKtAnyA19agxDlJSrEX+1OsH4xOU8XMgsxNW8UlzNK8WW+OpVdgPd7aWem16B2kDJ1Z5d1Kb2c9V07GGdveDvam/i1pAlGBvhg893X8CuMxmoVGv4uUoMZtoC82WMx16pQFSwB6KCPaRthWWViLucVz2L6nIekrKKcOlaCS5dK8GfJ9OkfTt5OqBXVf2byA6u6NHB1azWzLJ2Go2QghnWlqGm6tvRHe4OtrhWXIEjF6/hhtCWL9obdyUPL244iUdHd8a4Hn6t2Errw9lM7di1onKcuJQLABje1fqmZJsjJ5UNBoV6YlCND7i8kgrEXdZVMdYmGqfmlCA5uxjJ2cX4NfYKAO1YcJi3k14OTnd/V9grOVXYGP5NzsGlayVwVtlgXHd+kVDTKOQyjO7mgw3HL2NHQsZ1BTNv/ZWA2Et5WLA2Fn/Pd0Ggu0MrttS6iKr5TOYXyjCYMbp957OgEUA3X2d2oZuQq70thnT20iuBfq2oXLsOVdUU8ZOX8nAlrxTnMwpxPqMQG2ostNnV11kvByfcz5m1UFqBLvF3Ui9/BozULGMitMHMttPpePGmiBYd4/TVfOytmpxRWFaJZ9edwA9zB7GApwViMGNk1lL11xq5Oyoxoqs3RtQY/sssKMOpy9WLbMZeykNmQRkS0gqQkFaAdTUW2uzm56yXg9PVlwttNkdRWSX+PHkVAIeYqPlGdPWGjVyGxMwiJGUVIcTLsdnH+N/eJADAgE7uOHU5HwcTs/HdwWTMHhrS2s21CsKMC80wmDEiIQT2MF/Gong7qzA63Aejw/UX2tQmGOfiRFWgk1NUjrgr+Yi7ko+fUL3QZoS/s7SKeGSgK7r4ODE5sR5/n0pDcbkanTwdEBXsburmkIVxsbPFwBAPHLiQje2n0zF3eGizbp+eX4pfY7W9ry/eFIFTl/PwyqY4vPV3AkZ09UYoqwvXS2aG0QyDGSNKSCtARkEZ7G0V6N+JH9aWytfFDjd2t8ONNRbavJJXqjdF/MSlPOSVVCD2Uh5iL+UBSAGgXWizR4D+Mg2hXGgTQPUspjv6sbYMtUx0hC8OXMjGjoSMZgcz3x5IRoVaoH+wO/p2dEefIDdsiU/H3nNZWLA2FusfGswfIrWY75rZDGaMSjfENDjMEyob5gNYC5lMhg5u9ujgZo8JPbXrUAkhkJpTIlUwPnEpD6cuaxfaPJaSi2MpudLtHZQK9AyoWeTPFZ3a2UKbl64V48CFbADAbf1YJ4RaJjrcB6/9Ho9/k3KQX1oBF7umlVooLq/ED4e0Pzh0QZBMJsPyO3th3Ad7EJOaiy/2JOLR0Z2N1nZLVD2bybTtMITBjBHp6stwiMn6yWQydPR0QEdPB9zcKwBAwwtt/pucg3+Tq9ehclbZoKeUYKwdpgrysN51qH45pu3eHxLmydkj1GKdvBwR5u2IC5lF2HM2U3rvNWbdkUvIK6lAsKeD1OMKAP6u9lgyuQcWrI3Ff7adxehuPuge4GKs5lssc/xUYjBjJEVllThyUftlNYLBTLvUnIU2C8oqcTAxGwcTs6Xbu9rb1liDSjuLKsDVzuIDHCGE3hAT0fWIjvDFhcxEbD+d0aRgRq0R+HqfNvH3/mEhdYZ8b+vbAZvj0rA5Lh0L1sZg02ND2bNeRZjxQBODGSM5eCEbFWqBjh4O6OTJX56k1dyFNveey5KmjgKAp6Oyxiri2llUvi52pjqdFjl68RqSs4vhqFRgYiRry9D1iQ73wZd7ErHzTAbUGtFoPtrW+DSk5BTDzcHW4Cw6mUyGZbdF4kjyNSSkFeA/287h+Qnhxmq+ReEwUztUs+qvpf+SJuNqzkKb2UXl2HUmE7vOVC+06eOs0psiHhnoCi8n811oU1db5qZIfzgo+RFE1ycq2B2u9rbILa7AsZRrGNDJo8H9v9yTCAD4v0HB9b7+vJxUWHZbJB5adRRf7L6AsRE+elXG2zvOZmpHuIQBXY/mLLSZUVCGbaczsO109UKbAa52VQnG2llUkR1c4W4GC22WlKvxxwltbZk7WFuGWoGNQo5R3byxKeYKtp/OaDCYOXrxGo6l5EKpkOPeIcENHndCTz/c3rcDNhy/jKfXxuLPJ4cz+DZjfGaMIDmrCCk5xbBVyDA4rOVltolqMrTQZnF5JU5fza+z0OaVvFJcySvF5rjqhTaDPOyl+je9qtahauuFNrfEp6GgrBJBHvYY2MgvaKKmGhPuUxXMpOOFifUPCf1vr7ZXZkqfAPg4Nz48u2hyDxxMzEZydjHe+isBS6f0bLU2WyJRNc5kjoMNDGaMQNcr0z/YA45ctJCMyEFp0+SFNlNzSpCaU4I/qqruAkCIl2N1gnEbLLSpG2K6vW9gu5qKTsY1qqsPFHIZzmUUIiW7GB0N5CmmZBdjc5x2kdkHRjStJo2rvS2W39kL93z9L747eBHjuvthWBeuscdgpp3gEgZkSs1ZaDMpS1sKvi0W2ryaV4J957XJzJzFRK3J1cEW/YPdcSgpB9sT0nGfgeUIvtmfBI3QDv139XVu8rGHd/HGPTcE4/t/LuLZ9bH4e/6INu/RNBfCfCczMZhpbWWVahysKgY2oguDGTIP17PQpkIuQxcfp+teaHPDscsQAhgY4mHwlzPR9Rgb4YtDSTnYkZBRJ5jJLS7H2iPaZUceaGalYABYeFM49p7LRHJ2MZb8Fof3p/VpjSZbnOpYxvy6ZhjMtLIjyddQUqGGt7MKEf5Nj/6J2lpjC22eqFqLquGFNt2kIaqGFtqsWVuGi0qSMYyJ8MGyP0/jn8RsFJRWwLlGNeAfDqWguFyNcD9nDO3c/DxGB6UN3pvWG1M/P4gNxy5jfA8/jO/RfssKcJipHeCUbLJktRfaFEIgPb9M23NTIwdHb6HNf7W3Vdpop5jXHKLq7K1daPN4ai4SM4tgb6vATZH+JjxDslZh3k4I8XJEUlYR9p3LwsSq11l5pQbfHkgGoO2VaennclSwB+aNCMPnuy/gxQ0nERXsbtYlEIyBw0ztCJcwIGsik8ng52oHP1c/jKv6JSqEwOXcEmn21MmqXpz80krEpuYiNjVXur2drRw9AlxRVqkGAEzs6WfUBGNq38aE++DrfUnYdjpDCmZ+jb2CjIIy+LqocEvvpi13UJ+nbuyCXWcykJBWgJd+OYnP/y+qXf1o1VUANscz5qdKK0rLK8WZ9ALIZMCwzsx4J+skk8kQ6O6AQHcH6QtDCIGUnGK9ZRpOXc5HYVkljl68Jt2WtWXImKIjtMHMrqpqwHJZ9XTs2UNC6h0GbSqVjQLvTeuNWz/Zj81x6fjl+GXc3g6T2c0xfmMw04r2VA0x9Q50M4sCZURtRSaTIdjTEcGejtKvX41GICm7SFpF3NNJiSGsu0RGNKCTB5ztbJBdVI6Y1FwUl1ciIa0ADkoF7hrYsVXuo0eAK56M7oJ3t5zFol/jcEOoJwLc7Fvl2ObOnIeZri9MJT2s+ktUTS6XIczbCbf27YBXb+mOR0d3bldd8tT2bBVy6fN3R0I6vtqrXVByWv8guDq03nTqh0aGoU+QGwpKK/H8zyekYnLWTneW5ricAYOZVlKp1mDvOW0ww1WyiYhMIzpCm7z+6a4L2HM2E3KZdnXs1mSjkOO9ab1hZyvH3nNZWHUopVWPb+7M8TcJg5lWEnspD/mllXC1t0XvQFdTN4eIqF0a1dUHcln1kMjEnv4I8mj9ukZh3k7Satpv/HEayVlFrX4fZseMe6AYzLQS3RDTsC5esFHwYSUiMgV3R6Ve8DJ3eOv2ytQ0a3AnDA71REmFGs+si4VaY75f9q2hepjJ/PBbt5UwX4aIyDyoasxa6tvR3Wj3I5fL8M7UXnBS2eDIxWv4qmrmlLUzx9w3BjOt4FpROU5cygXAJQyIiEztpUndEe7njA2PDDH6fQW6O+DVm7sDAN7fchYJaflGv09TMeNRJgYzrWHv+SwIAYT7OcPPtfFl5YmIyHhGdvXG3/NHoJ8Re2Vqmto/ENHhPihXa7BgTSzKKzVtcr9tTcB8oxkGM62AVX+JiNovmUyGN++IhLuDLeKv5uOjHedM3SSjMsNRJgYz10sIgT2ckk1E1K75ONvh9VsjAWinhcfUWNbDWnCYyYqdvlqAzIIy2Nsq0L9T23RpEhGR+ZnUyx+TewdArRFYsDYGpRVqUzepVbXbonnLli3DkCFD4ODgADc3tzrXx8bGYubMmQgKCoK9vT0iIiLw4Ycf6u2zYcMG3HjjjfD29oaLiwsGDx6MzZs3G7PZzaKbxTQkzBMqG4WJW0NERKa0dEoP+DirkJhZhLf/TjB1c4yi3Q0zlZeXY+rUqXj44YcNXn/06FH4+Phg1apViIuLw0svvYSFCxfi448/lvbZs2cPbrzxRvz55584evQoRo8ejVtuuQXHjx83ZtObbPfZDADAyG4cYiIiau/cHJR4+85eAIAV+5Nx4EKWiVvUesx5mMmoC00uWbIEALBy5UqD18+ZM0fv79DQUBw8eBAbNmzAY489BgD4z3/+o7fPG2+8gU2bNuG3335D3759W73NzVFYVokjydoVgTklm4iIAGB0Nx/MHNgRP/2bgmfXncDf84fD2a711oYyFd1sJjPsmDG/nJm8vDx4eHjUe71Go0FBQUGD+7SVfy5ko1IjEOzpgE5ejqZuDhERmYmXJkUgyMMel3NL8Nrv8aZuTqtqd8NMzXXgwAGsWbMG8+bNq3efd999F4WFhZg2bVq9+5SVlSE/P1/vYgy6QnkDO5k+sCIiIvPhpLLBe1P7QCYD1h65hG3x6aZu0vUz42GmZgczL7zwAmQyWYOXhITmJz2dOnUKU6ZMwaJFizBu3DiD+/z4449YsmQJ1q5dCx8fn3qP9eabb8LV1VW6BAUFNbs9TeHraof+we7oF8xZTEREpG9giAfmVq3Y/cKGk8gpKjf6fZ6+mo+L2cZZ9FKazWSGXTPNzpl5+umnMXv27Ab3CQ0NbdYx4+PjER0djXnz5uHll182uM/q1asxd+5crFu3DmPHjm3weAsXLsSCBQukv/Pz840S0Nw9KBh3Dwpu9eMSEZF1eHpcN+w6k4lzGYV4ZeMpfHxXX6MFA9tPp2Pud0fg72KHAwujjXIfgHnmzDQ7mPH29oa3d+slu8bFxWHMmDGYNWsWli1bZnCfn376CXPmzMHq1asxadKkRo+pUqmgUqlarY1EREQtYWerwPvT+uC2T/fjj5NXMS7WF1P6dGj1+4m/ko/HfzoOIYDMwrJWPz6gLRJrroyaM5OSkoKYmBikpKRArVYjJiYGMTExKCwsBKAdWho9ejTGjRuHBQsWIC0tDWlpacjMzJSO8eOPP+Lee+/Fe++9h0GDBkn75OXlGbPpRERErSIy0BWPjekMAHh1UxzS80tb9fgZ+aWY++1hFJdri/QZK+YQ1VXzzI5Rg5lXX30Vffv2xaJFi1BYWIi+ffuib9++OHLkCABg/fr1yMzMxKpVq+Dv7y9dBgwYIB3jyy+/RGVlJR599FG9fZ588kljNp2IiKjVPDq6MyI7uCKvpALPrT/Rar0cJeVqPPDdEVzJK4W3c9uMSLS7CsArV66EEKLOZdSoUQCAxYsXG7w+OTlZOsauXbsM7lNf7RoiIiJzY6uQ4/1pvaG0kWP32UysPpx63cfUaASeXheD2Et5cHOwxccztbXXjDUYZL6DTGY2NZuIiMhadfF1xrPjugEAXv89Hqk5xdd1vPe3nsWfJ9Ngq5Dhi/+LMnq9M11nkhlOZmIwQ0RE1FbmDAvBwE4eKCpX4+l1sdBoWtbf8fPRS/h453kAwJu398KgUE9p8MfYibpmGMswmCEiImorCrkM707tDQelAv8m5eCb/UnNPsa/STl4YcMJAMAjo8JwZ1RgazfTIGHGA00MZoiIiNpQR08HvDypOwBg+eYzOJde0OTbXswuwoPfH0GFWmBiTz88UzVsBUDqMjFazgyHmYiIiEhn5sAgjOrmjfJKDRasjUWFWtPobfJKKjBn5WFcK65Ar0BXvD+tD+Tyto8s2t1sJiIiIqpLJpPh7Tt6wdXeFicv5+GTqvyX+lSoNXj0h2O4kFkEf1c7/O/e/rBXKvSPWRVkmHFtO6NhMENERGQCvi52WDqlBwDg4x3ncfKS4WKwQggs+jUO+85nwUGpwP9m9YePi12d/Yw9/KNLLOYwExEREUkm9w7ApEh/VGoEFqyNQWmFus4+X+9Lwo+HUiCTAf+d0Rc9AlwNHqtmjGHoONaMwQwREZGJyGQyvHZrT3g5qXAuoxDvbTmjd/22+HQs+/M0AOClmyIwtrtvvcdyd1Ai0N0eALD+6KVWb6sZr2bAYIaIiMiUPByVePuOSADA//Yl4VBiNgDt4pFPrNYuHjlzYBDuHxbS4HHkchkeGB4KAPhqbyLULaxhU5/qtZnML5xhMENERGRi0RG+mNY/EEIAz6yPRVJWkbR45NDOnlg6pSdkTQgipvYPhLuDLS5mF+PvU2lt0HLzwGCGiIjIDLxyc3d0cLNHak4JJn64B1fyShHq7YhP74qCraJpX9cOShvcO7gTAODz3RdatRqwrmie+fXLMJghIiIyC852tnhnai8AQGmFBm4Otvhm1gC4Otg26zizhnSCna0cJy/n4WDVkFVrYNE8IiIiatSQMC88PyEcoV6O+PKe/i1aPNLDUYnp/YMAAF/sTmztJpolBjNERERm5OFRYdjxzCgMDPFo8THmDg+FXAbsPpuJ+Cv5rdKu6tlM5tc1w2CGiIjIygR5OGBSrwAAwJd7LrTKMTnMRERERG3qwRHaadq/nbiKS9eKTdwa42IwQ0REZIV6dnDFsM5eUGsEvt6X1ApH5GwmIiIiamMPjtT2zqz+NxXXisqv61gcZiIiIqI2N6yzF3oEuKCkQo3v/7lo6uYYDYMZIiIiKyWTyfDgyDAAwMoDyde1AGV1z4z5dc0wmCEiIrJiN/X0Q6C7PXKKyrHuOhagFGjdtZ5aE4MZIiIiK2ajkFcvQLmn9RegNAcMZoiIiKycbgHKlJxi/HXqaouOwQRgIiIiMhkHpQ1mDekEQLvEwfUsQMkKwERERGQS9w6usQDlheYvQGnOg1MMZoiIiNqBmgtQfr6n+QtQcpiJiIiITE63AOWe61iA0gxjGQYzRERE7UXNBSi/aOYClJyaTURERGZBtwDl7yeuIjWnGQtQcpiJiIiIzEHPDq4Y3qXlC1ByNhMRERGZ3IMjtEscrDnc9AUozXeQicEMERFRuzO0s2ezF6DU1abhMBMRERGZXO0FKEvKW74ApTlgMENERNQO1VyAcv3R1Eb35zATERERmRW9BSj3JqFSrWlw/+qieeY3zsRghoiIqJ2a1j9IWoDy77i0Jt3G/EIZBjNERETtlr1S0eQFKDnMRERERGap5gKUBxpYgJKzmYiIiMgs6S1AubvxJQ7MMJZhMENERNTezR0eCoVchr3nshB3Jc/gPhxmIiIiIrMV5OGASZH+AIAv9yQa3omzmYiIiMiczWviApRmGMswmCEiIqLGF6AUZjzQxGCGiIiIAFQvQLn6cApyai1AKRXNa+tGNQGDGSIiIgJQvQBlaYUG3x+sZwFKMxxnYjBDREREAPQXoPz2oP4ClA3U0zM5BjNEREQkuamnH4I86i5AqcuZMb9+GQYzREREVEPNBSi/3JtYZwFKMxxlYjBDRERE+qZGaRegTM0pwV+ntAtQcpiJiIiILIbeApR7LkCI6onZMjMcaGIwQ0RERHXoFqA8dTlfbwFKDjMRERGRRai9ACWHmYiIiMji1FyAMiEtHwBnMxEREZEFqbkA5aVrJQA4zEREREQWRrcApTljMENERET10i1AqcPZTERERGRxdAtQAjDLpBkGM0RERNSgoZ090bODCwDAxc7WxK2py8bUDSAiIiLzJpPJ8L97B2BHQgaiI3xM3Zw6GMwQERFRo/xc7XDXoI6mboZBHGYiIiIii8ZghoiIiCwagxkiIiKyaAxmiIiIyKIxmCEiIiKLxmCGiIiILBqDGSIiIrJoDGaIiIjIojGYISIiIotm1GBm2bJlGDJkCBwcHODm5lbn+tjYWMycORNBQUGwt7dHREQEPvzww3qPt3//ftjY2KBPnz7GazQRERFZFKMuZ1BeXo6pU6di8ODB+Prrr+tcf/ToUfj4+GDVqlUICgrCgQMHMG/ePCgUCjz22GN6++bm5uLee+9FdHQ00tPTjdlsIiIisiAyIYQw9p2sXLkS8+fPR25ubqP7Pvroozh9+jR27Niht33GjBno0qULFAoFNm7ciJiYmCbff35+PlxdXZGXlwcXF5dmtp6IiIhMoanf32aXM5OXlwcPDw+9bStWrEBiYiIWLVrUpGOUlZUhPz9f70JERETWyaxWzT5w4ADWrFmDP/74Q9p27tw5vPDCC9i7dy9sbJrW3DfffBNLliyps51BDRERkeXQfW83Oogkmun5558XABq8nD59Wu82K1asEK6urg0e9+TJk8LLy0u89tpr0rbKykrRv39/8dlnn0nbFi1aJHr37t3gsUpLS0VeXp50iY+Pb7TNvPDCCy+88MKLeV5SU1Mb/N5vds5MZmYmsrOzG9wnNDQUSqVS+ruxnJn4+HiMHj0ac+fOxbJly6Ttubm5cHd3h0KhkLZpNBoIIaBQKLBlyxaMGTOm0TZrNBpcuXIFzs7OkMlkje5vKfLz8xEUFITU1NR2kQvE87Ve7elcAZ6vNWtP5woY/3yFECgoKEBAQADk8vozY5o9zOTt7Q1vb+/ralxNcXFxGDNmDGbNmqUXyACAi4sLTp48qbft008/xY4dO7B+/XqEhIQ06T7kcjkCAwNbrc3mxsXFpV28aXR4vtarPZ0rwPO1Zu3pXAHjnq+rq2uj+xg1ZyYlJQU5OTlISUmBWq2WZiB17twZTk5OOHXqFMaMGYPx48djwYIFSEtLAwAoFAp4e3tDLpejZ8+eesf08fGBnZ1dne1ERETUPhk1mHn11Vfx7bffSn/37dsXALBz506MGjUK69evR2ZmJlatWoVVq1ZJ+wUHByM5OdmYTSMiIiIrYdSp2StXroQQos5l1KhRAIDFixcbvL6hQGbx4sXNqjFjzVQqFRYtWgSVSmXqprQJnq/1ak/nCvB8rVl7OlfAfM63TYrmERERERmL2RXNIyIiImoOBjNERERk0RjMEBERkUVjMENEREQWjcGMib355psYMGAAnJ2d4ePjg1tvvRVnzpzR22fUqFGQyWR6l4ceekhvn5SUFEyaNAkODg7w8fHBs88+i8rKSr19du3ahX79+kGlUqFz585YuXKlsU9Pz+LFi+ucR3h4uHR9aWkpHn30UXh6esLJyQl33HEH0tPT9Y5hCeep06lTpzrnK5PJ8OijjwKw/Od1z549uOWWWxAQEACZTIaNGzfqXS+EwKuvvgp/f3/Y29tj7NixOHfunN4+OTk5uPvuu+Hi4gI3Nzfcf//9KCws1NvnxIkTGD58OOzs7BAUFITly5fXacu6desQHh4OOzs7REZG4s8//2zT862oqMDzzz+PyMhIODo6IiAgAPfeey+uXLmidwxDr4m33nrL7M63sed29uzZdc5jwoQJevtYy3MLwOD7WCaT4Z133pH2sZTntinfOW35WfzJJ5+gU6dOsLOzw6BBg/Dvv/+27MQaXOyAjG78+PFixYoV4tSpUyImJkbcdNNNomPHjqKwsFDaZ+TIkeKBBx4QV69elS55eXnS9ZWVlaJnz55i7Nix4vjx4+LPP/8UXl5eYuHChdI+iYmJwsHBQSxYsEDEx8eLjz76SCgUCvH333+32bkuWrRI9OjRQ+88MjMzpesfeughERQUJLZv3y6OHDkibrjhBjFkyBCLO0+djIwMvXPdunWrACB27twphLD85/XPP/8UL730ktiwYYMAIH755Re969966y3h6uoqNm7cKGJjY8XkyZNFSEiIKCkpkfaZMGGC6N27t/jnn3/E3r17RefOncXMmTOl6/Py8oSvr6+4++67xalTp8RPP/0k7O3txRdffCHts3//fqFQKMTy5ctFfHy8ePnll4Wtra04efJkm51vbm6uGDt2rFizZo1ISEgQBw8eFAMHDhRRUVF6xwgODhZLly7Ve85rvtfN5Xwbe25nzZolJkyYoHceOTk5evtYy3MrhNA7z6tXr4pvvvlGyGQyceHCBWkfS3lum/Kd01afxatXrxZKpVJ88803Ii4uTjzwwAPCzc1NpKenN/u8GMyYmYyMDAFA7N69W9o2cuRI8eSTT9Z7mz///FPI5XKRlpYmbfvss8+Ei4uLKCsrE0II8dxzz4kePXro3W769Oli/PjxrXsCDWhokdDc3Fxha2sr1q1bJ207ffq0ACAOHjwohLCc86zPk08+KcLCwoRGoxFCWM/zKoSo8wWg0WiEn5+feOedd6Rtubm5QqVSiZ9++kkIIaQFYA8fPizt89dffwmZTCYuX74shBDi008/Fe7u7tL5CqFd7LZbt27S39OmTROTJk3Sa8+gQYPEgw8+2KrnWJOhL7za/v33XwFAXLx4UdoWHBwsPvjgg3pvY47nW18wM2XKlHpvY+3P7ZQpU8SYMWP0tlnicytE3e+ctvwsHjhwoHj00Uelv9VqtQgICBBvvvlms8+Dw0xmJi8vDwDg4eGht/2HH36Al5cXevbsiYULF6K4uFi67uDBg4iMjISvr6+0bfz48cjPz0dcXJy0z9ixY/WOOX78eBw8eNBYp2LQuXPnEBAQgNDQUNx9991ISUkBABw9ehQVFRV6bQwPD0fHjh2lNlrSedZWXl6OVatWYc6cOXqLnVrL81pbUlIS0tLS9Nrm6uqKQYMG6T2fbm5u6N+/v7TP2LFjIZfLcejQIWmfESNG6C1cO378eJw5cwbXrl2T9jHHxyAvLw8ymQxubm5629966y14enqib9++eOedd/S65i3pfHft2gUfHx9069YNDz/8sN4CxNb83Kanp+OPP/7A/fffX+c6S3xua3/ntNVncXl5OY4ePaq3j1wux9ixY1t0vkZdzoCaR6PRYP78+Rg6dKje2lN33XUXgoODERAQgBMnTuD555/HmTNnsGHDBgBAWlqa3osKgPS3br2r+vbJz89HSUkJ7O3tjXlqAIBBgwZh5cqV6NatG65evYolS5Zg+PDhOHXqFNLS0qBUKut88Pv6+jZ6DrrrGtqnLc/TkI0bNyI3NxezZ8+WtlnL82qIrn2G2laz7T4+PnrX29jYwMPDQ2+f2gvK1nwM3N3d630MdMcwhdLSUjz//POYOXOm3uJ7TzzxBPr16wcPDw8cOHAACxcuxNWrV/H+++8DsJzznTBhAm6//XaEhITgwoULePHFFzFx4kQcPHgQCoXCqp/bb7/9Fs7Ozrj99tv1tlvic2voO6etPouvXbsGtVptcJ+EhIRmnwuDGTPy6KOP4tSpU9i3b5/e9nnz5kn/j4yMhL+/P6Kjo3HhwgWEhYW1dTNbbOLEidL/e/XqhUGDBiE4OBhr16412ZduW/n6668xceJEBAQESNus5XklfRUVFZg2bRqEEPjss8/0rluwYIH0/169ekGpVOLBBx/Em2++afJy8M0xY8YM6f+RkZHo1asXwsLCsGvXLkRHR5uwZcb3zTff4O6774adnZ3edkt8buv7zrFEHGYyE4899hh+//137Ny5E4GBgQ3uO2jQIADA+fPnAQB+fn51Ms11f/v5+TW4j4uLi8kCCTc3N3Tt2hXnz5+Hn58fysvLkZubW6eNjZ2D7rqG9jHleV68eBHbtm3D3LlzG9zPWp5XoLp9htpWs+0ZGRl611dWViInJ6dVnnPd9W1JF8hcvHgRW7du1euVMWTQoEGorKyU1qOztPPVCQ0NhZeXl95r19qeWwDYu3cvzpw50+h7GTD/57a+75y2+iz28vKCQqFotfNlMGNiQgg89thj+OWXX7Bjx4463ZCG6Bba9Pf3BwAMHjwYJ0+e1Pvw0H2Qdu/eXdpn+/btesfZunUrBg8e3Epn0nyFhYW4cOEC/P39ERUVBVtbW702njlzBikpKVIbLfU8V6xYAR8fH0yaNKnB/azleQWAkJAQ+Pn56bUtPz8fhw4d0ns+c3NzcfToUWmfHTt2QKPRSIHd4MGDsWfPHlRUVEj7bN26Fd26dYO7u7u0jzk8BrpA5ty5c9i2bRs8PT0bvU1MTAzkcrk0JGNJ51vTpUuXkJ2drffatabnVufrr79GVFQUevfu3ei+5vrcNvad01afxUqlElFRUXr7aDQabN++vWXn2+yUYWpVDz/8sHB1dRW7du3Sm9JXXFwshBDi/PnzYunSpeLIkSMiKSlJbNq0SYSGhooRI0ZIx9BNkxs3bpyIiYkRf//9t/D29jY4Te7ZZ58Vp0+fFp988kmbT1l++umnxa5du0RSUpLYv3+/GDt2rPDy8hIZGRlCCO10wI4dO4odO3aII0eOiMGDB4vBgwdb3HnWpFarRceOHcXzzz+vt90anteCggJx/Phxcfz4cQFAvP/+++L48ePS7J233npLuLm5iU2bNokTJ06IKVOmGJya3bdvX3Ho0CGxb98+0aVLF73pu7m5ucLX11fcc8894tSpU2L16tXCwcGhznRWGxsb8e6774rTp0+LRYsWGWX6bkPnW15eLiZPniwCAwNFTEyM3ntZN7vjwIED4oMPPhAxMTHiwoULYtWqVcLb21vce++9Zne+DZ1rQUGBeOaZZ8TBgwdFUlKS2LZtm+jXr5/o0qWLKC0tlY5hLc+tTl5ennBwcBCfffZZndtb0nPb2HeOEG33Wbx69WqhUqnEypUrRXx8vJg3b55wc3PTmyXVVAxmTAyAwcuKFSuEEEKkpKSIESNGCA8PD6FSqUTnzp3Fs88+q1ePRAghkpOTxcSJE4W9vb3w8vISTz/9tKioqNDbZ+fOnaJPnz5CqVSK0NBQ6T7ayvTp04W/v79QKpWiQ4cOYvr06eL8+fPS9SUlJeKRRx4R7u7uwsHBQdx2223i6tWresewhPOsafPmzQKAOHPmjN52a3hed+7cafC1O2vWLCGEdnr2K6+8Inx9fYVKpRLR0dF1Hofs7Gwxc+ZM4eTkJFxcXMR9990nCgoK9PaJjY0Vw4YNEyqVSnTo0EG89dZbddqydu1a0bVrV6FUKkWPHj3EH3/80abnm5SUVO97WVdX6OjRo2LQoEHC1dVV2NnZiYiICPHGG2/oBQDmcr4NnWtxcbEYN26c8Pb2Fra2tiI4OFg88MADdb6ArOW51fniiy+Evb29yM3NrXN7S3puG/vOEaJtP4s/+ugj0bFjR6FUKsXAgQPFP//806LzklWdHBEREZFFYs4MERERWTQGM0RERGTRGMwQERGRRWMwQ0RERBaNwQwRERFZNAYzREREZNEYzBAREZFFYzBDREREFo3BDBEREVk0BjNERERk0RjMEBERkUVjMENEREQW7f8BLQm4Y4WLagoAAAAASUVORK5CYII=",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "df_smoothed_w_bins[df_w_bins[\"MedHouseVal_bins\"] == 5][\"Longitude\"].plot(\n",
-    "    title=\"Longitude feature values after calibration\\n(only slight change in values)\"\n",
-    ");"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "771a9011",
-   "metadata": {},
-   "source": [
-    "# Data preparation"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "6d1a2d52",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_train, df_valid = train_test_split(df, test_size=0.2, random_state=1)\n",
-    "df_valid, df_test = train_test_split(df_valid, test_size=0.5, random_state=1)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "78a5061f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "continuous_cols = df.drop(columns=[\"MedHouseVal\"]).columns.values.tolist()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "61ef884d",
-   "metadata": {},
-   "source": [
-    "# Model with LDS & FDS"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "id": "368867f4",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n",
-      "  self._check_inputs(cat_embed_cols)\n"
-     ]
-    }
-   ],
-   "source": [
-    "# deeptabular\n",
-    "tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\n",
-    "X_tab_train = tab_preprocessor.fit_transform(df_train)\n",
-    "X_tab_valid = tab_preprocessor.transform(df_valid)\n",
-    "X_tab_test = tab_preprocessor.transform(df_test)\n",
-    "\n",
-    "# target\n",
-    "y_train = df_train[\"MedHouseVal\"].values\n",
-    "y_valid = df_valid[\"MedHouseVal\"].values\n",
-    "y_test = df_test[\"MedHouseVal\"].values\n",
-    "\n",
-    "X_train = {\"X_tab\": X_tab_train, \"target\": y_train}\n",
-    "X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "id": "1329d458",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "WideDeep(\n",
-       "  (deeptabular): TabMlp(\n",
-       "    (cont_norm): Identity()\n",
-       "    (encoder): MLP(\n",
-       "      (mlp): Sequential(\n",
-       "        (dense_layer_0): Sequential(\n",
-       "          (0): Linear(in_features=8, out_features=200, bias=True)\n",
-       "          (1): ReLU(inplace=True)\n",
-       "          (2): Dropout(p=0.1, inplace=False)\n",
-       "        )\n",
-       "        (dense_layer_1): Sequential(\n",
-       "          (0): Linear(in_features=200, out_features=100, bias=True)\n",
-       "          (1): ReLU(inplace=True)\n",
-       "          (2): Dropout(p=0.1, inplace=False)\n",
-       "        )\n",
-       "      )\n",
-       "    )\n",
-       "  )\n",
-       "  (fds_layer): FDSLayer(\n",
-       "    (pred_layer): Linear(in_features=100, out_features=1, bias=True)\n",
-       "  )\n",
-       ")"
-      ]
-     },
-     "execution_count": 15,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "deeptabular = TabMlp(\n",
-    "    column_idx=tab_preprocessor.column_idx,\n",
-    "    continuous_cols=tab_preprocessor.continuous_cols,\n",
-    ")\n",
-    "model = WideDeep(deeptabular=deeptabular, with_fds=True)\n",
-    "model"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "id": "2b42ad10",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "epoch 1: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 165.40it/s, loss=0.591, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 218.64it/s, loss=0.479, metrics={}]\n",
-      "FDS update: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 331/331 [00:00<00:00, 366.86it/s]\n",
-      "epoch 2: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:01<00:00, 182.78it/s, loss=0.497, metrics={}]\n",
-      "valid: 100%|███████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 214.30it/s, loss=0.47, metrics={}]\n",
-      "FDS update: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 331/331 [00:00<00:00, 350.68it/s]\n",
-      "epoch 3: 100%|████████████████████████████████████████████████████████████████████████| 331/331 [00:04<00:00, 81.28it/s, loss=0.52, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 146.50it/s, loss=0.452, metrics={}]\n",
-      "FDS update: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 122.88it/s]\n",
-      "epoch 4: 100%|███████████████████████████████████████████████████████████████████████| 331/331 [00:03<00:00, 99.77it/s, loss=0.508, metrics={}]\n",
-      "valid: 100%|███████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 157.48it/s, loss=0.45, metrics={}]\n",
-      "FDS update: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 147.92it/s]\n",
-      "epoch 5: 100%|███████████████████████████████████████████████████████████████████████| 331/331 [00:03<00:00, 93.21it/s, loss=0.591, metrics={}]\n",
-      "valid: 100%|███████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 154.60it/s, loss=0.45, metrics={}]\n",
-      "FDS update: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 145.73it/s]\n",
-      "predict: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 243.46it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "test RMSE: 0.7417540528440087\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Optimizers\n",
-    "deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n",
-    "# LR Scheduler\n",
-    "deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n",
-    "# Hyperparameters\n",
-    "trainer = Trainer(\n",
-    "    model,\n",
-    "    objective=\"huber\",\n",
-    "    lr_schedulers={\"deeptabular\": deep_sch},\n",
-    "    initializers={\n",
-    "        \"deeptabular\": XavierNormal,\n",
-    "        \"fds_layer\": XavierNormal,\n",
-    "        # \"FDS_dropout\": XavierNormal,\n",
-    "        # \"pred_layer\": XavierNormal,\n",
-    "    },\n",
-    "    optimizers={\"deeptabular\": deep_opt},\n",
-    "    metrics=[],\n",
-    "    with_lds=True,\n",
-    "    lds_kernel=\"gaussian\",\n",
-    "    lds_ks=5,\n",
-    "    lds_sigma=2,\n",
-    "    lds_granularity=100,\n",
-    "    lds_reweight=False,\n",
-    "    lds_y_max=None,\n",
-    "    lds_y_min=None,\n",
-    ")\n",
-    "\n",
-    "trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n",
-    "\n",
-    "print(\n",
-    "    f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "29f977db",
-   "metadata": {},
-   "source": [
-    "# Model with LDS only"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "id": "51ec337d",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "epoch 1: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 135.40it/s, loss=0.449, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 158.01it/s, loss=0.386, metrics={}]\n",
-      "epoch 2: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 137.79it/s, loss=0.377, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 156.84it/s, loss=0.399, metrics={}]\n",
-      "epoch 3: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 138.69it/s, loss=0.358, metrics={}]\n",
-      "valid: 100%|███████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 150.62it/s, loss=0.41, metrics={}]\n",
-      "epoch 4: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 131.96it/s, loss=0.339, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 146.01it/s, loss=0.321, metrics={}]\n",
-      "epoch 5: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 136.04it/s, loss=0.331, metrics={}]\n",
-      "valid: 100%|███████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 173.22it/s, loss=0.32, metrics={}]\n",
-      "predict: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 296.77it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "test RMSE: 0.6000006967500053\n"
-     ]
-    }
-   ],
-   "source": [
-    "deeptabular = TabMlp(\n",
-    "    column_idx=tab_preprocessor.column_idx,\n",
-    "    continuous_cols=tab_preprocessor.continuous_cols,\n",
-    ")\n",
-    "model = WideDeep(deeptabular=deeptabular, fds=False)\n",
-    "\n",
-    "# Optimizers\n",
-    "deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n",
-    "# LR Scheduler\n",
-    "deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n",
-    "# Hyperparameters\n",
-    "trainer = Trainer(\n",
-    "    model,\n",
-    "    objective=\"huber\",\n",
-    "    lr_schedulers={\"deeptabular\": deep_sch},\n",
-    "    initializers={\"deeptabular\": XavierNormal},\n",
-    "    optimizers={\"deeptabular\": deep_opt},\n",
-    "    metrics=[],\n",
-    "    with_lds=True,\n",
-    "    lds_kernel=\"gaussian\",\n",
-    "    lds_ks=5,\n",
-    "    lds_sigma=2,\n",
-    "    lds_granularity=100,\n",
-    "    lds_reweight=False,\n",
-    "    lds_y_max=None,\n",
-    "    lds_y_min=None,\n",
-    ")\n",
-    "\n",
-    "trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n",
-    "\n",
-    "print(\n",
-    "    f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8ee72521",
-   "metadata": {},
-   "source": [
-    "# Model without FDS or LDS"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 18,
-   "id": "0836c4e9",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "epoch 1: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 129.54it/s, loss=0.445, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 137.64it/s, loss=0.427, metrics={}]\n",
-      "epoch 2: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 135.98it/s, loss=0.374, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 148.50it/s, loss=0.389, metrics={}]\n",
-      "epoch 3: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 127.72it/s, loss=0.359, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 147.63it/s, loss=0.383, metrics={}]\n",
-      "epoch 4: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 136.54it/s, loss=0.339, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 146.55it/s, loss=0.323, metrics={}]\n",
-      "epoch 5: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 131.18it/s, loss=0.331, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 174.87it/s, loss=0.318, metrics={}]\n",
-      "predict: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 287.25it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "test RMSE: 0.6014019159826868\n"
-     ]
-    }
-   ],
-   "source": [
-    "deeptabular = TabMlp(\n",
-    "    column_idx=tab_preprocessor.column_idx,\n",
-    "    continuous_cols=tab_preprocessor.continuous_cols,\n",
-    ")\n",
-    "model = WideDeep(deeptabular=deeptabular, fds=False)\n",
-    "\n",
-    "# Optimizers\n",
-    "deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n",
-    "# LR Scheduler\n",
-    "deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n",
-    "# Hyperparameters\n",
-    "trainer = Trainer(\n",
-    "    model,\n",
-    "    objective=\"huber\",\n",
-    "    lr_schedulers={\"deeptabular\": deep_sch},\n",
-    "    initializers={\"deeptabular\": XavierNormal},\n",
-    "    optimizers={\"deeptabular\": deep_opt},\n",
-    "    metrics=[],\n",
-    "    with_lds=False,\n",
-    ")\n",
-    "\n",
-    "trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n",
-    "\n",
-    "print(\n",
-    "    f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "09055e82-c289-4e5d-a697-fd32bb68d018",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.13"
-  },
-  "toc": {
-   "base_numbering": 1,
-   "nav_menu": {},
-   "number_sections": true,
-   "sideBar": true,
-   "skip_h1_title": false,
-   "title_cell": "Table of Contents",
-   "title_sidebar": "Contents",
-   "toc_cell": false,
-   "toc_position": {},
-   "toc_section_display": true,
-   "toc_window_display": false
-  },
-  "vscode": {
-   "interpreter": {
-    "hash": "bee110fa72fc220f84be99700c69baf478c6696e63cfda5b1944123ebc470d26"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
diff --git a/examples/notebooks/16_Self_Supervised_Pretraning_pt1.ipynb b/examples/notebooks/15_Self_Supervised_Pretraning_pt1.ipynb
similarity index 100%
rename from examples/notebooks/16_Self_Supervised_Pretraning_pt1.ipynb
rename to examples/notebooks/15_Self_Supervised_Pretraning_pt1.ipynb
diff --git a/examples/notebooks/16_Self_Supervised_Pretraning_pt2.ipynb b/examples/notebooks/15_Self_Supervised_Pretraning_pt2.ipynb
similarity index 100%
rename from examples/notebooks/16_Self_Supervised_Pretraning_pt2.ipynb
rename to examples/notebooks/15_Self_Supervised_Pretraning_pt2.ipynb
diff --git a/examples/notebooks/17_Usign_a_custom_hugging_face_model.ipynb b/examples/notebooks/16_Usign_a_custom_hugging_face_model.ipynb
similarity index 100%
rename from examples/notebooks/17_Usign_a_custom_hugging_face_model.ipynb
rename to examples/notebooks/16_Usign_a_custom_hugging_face_model.ipynb
diff --git a/examples/notebooks/18_feature_importance_via_attention_weights.ipynb b/examples/notebooks/17_feature_importance_via_attention_weights.ipynb
similarity index 100%
rename from examples/notebooks/18_feature_importance_via_attention_weights.ipynb
rename to examples/notebooks/17_feature_importance_via_attention_weights.ipynb
diff --git a/examples/notebooks/19_wide_and_deep_for_recsys_pt1.ipynb b/examples/notebooks/18_wide_and_deep_for_recsys_pt1.ipynb
similarity index 100%
rename from examples/notebooks/19_wide_and_deep_for_recsys_pt1.ipynb
rename to examples/notebooks/18_wide_and_deep_for_recsys_pt1.ipynb
diff --git a/examples/notebooks/19_wide_and_deep_for_recsys_pt2.ipynb b/examples/notebooks/18_wide_and_deep_for_recsys_pt2.ipynb
similarity index 100%
rename from examples/notebooks/19_wide_and_deep_for_recsys_pt2.ipynb
rename to examples/notebooks/18_wide_and_deep_for_recsys_pt2.ipynb
diff --git a/examples/notebooks/20_load_from_folder_functionality.ipynb b/examples/notebooks/19_load_from_folder_functionality.ipynb
similarity index 100%
rename from examples/notebooks/20_load_from_folder_functionality.ipynb
rename to examples/notebooks/19_load_from_folder_functionality.ipynb
diff --git a/examples/notebooks/21_Using_huggingface_within_widedeep.ipynb b/examples/notebooks/20_Using_huggingface_within_widedeep.ipynb
similarity index 100%
rename from examples/notebooks/21_Using_huggingface_within_widedeep.ipynb
rename to examples/notebooks/20_Using_huggingface_within_widedeep.ipynb
diff --git a/examples/scripts/california_housing_fds_lds.py b/examples/scripts/california_housing_fds_lds.py
deleted file mode 100644
index 7fd002e7..00000000
--- a/examples/scripts/california_housing_fds_lds.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import torch
-import pandas as pd
-
-from pytorch_widedeep import Trainer
-from pytorch_widedeep.models import TabMlp, WideDeep
-from pytorch_widedeep.datasets import load_california_housing
-from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint
-from pytorch_widedeep.preprocessing import TabPreprocessor
-
-use_cuda = torch.cuda.is_available()
-
-if __name__ == "__main__":
-    df: pd.DataFrame = load_california_housing(as_frame=True)
-
-    target = df.MedHouseVal.values
-    df = df.drop("MedHouseVal", axis=1)
-
-    continuous_cols = df.columns.tolist()
-    tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)
-    X_tab = tab_preprocessor.fit_transform(df)
-
-    tab_mlp = TabMlp(
-        column_idx=tab_preprocessor.column_idx,
-        continuous_cols=continuous_cols,
-        mlp_hidden_dims=[200, 100],
-        mlp_dropout=[0.2, 0.2],
-    )
-    model = WideDeep(deeptabular=tab_mlp, with_fds=True, enforce_positive=True)
-
-    model_checkpoint = ModelCheckpoint(
-        filepath="model_weights/wd_out",
-        save_best_only=True,
-        max_save=1,
-    )
-    early_stopping = EarlyStopping(patience=5)
-    callbacks = [early_stopping, model_checkpoint]
-
-    trainer = Trainer(
-        model,
-        objective="regression",
-        callbacks=callbacks,
-    )
-
-    trainer.fit(
-        X_tab=X_tab,
-        target=target,
-        n_epochs=2,
-        batch_size=256,
-        val_split=0.2,
-        with_lds=True,
-        lds_kernel="triang",
-        lds_granularity=200,
-    )
diff --git a/examples/scripts/readme_snippets.py b/examples/scripts/readme_snippets.py
index 53b305f6..9e230e8c 100644
--- a/examples/scripts/readme_snippets.py
+++ b/examples/scripts/readme_snippets.py
@@ -407,7 +407,90 @@ def output_dim(self):
 )
 
 
-# 7. Simply Tabular with a multi-target loss
+# 7. A Two tower model
+np.random.seed(42)
+
+# user_features dataframe
+user_ids = np.arange(1, 101)
+ages = np.random.randint(18, 60, size=100)
+genders = np.random.choice(["male", "female"], size=100)
+locations = np.random.choice(["city_a", "city_b", "city_c", "city_d"], size=100)
+user_features = pd.DataFrame(
+    {"id": user_ids, "age": ages, "gender": genders, "location": locations}
+)
+
+# item_features dataframe
+item_ids = np.arange(1, 101)
+prices = np.random.uniform(10, 500, size=100).round(2)
+colors = np.random.choice(["red", "blue", "green", "black"], size=100)
+categories = np.random.choice(["electronics", "clothing", "home", "toys"], size=100)
+
+item_features = pd.DataFrame(
+    {"id": item_ids, "price": prices, "color": colors, "category": categories}
+)
+
+# Interactions dataframe
+interaction_user_ids = np.random.choice(user_ids, size=1000)
+interaction_item_ids = np.random.choice(item_ids, size=1000)
+purchased = np.random.choice([0, 1], size=1000, p=[0.7, 0.3])
+interactions = pd.DataFrame(
+    {
+        "user_id": interaction_user_ids,
+        "item_id": interaction_item_ids,
+        "purchased": purchased,
+    }
+)
+user_item_purchased = interactions.merge(
+    user_features, left_on="user_id", right_on="id"
+).merge(item_features, left_on="item_id", right_on="id")
+
+
+# Users
+tab_preprocessor_user = TabPreprocessor(
+    cat_embed_cols=["gender", "location"],
+    continuous_cols=["age"],
+)
+X_user = tab_preprocessor_user.fit_transform(user_item_purchased)
+tab_mlp_user = TabMlp(
+    column_idx=tab_preprocessor_user.column_idx,
+    cat_embed_input=tab_preprocessor_user.cat_embed_input,
+    continuous_cols=["age"],
+    mlp_hidden_dims=[16, 8],
+    mlp_dropout=[0.2, 0.2],
+)
+
+# Items
+tab_preprocessor_item = TabPreprocessor(
+    cat_embed_cols=["color", "category"],
+    continuous_cols=["price"],
+)
+X_item = tab_preprocessor_item.fit_transform(user_item_purchased)
+tab_mlp_item = TabMlp(
+    column_idx=tab_preprocessor_item.column_idx,
+    cat_embed_input=tab_preprocessor_item.cat_embed_input,
+    continuous_cols=["price"],
+    mlp_hidden_dims=[16, 8],
+    mlp_dropout=[0.2, 0.2],
+)
+
+two_tower_model = ModelFuser([tab_mlp_user, tab_mlp_item], fusion_method="dot")
+
+model = WideDeep(deeptabular=two_tower_model)
+
+trainer = Trainer(
+    model,
+    objective="binary",
+)
+
+trainer.fit(
+    X_tab=[X_user, X_item],
+    target=interactions.purchased.values,
+    n_epochs=1,
+    batch_size=32,
+)
+
+
+# 8. Simply Tabular with a multi-target loss
 
 # let's add a second target to the dataframe
 df["target2"] = [random.choice([0, 1]) for _ in range(100)]
diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml
index 0f4165b5..3f8347cf 100644
--- a/mkdocs/mkdocs.yml
+++ b/mkdocs/mkdocs.yml
@@ -52,15 +52,14 @@ nav:
         - 12_ZILNLoss_origkeras_vs_pytorch_widedeep: examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.ipynb
         - 13_model_uncertainty_prediction: examples/13_model_uncertainty_prediction.ipynb
         - 14_bayesian_models: examples/14_bayesian_models.ipynb
-        - 15_DIR-LDS_and_FDS: examples/15_DIR-LDS_and_FDS.ipynb
-        - 16_Self-Supervised Pre-Training pt 1: examples/16_Self_Supervised_Pretraning_pt1.ipynb
-        - 16_Self-Supervised Pre-Training pt 2: examples/16_Self_Supervised_Pretraning_pt2.ipynb
-        - 17_Usign-a-custom-hugging-face-model: examples/17_Usign_a_custom_hugging_face_model.ipynb
-        - 18_feature_importance_via_attention_weights: examples/18_feature_importance_via_attention_weights.ipynb
-        - 19_wide_and_deep_for_recsys_pt1: examples/19_wide_and_deep_for_recsys_pt1.ipynb
-        - 19_wide_and_deep_for_recsys_pt2: examples/19_wide_and_deep_for_recsys_pt2.ipynb
-        - 20_load_from_folder_functionality: examples/20_load_from_folder_functionality.ipynb
-        - 21-Using-huggingface-within-widedeep: examples/21_Using_huggingface_within_widedeep.ipynb
+        - 15_Self-Supervised Pre-Training pt 1: examples/16_Self_Supervised_Pretraning_pt1.ipynb
+        - 15_Self-Supervised Pre-Training pt 2: examples/16_Self_Supervised_Pretraning_pt2.ipynb
+        - 16_Usign-a-custom-hugging-face-model: examples/17_Usign_a_custom_hugging_face_model.ipynb
+        - 17_feature_importance_via_attention_weights: examples/18_feature_importance_via_attention_weights.ipynb
+        - 18_wide_and_deep_for_recsys_pt1: examples/19_wide_and_deep_for_recsys_pt1.ipynb
+        - 18_wide_and_deep_for_recsys_pt2: examples/19_wide_and_deep_for_recsys_pt2.ipynb
+        - 19_load_from_folder_functionality: examples/20_load_from_folder_functionality.ipynb
+        - 20-Using-huggingface-within-widedeep: examples/21_Using_huggingface_within_widedeep.ipynb
     - Contributing: contributing.md
 
 theme:
diff --git a/mkdocs/site/404.html b/mkdocs/site/404.html
index c1f720b7..fe1eed8b 100644
--- a/mkdocs/site/404.html
+++ b/mkdocs/site/404.html
@@ -12,7 +12,7 @@
       
       
       <link rel="icon" href="/assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -20,7 +20,7 @@
       
     
     
-      <link rel="stylesheet" href="/assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="/assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="/assets/stylesheets/palette.06af60db.min.css">
@@ -165,7 +165,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -182,7 +182,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -357,7 +357,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -960,8 +960,6 @@
         
       
         
-      
-        
       
         
       
@@ -1299,32 +1297,11 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="/examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="/examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1341,11 +1318,11 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="/examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1362,11 +1339,11 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="/examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1383,11 +1360,11 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="/examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1404,11 +1381,11 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="/examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1425,11 +1402,11 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="/examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1446,11 +1423,11 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="/examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1467,11 +1444,11 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="/examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1559,7 +1536,7 @@ <h1>404 - Not found</h1>
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -1577,7 +1554,7 @@ <h1>404 - Not found</h1>
     <script id="__config" type="application/json">{"base": "/", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "/assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="/assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="/assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="/stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/assets/_mkdocstrings.css b/mkdocs/site/assets/_mkdocstrings.css
index 4b7d98b8..85449ec7 100644
--- a/mkdocs/site/assets/_mkdocstrings.css
+++ b/mkdocs/site/assets/_mkdocstrings.css
@@ -26,6 +26,11 @@
   float: right;
 }
 
+/* Backward-compatibility: docstring section titles in bold. */
+.doc-section-title {
+  font-weight: bold;
+}
+
 /* Symbols in Navigation and ToC. */
 :root,
 [data-md-color-scheme="default"] {
@@ -106,4 +111,9 @@ code.doc-symbol-module {
 
 code.doc-symbol-module::after {
   content: "mod";
-}
\ No newline at end of file
+}
+
+.doc-signature .autorefs {
+  color: inherit;
+  border-bottom: 1px dotted currentcolor;
+}
diff --git a/mkdocs/site/contributing.html b/mkdocs/site/contributing.html
index 57210de3..0f6e3785 100644
--- a/mkdocs/site/contributing.html
+++ b/mkdocs/site/contributing.html
@@ -11,12 +11,12 @@
         <link rel="canonical" href="https://pytorch-widedeep.readthedocs.io/contributing.html">
       
       
-        <link rel="prev" href="examples/21_Using_huggingface_within_widedeep.html">
+        <link rel="prev" href="examples/14_bayesian_models.html">
       
       
       
       <link rel="icon" href="assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -24,7 +24,7 @@
       
     
     
-      <link rel="stylesheet" href="assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="assets/stylesheets/palette.06af60db.min.css">
@@ -169,7 +169,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -186,7 +186,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -363,7 +363,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -966,8 +966,6 @@
         
       
         
-      
-        
       
         
       
@@ -1305,32 +1303,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1347,11 +1324,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1368,11 +1345,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1389,11 +1366,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1410,11 +1387,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1431,11 +1408,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1452,11 +1429,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1473,11 +1450,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1622,7 +1599,7 @@ <h1>Contributing</h1>
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -1640,7 +1617,7 @@ <h1>Contributing</h1>
     <script id="__config" type="application/json">{"base": ".", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/index.html b/mkdocs/site/index.html
index 95163aff..78bf4577 100644
--- a/mkdocs/site/index.html
+++ b/mkdocs/site/index.html
@@ -16,7 +16,7 @@
       
       
       <link rel="icon" href="assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -24,7 +24,7 @@
       
     
     
-      <link rel="stylesheet" href="assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="assets/stylesheets/palette.06af60db.min.css">
@@ -174,7 +174,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -191,7 +191,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -368,7 +368,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -981,8 +981,6 @@
         
       
         
-      
-        
       
         
       
@@ -1320,32 +1318,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1362,11 +1339,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1383,11 +1360,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1404,11 +1381,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1425,11 +1402,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1446,11 +1423,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1467,11 +1444,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1488,11 +1465,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -2270,8 +2247,7 @@ <h4 id="apa">APA<a class="headerlink" href="#apa" title="Permanent link">&para;<
       
         <a href="mailto:jrzaurin@gmail.com">Javier</a>, 
         <a href="mailto:mulinka.pavol@gmail.com">Pavol Mulinka</a>, 
-        <a href="mailto:javierrodriguezzaurin@javiers-macbook-pro.local">Javier Rodriguez Zaurin</a>, 
-        <a href="mailto:not.committed.yet">Not Committed Yet</a>
+        <a href="mailto:javierrodriguezzaurin@javiers-macbook-pro.local">Javier Rodriguez Zaurin</a>
     </nav>
   </span>
 
@@ -2321,7 +2297,7 @@ <h4 id="apa">APA<a class="headerlink" href="#apa" title="Permanent link">&para;<
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -2339,7 +2315,7 @@ <h4 id="apa">APA<a class="headerlink" href="#apa" title="Permanent link">&para;<
     <script id="__config" type="application/json">{"base": ".", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/installation.html b/mkdocs/site/installation.html
index 58627b83..0cc41b2a 100644
--- a/mkdocs/site/installation.html
+++ b/mkdocs/site/installation.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -370,7 +370,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1022,8 +1022,6 @@
         
       
         
-      
-        
       
         
       
@@ -1361,32 +1359,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1403,11 +1380,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1424,11 +1401,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1445,11 +1422,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1466,11 +1443,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1487,11 +1464,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1508,11 +1485,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1529,11 +1506,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1699,7 +1676,7 @@ <h2 id="dependencies">Dependencies<a class="headerlink" href="#dependencies" tit
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -1717,7 +1694,7 @@ <h2 id="dependencies">Dependencies<a class="headerlink" href="#dependencies" tit
     <script id="__config" type="application/json">{"base": ".", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/objects.inv b/mkdocs/site/objects.inv
index f82a2a58..a976c4c7 100644
Binary files a/mkdocs/site/objects.inv and b/mkdocs/site/objects.inv differ
diff --git a/mkdocs/site/pytorch-widedeep/bayesian_models.html b/mkdocs/site/pytorch-widedeep/bayesian_models.html
index aecb23bd..5c3b989d 100644
--- a/mkdocs/site/pytorch-widedeep/bayesian_models.html
+++ b/mkdocs/site/pytorch-widedeep/bayesian_models.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1036,8 +1036,6 @@
         
       
         
-      
-        
       
         
       
@@ -1375,11 +1373,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1396,11 +1394,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1417,11 +1415,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1438,11 +1436,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1459,11 +1457,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1480,11 +1478,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1501,11 +1499,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1522,32 +1520,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1616,11 +1593,11 @@ <h1 id="the-bayesian-models-module">The <code>bayesian models</code> module<a cl
 
 
 <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">BayesianWide</span>
+            <span class="doc doc-object-name doc-class-name">BayesianWide</span>
 
 
 <a href="#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">BayesianWide</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">BayesianWide</span><span class="p">(</span>
     <span class="n">input_dim</span><span class="p">,</span>
     <span class="n">pred_dim</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
     <span class="n">prior_sigma_1</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
@@ -1631,47 +1608,46 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.bayesian_models._base_bayesian_model.BaseBayesianModel">BaseBayesianModel</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.bayesian_models._base_bayesian_model.BaseBayesianModel">BaseBayesianModel</span></code></p>
+
 
-  
       <p>Defines a <code>Wide</code> model. This is a linear model where the
 non-linearlities are captured via crossed-columns</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input_dim</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input_dim</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the
 individual values for all the features that go through the wide
 component. For example, if the wide component receives 2 features with
 5 individual values each, <code>input_dim = 10</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pred_dim</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pred_dim</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>size of the ouput tensor containing the predictions</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>prior_sigma_1</code></b>
-            (<code>float</code>, default:
-                <code>1.0</code>
+          –
+          <div class="doc-md-description">
+            <p>size of the ouput tensor containing the predictions</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>prior_sigma_1</code></b>
+              (<code>float</code>, default:
+                  <code>1.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The prior weight distribution is a scaled mixture of two Gaussian
+          –
+          <div class="doc-md-description">
+            <p>The prior weight distribution is a scaled mixture of two Gaussian
 densities:</p>
 <div class="arithmatex">\[
    \begin{aligned}
@@ -1681,39 +1657,39 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two
 Gaussians that will be mixed to produce the prior weight
 distribution.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>prior_sigma_2</code></b>
-            (<code>float</code>, default:
-                <code>0.002</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>prior_sigma_2</code></b>
+              (<code>float</code>, default:
+                  <code>0.002</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Prior of the sigma parameter for the second of the two Gaussian
+          –
+          <div class="doc-md-description">
+            <p>Prior of the sigma parameter for the second of the two Gaussian
 distributions that will be mixed to produce the prior weight
 distribution</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>prior_pi</code></b>
-            (<code>float</code>, default:
-                <code>0.8</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>prior_pi</code></b>
+              (<code>float</code>, default:
+                  <code>0.8</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Scaling factor that will be used to mix the Gaussians to produce the
+          –
+          <div class="doc-md-description">
+            <p>Scaling factor that will be used to mix the Gaussians to produce the
 prior weight distribution</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>posterior_mu_init</code></b>
-            (<code>float</code>, default:
-                <code>0.0</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>posterior_mu_init</code></b>
+              (<code>float</code>, default:
+                  <code>0.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The posterior sample of the weights is defined as:</p>
+          –
+          <div class="doc-md-description">
+            <p>The posterior sample of the weights is defined as:</p>
 <div class="arithmatex">\[
    \begin{aligned}
    \mathbf{w} &amp;= \mu + log(1 + exp(\rho))
@@ -1728,39 +1704,37 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 \]</div>
 <p><span class="arithmatex">\(\mu\)</span> is initialised using a normal distributtion with mean
 <code>posterior_mu_init</code> and std equal to 0.1.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>posterior_rho_init</code></b>
-            (<code>float</code>, default:
-                <code>-7.0</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>posterior_rho_init</code></b>
+              (<code>float</code>, default:
+                  <code>-7.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>As in the case of <span class="arithmatex">\(\mu\)</span>, <span class="arithmatex">\(\rho\)</span> is initialised using a
+          –
+          <div class="doc-md-description">
+            <p>As in the case of <span class="arithmatex">\(\mu\)</span>, <span class="arithmatex">\(\rho\)</span> is initialised using a
 normal distributtion with mean <code>posterior_rho_init</code> and std equal to
 0.1.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide.bayesian_wide_linear">bayesian_wide_linear</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>the linear layer that comprises the wide branch of the model</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide.bayesian_wide_linear">bayesian_wide_linear</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>the linear layer that comprises the wide branch of the model</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.bayesian_models</span> <span class="kn">import</span> <span class="n">BayesianWide</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span>
@@ -1768,9 +1742,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">wide</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 83</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 83</span>
 <span class="normal"> 84</span>
 <span class="normal"> 85</span>
 <span class="normal"> 86</span>
@@ -1816,12 +1790,11 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
     <span class="p">)</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">bias</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">pred_dim</span><span class="p">))</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1832,10 +1805,10 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -1844,11 +1817,11 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 
 
 <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">BayesianTabMlp</span>
+            <span class="doc doc-object-name doc-class-name">BayesianTabMlp</span>
 
 
 <a href="#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">BayesianTabMlp</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">BayesianTabMlp</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -1871,154 +1844,153 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.bayesian_models._base_bayesian_model.BaseBayesianModel">BaseBayesianModel</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.bayesian_models._base_bayesian_model.BaseBayesianModel">BaseBayesianModel</span></code></p>
+
 
-  
       <p>Defines a <code>BayesianTabMlp</code> model.</p>
 <p>This class combines embedding representations of the categorical features
 with numerical (aka continuous) features, embedded or not. These are then
 passed through a series of probabilistic dense layers (i.e. a MLP).</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name, number of unique values and
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name, number of unique values and
 embedding dimension. e.g. <em>[(education, 11, 32), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features. Options
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features. Options
 are: 'layernorm', 'batchnorm' or None.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the continuous columns will be embedded
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the continuous columns will be embedded
 (i.e. passed each through a linear layer with or without activation)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dim</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dim</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Size of the continuous embeddings</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>Size of the continuous embeddings</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cont_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cont_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the continuous embeddings</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the continuous embeddings</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
-                <code>[200, 100]</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+                  <code>[200, 100]</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the mlp.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;leaky_relu&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the mlp.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;leaky_relu&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers of the MLP. Currently
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>prior_sigma_1</code></b>
-            (<code>float</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>prior_sigma_1</code></b>
+              (<code>float</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The prior weight distribution is a scaled mixture of two Gaussian
+          –
+          <div class="doc-md-description">
+            <p>The prior weight distribution is a scaled mixture of two Gaussian
 densities:</p>
 <div class="arithmatex">\[
    \begin{aligned}
@@ -2028,40 +2000,40 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two
 Gaussians that will be mixed to produce the prior weight
 distribution.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>prior_sigma_2</code></b>
-            (<code>float</code>, default:
-                <code>0.002</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>prior_sigma_2</code></b>
+              (<code>float</code>, default:
+                  <code>0.002</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Prior of the sigma parameter for the second of the two Gaussian
+          –
+          <div class="doc-md-description">
+            <p>Prior of the sigma parameter for the second of the two Gaussian
 distributions that will be mixed to produce the prior weight
 distribution for each Bayesian linear and embedding layer</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>prior_pi</code></b>
-            (<code>float</code>, default:
-                <code>0.8</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>prior_pi</code></b>
+              (<code>float</code>, default:
+                  <code>0.8</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Scaling factor that will be used to mix the Gaussians to produce the
+          –
+          <div class="doc-md-description">
+            <p>Scaling factor that will be used to mix the Gaussians to produce the
 prior weight distribution ffor each Bayesian linear and embedding
 layer</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>posterior_mu_init</code></b>
-            (<code>float</code>, default:
-                <code>0.0</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>posterior_mu_init</code></b>
+              (<code>float</code>, default:
+                  <code>0.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The posterior sample of the weights is defined as:</p>
+          –
+          <div class="doc-md-description">
+            <p>The posterior sample of the weights is defined as:</p>
 <p>$$
    \begin{aligned}
    \mathbf{w} &amp;= \mu + log(1 + exp(\rho))
@@ -2076,48 +2048,46 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 \]</div>
 <p><span class="arithmatex">\(\mu\)</span> is initialised using a normal distributtion with mean
 <code>posterior_mu_init</code> and std equal to 0.1.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>posterior_rho_init</code></b>
-            (<code>float</code>, default:
-                <code>-7.0</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>posterior_rho_init</code></b>
+              (<code>float</code>, default:
+                  <code>-7.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>As in the case of <span class="arithmatex">\(\mu\)</span>, <span class="arithmatex">\(\rho\)</span> is initialised using a
+          –
+          <div class="doc-md-description">
+            <p>As in the case of <span class="arithmatex">\(\mu\)</span>, <span class="arithmatex">\(\rho\)</span> is initialised using a
 normal distributtion with mean <code>posterior_rho_init</code> and std equal to
 0.1.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp.bayesian_cat_and_cont_embed">bayesian_cat_and_cont_embed</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>This is the module that processes the categorical and continuous columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp.bayesian_tab_mlp">bayesian_tab_mlp</span></code></b>
-            (<code><span title="torch.nn.Sequential">Sequential</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>mlp model that will receive the concatenation of the embeddings and
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp.bayesian_cat_and_cont_embed">bayesian_cat_and_cont_embed</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>This is the module that processes the categorical and continuous columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp.bayesian_tab_mlp">bayesian_tab_mlp</span></code></b>
+              (<code><span title="torch.nn.Sequential">Sequential</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>mlp model that will receive the concatenation of the embeddings and
 the continuous columns</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.bayesian_models</span> <span class="kn">import</span> <span class="n">BayesianTabMlp</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -2129,9 +2099,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">134</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">134</span>
 <span class="normal">135</span>
 <span class="normal">136</span>
 <span class="normal">137</span>
@@ -2369,12 +2339,11 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
         <span class="n">posterior_rho_init</span><span class="p">,</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2385,10 +2354,10 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -2464,7 +2433,7 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -2482,7 +2451,7 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/bayesian_trainer.html b/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
index f716a54e..fbaa3b6e 100644
--- a/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
+++ b/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1027,8 +1027,6 @@
         
       
         
-      
-        
       
         
       
@@ -1366,11 +1364,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1387,11 +1385,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1408,11 +1406,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1429,11 +1427,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1450,11 +1448,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1471,11 +1469,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1492,11 +1490,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1513,32 +1511,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1601,11 +1578,11 @@ <h1 id="training-deep-learning-probabilistic-models">Training Deep Learning Prob
 
 
 <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">BayesianTrainer</span>
+            <span class="doc doc-object-name doc-class-name">BayesianTrainer</span>
 
 
 <a href="#pytorch_widedeep.training.BayesianTrainer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">BayesianTrainer</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">BayesianTrainer</span><span class="p">(</span>
     <span class="n">model</span><span class="p">,</span>
     <span class="n">objective</span><span class="p">,</span>
     <span class="n">custom_loss_function</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -1619,97 +1596,96 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.training._base_bayesian_trainer.BaseBayesianTrainer">BaseBayesianTrainer</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.training._base_bayesian_trainer.BaseBayesianTrainer">BaseBayesianTrainer</span></code></p>
+
 
-  
       <p>Class to set the of attributes that will be used during the
 training process.</p>
 <p>Both the Bayesian models and the Trainer in this repo are based on the paper:
 <a href="https://arxiv.org/pdf/1505.05424.pdf">Weight Uncertainty in Neural Networks</a>.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>model</code></b>
-            (<code><span title="pytorch_widedeep.bayesian_models._base_bayesian_model.BaseBayesianModel">BaseBayesianModel</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>model</code></b>
+              (<code><span title="pytorch_widedeep.bayesian_models._base_bayesian_model.BaseBayesianModel">BaseBayesianModel</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code>
 section here in the docs.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>objective</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>Defines the objective, loss or cost function.<br/>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>objective</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>Defines the objective, loss or cost function.<br/>
 Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>,
 <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code><br/>
 Possible values are: <em>'binary'</em>, <em>'multiclass'</em>, <em>'regression'</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>custom_loss_function</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Module">Module</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>custom_loss_function</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Module">Module</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>If none of the loss functions available suits the user, it is possible
+          –
+          <div class="doc-md-description">
+            <p>If none of the loss functions available suits the user, it is possible
 to pass a custom loss function. See for example
 <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the
 object or the Examples folder in the repo.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>optimizer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>optimizer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam
+          –
+          <div class="doc-md-description">
+            <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam
 ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lr_scheduler</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>lr_scheduler</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>An instance of Pytorch's <code>LRScheduler</code> object
+          –
+          <div class="doc-md-description">
+            <p>An instance of Pytorch's <code>LRScheduler</code> object
 (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>callbacks</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>callbacks</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with <code>Callback</code> objects. The three callbacks available in
+          –
+          <div class="doc-md-description">
+            <p>List with <code>Callback</code> objects. The three callbacks available in
 <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and
 <code>EarlyStopping</code>. This can also be a custom callback. See
 <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the
 repo.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>metrics</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.metrics.Metric">Metric</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="torchmetrics.Metric">Metric</span>]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>metrics</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.metrics.Metric">Metric</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="torchmetrics.Metric">Metric</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <ul>
+          –
+          <div class="doc-md-description">
+            <ul>
 <li>List of objects of type <code>Metric</code>. Metrics available are:
   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,
   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as
@@ -1721,39 +1697,38 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
   long as it is an object of type</code>Metric<code>.
   See</code>the <a href="(https://lightning.ai/docs/torchmetrics)">instructions</a></li>
 </ul>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Setting it to 0 will print nothing during training.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>seed</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>Setting it to 0 will print nothing during training.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>seed</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Random seed to be used internally for train_test_split</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>Random seed to be used internally for train_test_split</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Other Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
+<p><span class="doc-section-title">Other Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
 <ul>
 <li>
 <p><strong>device</strong>: <code>str</code><br/>
@@ -1775,29 +1750,28 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
     learning rate is a bit particular.</p>
 </li>
 </ul>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.training.BayesianTrainer.cyclic_lr">cyclic_lr</span></code></b>
-            (<code>bool</code>)
-        –
-        <div class="doc-md-description">
-          <p>Attribute that indicates if  the lr_scheduler is cyclic_lr
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.training.BayesianTrainer.cyclic_lr">cyclic_lr</span></code></b>
+              (<code>bool</code>)
+          –
+          <div class="doc-md-description">
+            <p>Attribute that indicates if  the lr_scheduler is cyclic_lr
 (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See <code>Pytorch schedulers
 &lt;https://pytorch.org/docs/stable/optim.html&gt;</code>_.</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">115</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">115</span>
 <span class="normal">116</span>
 <span class="normal">117</span>
 <span class="normal">118</span>
@@ -1855,12 +1829,11 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
         <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1873,13 +1846,12 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit</span>
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
 <a href="#pytorch_widedeep.training.BayesianTrainer.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span>
     <span class="n">X_tab</span><span class="p">,</span>
     <span class="n">target</span><span class="p">,</span>
     <span class="n">X_tab_val</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -1893,118 +1865,117 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Fit method.</p>
+    <div class="doc doc-contents ">
 
+      <p>Fit method.</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>X_tab</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>tabular dataset</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>target values</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_tab_val</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>tabular dataset</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>target values</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab_val</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>validation data</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target_val</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>validation data</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target_val</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>validation target values</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>val_split</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>validation target values</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>val_split</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>An alterative to passing the validation set is to use a train/val
+          –
+          <div class="doc-md-description">
+            <p>An alterative to passing the validation set is to use a train/val
 split fraction via <code>val_split</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_epochs</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_epochs</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of epochs</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>validation_freq</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>number of epochs</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>validation_freq</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>epochs validation frequency</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>batch_size</code></b>
-            (<code>int</code>, default:
-                <code>32</code>
+          –
+          <div class="doc-md-description">
+            <p>epochs validation frequency</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>batch_size</code></b>
+              (<code>int</code>, default:
+                  <code>32</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>batch size</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_train_samples</code></b>
-            (<code>int</code>, default:
-                <code>2</code>
+          –
+          <div class="doc-md-description">
+            <p>batch size</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_train_samples</code></b>
+              (<code>int</code>, default:
+                  <code>2</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of samples to average over during the training process.
+          –
+          <div class="doc-md-description">
+            <p>number of samples to average over during the training process.
 See <a href="https://arxiv.org/pdf/1505.05424.pdf">Weight Uncertainty in Neural Networks</a> for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_val_samples</code></b>
-            (<code>int</code>, default:
-                <code>2</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_val_samples</code></b>
+              (<code>int</code>, default:
+                  <code>2</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of samples to average over during the validation process.
+          –
+          <div class="doc-md-description">
+            <p>number of samples to average over during the validation process.
 See <a href="https://arxiv.org/pdf/1505.05424.pdf">Weight Uncertainty in Neural Networks</a> for details.</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">145</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">145</span>
 <span class="normal">146</span>
 <span class="normal">147</span>
 <span class="normal">148</span>
@@ -2234,91 +2205,87 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">_restore_best_weights</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.training.BayesianTrainer.predict" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">predict</span>
+            <span class="doc doc-object-name doc-function-name">predict</span>
 
 
 <a href="#pytorch_widedeep.training.BayesianTrainer.predict" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">predict</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">predict</span><span class="p">(</span>
     <span class="n">X_tab</span><span class="p">,</span> <span class="n">n_samples</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">return_samples</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">256</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Returns the predictions</p>
+    <div class="doc doc-contents ">
 
+      <p>Returns the predictions</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>X_tab</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>tabular dataset</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_samples</code></b>
-            (<code>int</code>, default:
-                <code>5</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>tabular dataset</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_samples</code></b>
+              (<code>int</code>, default:
+                  <code>5</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of samples that will be either returned or averaged to
+          –
+          <div class="doc-md-description">
+            <p>number of samples that will be either returned or averaged to
 produce an overal prediction</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>return_samples</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>return_samples</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the n samples will be averaged or directly returned</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>batch_size</code></b>
-            (<code>int</code>, default:
-                <code>256</code>
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the n samples will be averaged or directly returned</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>batch_size</code></b>
+              (<code>int</code>, default:
+                  <code>256</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>batch size</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>batch size</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code>np.ndarray:</code>
-        –
-        <div class="doc-md-description">
-          <p>array with the predictions</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code>np.ndarray:</code>
+          –
+          <div class="doc-md-description">
+            <p>array with the predictions</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">261</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">261</span>
 <span class="normal">262</span>
 <span class="normal">263</span>
 <span class="normal">264</span>
@@ -2392,91 +2359,87 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict" class="doc doc-headin
     <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
         <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="n">axis</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.training.BayesianTrainer.predict_proba" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">predict_proba</span>
+            <span class="doc doc-object-name doc-function-name">predict_proba</span>
 
 
 <a href="#pytorch_widedeep.training.BayesianTrainer.predict_proba" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">predict_proba</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">predict_proba</span><span class="p">(</span>
     <span class="n">X_tab</span><span class="p">,</span> <span class="n">n_samples</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">return_samples</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">256</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Returns the predicted probabilities</p>
+    <div class="doc doc-contents ">
 
+      <p>Returns the predicted probabilities</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>X_tab</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>tabular dataset</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_samples</code></b>
-            (<code>int</code>, default:
-                <code>5</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>tabular dataset</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_samples</code></b>
+              (<code>int</code>, default:
+                  <code>5</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of samples that will be either returned or averaged to
+          –
+          <div class="doc-md-description">
+            <p>number of samples that will be either returned or averaged to
 produce an overal prediction</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>return_samples</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>return_samples</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the n samples will be averaged or directly returned</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>batch_size</code></b>
-            (<code>int</code>, default:
-                <code>256</code>
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the n samples will be averaged or directly returned</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>batch_size</code></b>
+              (<code>int</code>, default:
+                  <code>256</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>batch size</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>batch size</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>array with the probabilities per class</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>array with the probabilities per class</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">299</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">299</span>
 <span class="normal">300</span>
 <span class="normal">301</span>
 <span class="normal">302</span>
@@ -2564,30 +2527,28 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict_proba" class="doc doc-
     <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
         <span class="k">return</span> <span class="n">preds</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">save</span>
+            <span class="doc doc-object-name doc-function-name">save</span>
 
 
 <a href="#pytorch_widedeep.training.BayesianTrainer.save" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">save</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">save</span><span class="p">(</span>
     <span class="n">path</span><span class="p">,</span>
     <span class="n">save_state_dict</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
     <span class="n">model_filename</span><span class="o">=</span><span class="s2">&quot;bayesian_model.pt&quot;</span><span class="p">,</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Saves the model, training and evaluation history to disk</p>
 <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With
 that in mind, all the torch related parameters (such as optimizers or
@@ -2598,44 +2559,43 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
 any other torch model (e.g. <code>torch.save(model, path)</code>).</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>path</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>path to the directory where the model and the feature importance
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>path</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>path to the directory where the model and the feature importance
 attribute will be saved.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>save_state_dict</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>save_state_dict</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether to save directly the model or the
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether to save directly the model or the
 model's state dictionary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>model_filename</code></b>
-            (<code>str</code>, default:
-                <code>&#39;bayesian_model.pt&#39;</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>model_filename</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;bayesian_model.pt&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>filename where the model weights will be store</p>
-        </div>
-      </li>
-  </ul>
+          –
+          <div class="doc-md-description">
+            <p>filename where the model weights will be store</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">344</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">344</span>
 <span class="normal">345</span>
 <span class="normal">346</span>
 <span class="normal">347</span>
@@ -2731,8 +2691,8 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
     <span class="k">else</span><span class="p">:</span>
         <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">,</span> <span class="n">model_path</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2740,8 +2700,7 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2816,7 +2775,7 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -2834,7 +2793,7 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/callbacks.html b/mkdocs/site/pytorch-widedeep/callbacks.html
index 7360d40a..bc9f877b 100644
--- a/mkdocs/site/pytorch-widedeep/callbacks.html
+++ b/mkdocs/site/pytorch-widedeep/callbacks.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1045,8 +1045,6 @@
         
       
         
-      
-        
       
         
       
@@ -1384,11 +1382,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1405,11 +1403,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1426,11 +1424,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1447,11 +1445,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1468,11 +1466,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1489,11 +1487,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1510,11 +1508,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1531,32 +1529,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1624,40 +1601,38 @@ <h1 id="callbacks">Callbacks<a class="headerlink" href="#callbacks" title="Perma
 
 
 <h2 id="pytorch_widedeep.callbacks.LRHistory" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">LRHistory</span>
+            <span class="doc doc-object-name doc-class-name">LRHistory</span>
 
 
 <a href="#pytorch_widedeep.callbacks.LRHistory" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">LRHistory</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">LRHistory</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.callbacks.Callback">Callback</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.callbacks.Callback">Callback</span></code></p>
+
 
-  
       <p>Saves the learning rates during training in the <code>lr_history</code> attribute
 of the <code>Trainer</code>.</p>
 <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See
 <code>pytorch_widedeep.trainer.Trainer</code></p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>n_epochs</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>number of training epochs</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>n_epochs</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>number of training epochs</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.callbacks</span> <span class="kn">import</span> <span class="n">LRHistory</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabMlp</span><span class="p">,</span> <span class="n">Wide</span><span class="p">,</span> <span class="n">WideDeep</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.training</span> <span class="kn">import</span> <span class="n">Trainer</span>
@@ -1670,20 +1645,19 @@ <h2 id="pytorch_widedeep.callbacks.LRHistory" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">objective</span><span class="o">=</span><span class="s2">&quot;regression&quot;</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">LRHistory</span><span class="p">(</span><span class="n">n_epochs</span><span class="o">=</span><span class="mi">10</span><span class="p">)])</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">277</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">277</span>
 <span class="normal">278</span>
 <span class="normal">279</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">LRHistory</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">n_epochs</span> <span class="o">=</span> <span class="n">n_epochs</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1694,10 +1668,10 @@ <h2 id="pytorch_widedeep.callbacks.LRHistory" class="doc doc-heading">
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -1706,11 +1680,11 @@ <h2 id="pytorch_widedeep.callbacks.LRHistory" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">ModelCheckpoint</span>
+            <span class="doc doc-object-name doc-class-name">ModelCheckpoint</span>
 
 
 <a href="#pytorch_widedeep.callbacks.ModelCheckpoint" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">ModelCheckpoint</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ModelCheckpoint</span><span class="p">(</span>
     <span class="n">filepath</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">monitor</span><span class="o">=</span><span class="s2">&quot;val_loss&quot;</span><span class="p">,</span>
     <span class="n">min_delta</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
@@ -1722,11 +1696,11 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.callbacks.Callback">Callback</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.callbacks.Callback">Callback</span></code></p>
+
 
-  
       <p>Saves the model after every epoch.</p>
 <p>This class is almost identical to the corresponding keras class.
 Therefore, <strong>credit</strong> to the Keras Team.</p>
@@ -1734,142 +1708,139 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
 <code>pytorch_widedeep.trainer.Trainer</code></p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>filepath</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>filepath</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Full path to save the output weights. It must contain only the root of
+          –
+          <div class="doc-md-description">
+            <p>Full path to save the output weights. It must contain only the root of
 the filenames. Epoch number and <code>.pt</code> extension (for pytorch) will be
 added. e.g. <code>filepath="path/to/output_weights/weights_out"</code> And the
 saved files in that directory will be named:
 <em>'weights_out_1.pt', 'weights_out_2.pt', ...</em>. If set to <code>None</code> the
 class just report best metric and best_epoch.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>monitor</code></b>
-            (<code>str</code>, default:
-                <code>&#39;val_loss&#39;</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>monitor</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;val_loss&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>quantity to monitor. Typically <em>'val_loss'</em> or metric name
+          –
+          <div class="doc-md-description">
+            <p>quantity to monitor. Typically <em>'val_loss'</em> or metric name
 (e.g. <em>'val_acc'</em>)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>min_delta</code></b>
-            (<code>float</code>, default:
-                <code>0.0</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>min_delta</code></b>
+              (<code>float</code>, default:
+                  <code>0.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>minimum change in the monitored quantity to qualify as an
+          –
+          <div class="doc-md-description">
+            <p>minimum change in the monitored quantity to qualify as an
 improvement, i.e. an absolute change of less than min_delta, will
 count as no improvement.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>0</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>verbosity mode</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>save_best_only</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          –
+          <div class="doc-md-description">
+            <p>verbosity mode</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>save_best_only</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>the latest best model according to the quantity monitored will not be
+          –
+          <div class="doc-md-description">
+            <p>the latest best model according to the quantity monitored will not be
 overwritten.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mode</code></b>
-            (<code>str</code>, default:
-                <code>&#39;auto&#39;</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mode</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;auto&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>If <code>save_best_only=True</code>, the decision to overwrite the current save
+          –
+          <div class="doc-md-description">
+            <p>If <code>save_best_only=True</code>, the decision to overwrite the current save
 file is made based on either the maximization or the minimization of
 the monitored quantity. For <em>'acc'</em>, this should be <em>'max'</em>, for
 <em>'loss'</em> this should be <em>'min'</em>, etc. In '<em>auto'</em> mode, the
 direction is automatically inferred from the name of the monitored
 quantity.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>period</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>period</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Interval (number of epochs) between checkpoints.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>max_save</code></b>
-            (<code>int</code>, default:
-                <code>-1</code>
+          –
+          <div class="doc-md-description">
+            <p>Interval (number of epochs) between checkpoints.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>max_save</code></b>
+              (<code>int</code>, default:
+                  <code>-1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Maximum number of outputs to save. If -1 will save all outputs</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>Maximum number of outputs to save. If -1 will save all outputs</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best">best</span></code></b>
-            (<code>float</code>)
-        –
-        <div class="doc-md-description">
-          <p>best metric</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best_epoch">best_epoch</span></code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>best epoch</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best_state_dict">best_state_dict</span></code></b>
-            (<code>dict</code>)
-        –
-        <div class="doc-md-description">
-          <p>best model state dictionary.<br/>
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best">best</span></code></b>
+              (<code>float</code>)
+          –
+          <div class="doc-md-description">
+            <p>best metric</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best_epoch">best_epoch</span></code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>best epoch</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best_state_dict">best_state_dict</span></code></b>
+              (<code>dict</code>)
+          –
+          <div class="doc-md-description">
+            <p>best model state dictionary.<br/>
 To restore model to its best state use <code>Trainer.model.load_state_dict
 (model_checkpoint.best_state_dict)</code> where <code>model_checkpoint</code> is an
 instance of the class <code>ModelCheckpoint</code>. See the Examples folder in
 the repo or the Examples section in this documentation for details</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.callbacks</span> <span class="kn">import</span> <span class="n">ModelCheckpoint</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabMlp</span><span class="p">,</span> <span class="n">Wide</span><span class="p">,</span> <span class="n">WideDeep</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.training</span> <span class="kn">import</span> <span class="n">Trainer</span>
@@ -1882,9 +1853,9 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">objective</span><span class="o">=</span><span class="s2">&quot;regression&quot;</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">ModelCheckpoint</span><span class="p">(</span><span class="n">filepath</span><span class="o">=</span><span class="s1">&#39;checkpoints/weights_out&#39;</span><span class="p">)])</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">446</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">446</span>
 <span class="normal">447</span>
 <span class="normal">448</span>
 <span class="normal">449</span>
@@ -2010,12 +1981,11 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
     <span class="k">else</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">*=</span> <span class="o">-</span><span class="mi">1</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2026,10 +1996,10 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -2038,11 +2008,11 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">EarlyStopping</span>
+            <span class="doc doc-object-name doc-class-name">EarlyStopping</span>
 
 
 <a href="#pytorch_widedeep.callbacks.EarlyStopping" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">EarlyStopping</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">EarlyStopping</span><span class="p">(</span>
     <span class="n">monitor</span><span class="o">=</span><span class="s2">&quot;val_loss&quot;</span><span class="p">,</span>
     <span class="n">min_delta</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
     <span class="n">patience</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
@@ -2053,11 +2023,11 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.callbacks.Callback">Callback</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.callbacks.Callback">Callback</span></code></p>
+
 
-  
       <p>Stop training when a monitored quantity has stopped improving.</p>
 <p>This class is almost identical to the corresponding keras class.
 Therefore, <strong>credit</strong> to the Keras Team.</p>
@@ -2065,117 +2035,114 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
 <code>pytorch_widedeep.trainer.Trainer</code></p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>monitor</code></b>
-            (<code>str</code>, default:
-                <code>&#39;val_loss&#39;</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>monitor</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;val_loss&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Quantity to monitor. Typically <em>'val_loss'</em> or metric name
+          –
+          <div class="doc-md-description">
+            <p>Quantity to monitor. Typically <em>'val_loss'</em> or metric name
 (e.g. <em>'val_acc'</em>)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>min_delta</code></b>
-            (<code>float</code>, default:
-                <code>0.0</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>min_delta</code></b>
+              (<code>float</code>, default:
+                  <code>0.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>minimum change in the monitored quantity to qualify as an
+          –
+          <div class="doc-md-description">
+            <p>minimum change in the monitored quantity to qualify as an
 improvement, i.e. an absolute change of less than min_delta, will
 count as no improvement.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>patience</code></b>
-            (<code>int</code>, default:
-                <code>10</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>patience</code></b>
+              (<code>int</code>, default:
+                  <code>10</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Number of epochs that produced the monitored quantity with no
+          –
+          <div class="doc-md-description">
+            <p>Number of epochs that produced the monitored quantity with no
 improvement after which training will be stopped.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>0</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>verbosity mode.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mode</code></b>
-            (<code>str</code>, default:
-                <code>&#39;auto&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>verbosity mode.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mode</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;auto&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>one of <em>{'auto', 'min', 'max'}</em>. In <em>'min'</em> mode, training will
+          –
+          <div class="doc-md-description">
+            <p>one of <em>{'auto', 'min', 'max'}</em>. In <em>'min'</em> mode, training will
 stop when the quantity monitored has stopped decreasing; in <em>'max'</em>
 mode it will stop when the quantity monitored has stopped increasing;
 in <em>'auto'</em> mode, the direction is automatically inferred from the
 name of the monitored quantity.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>baseline</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>baseline</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Baseline value for the monitored quantity to reach. Training will
+          –
+          <div class="doc-md-description">
+            <p>Baseline value for the monitored quantity to reach. Training will
 stop if the model does not show improvement over the baseline.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>restore_best_weights</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>restore_best_weights</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Whether to restore model weights from the epoch with the best
+          –
+          <div class="doc-md-description">
+            <p>Whether to restore model weights from the epoch with the best
 value of the monitored quantity. If <code>False</code>, the model weights
 obtained at the last step of training are used.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.callbacks.EarlyStopping.best">best</span></code></b>
-            (<code>float</code>)
-        –
-        <div class="doc-md-description">
-          <p>best metric</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.callbacks.EarlyStopping.stopped_epoch">stopped_epoch</span></code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>epoch when the training stopped</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.callbacks.EarlyStopping.best">best</span></code></b>
+              (<code>float</code>)
+          –
+          <div class="doc-md-description">
+            <p>best metric</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.callbacks.EarlyStopping.stopped_epoch">stopped_epoch</span></code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>epoch when the training stopped</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.callbacks</span> <span class="kn">import</span> <span class="n">EarlyStopping</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabMlp</span><span class="p">,</span> <span class="n">Wide</span><span class="p">,</span> <span class="n">WideDeep</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.training</span> <span class="kn">import</span> <span class="n">Trainer</span>
@@ -2188,9 +2155,9 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">objective</span><span class="o">=</span><span class="s2">&quot;regression&quot;</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">EarlyStopping</span><span class="p">(</span><span class="n">patience</span><span class="o">=</span><span class="mi">10</span><span class="p">)])</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">638</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">638</span>
 <span class="normal">639</span>
 <span class="normal">640</span>
 <span class="normal">641</span>
@@ -2282,12 +2249,11 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
     <span class="k">else</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">*=</span> <span class="o">-</span><span class="mi">1</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2298,10 +2264,10 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -2377,7 +2343,7 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -2395,7 +2361,7 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/dataloaders.html b/mkdocs/site/pytorch-widedeep/dataloaders.html
index f2d76eaa..97ae5a1c 100644
--- a/mkdocs/site/pytorch-widedeep/dataloaders.html
+++ b/mkdocs/site/pytorch-widedeep/dataloaders.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1027,8 +1027,6 @@
         
       
         
-      
-        
       
         
       
@@ -1366,11 +1364,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1387,11 +1385,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1408,11 +1406,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1429,11 +1427,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1450,11 +1448,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1471,11 +1469,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1492,11 +1490,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1513,32 +1511,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1606,63 +1583,61 @@ <h1 id="dataloaders">Dataloaders<a class="headerlink" href="#dataloaders" title=
 
 
 <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">DataLoaderImbalanced</span>
+            <span class="doc doc-object-name doc-class-name">DataLoaderImbalanced</span>
 
 
 <a href="#pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">DataLoaderImbalanced</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">DataLoaderImbalanced</span><span class="p">(</span>
     <span class="n">dataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">num_workers</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.utils.data.DataLoader">DataLoader</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.utils.data.DataLoader">DataLoader</span></code></p>
+
 
-  
       <p>Class to load and shuffle batches with adjusted weights for imbalanced
 datasets. If the classes do not begin from 0 remapping is necessary. See
 <a href="https://towardsdatascience.com/pytorch-tabular-multiclass-classification-9f8211a123ab">here</a>.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>dataset</code></b>
-            (<code><span title="pytorch_widedeep.training._wd_dataset.WideDeepDataset">WideDeepDataset</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>see <code>pytorch_widedeep.training._wd_dataset</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>batch_size</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>size of batch</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>num_workers</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>number of workers</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>dataset</code></b>
+              (<code><span title="pytorch_widedeep.training._wd_dataset.WideDeepDataset">WideDeepDataset</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>see <code>pytorch_widedeep.training._wd_dataset</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>batch_size</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>size of batch</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>num_workers</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>number of workers</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Other Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p>This can include any parameter that can be passed to the <em>'standard'</em>
+<p><span class="doc-section-title">Other Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p>This can include any parameter that can be passed to the <em>'standard'</em>
 pytorch
 <a href="https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader">DataLoader</a>
 and that is not already explicitely passed to the class. In addition,
@@ -1673,33 +1648,32 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
 <div class="arithmatex">\[
 minority \space class \space count \times number \space of \space classes \times oversample\_mul
 \]</div>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/dataloaders.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 82</span>
-<span class="normal"> 83</span>
-<span class="normal"> 84</span>
-<span class="normal"> 85</span>
-<span class="normal"> 86</span>
-<span class="normal"> 87</span>
-<span class="normal"> 88</span>
-<span class="normal"> 89</span>
-<span class="normal"> 90</span>
-<span class="normal"> 91</span>
-<span class="normal"> 92</span>
-<span class="normal"> 93</span>
-<span class="normal"> 94</span>
-<span class="normal"> 95</span>
-<span class="normal"> 96</span>
-<span class="normal"> 97</span>
-<span class="normal"> 98</span>
-<span class="normal"> 99</span>
-<span class="normal">100</span>
-<span class="normal">101</span>
-<span class="normal">102</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/dataloaders.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">72</span>
+<span class="normal">73</span>
+<span class="normal">74</span>
+<span class="normal">75</span>
+<span class="normal">76</span>
+<span class="normal">77</span>
+<span class="normal">78</span>
+<span class="normal">79</span>
+<span class="normal">80</span>
+<span class="normal">81</span>
+<span class="normal">82</span>
+<span class="normal">83</span>
+<span class="normal">84</span>
+<span class="normal">85</span>
+<span class="normal">86</span>
+<span class="normal">87</span>
+<span class="normal">88</span>
+<span class="normal">89</span>
+<span class="normal">90</span>
+<span class="normal">91</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span> <span class="n">dataset</span><span class="p">:</span> <span class="n">WideDeepDataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">num_workers</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
 <span class="p">):</span>
     <span class="k">assert</span> <span class="n">dataset</span><span class="o">.</span><span class="n">Y</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="p">(</span>
@@ -1707,7 +1681,6 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
         <span class="s2">&quot;target array &#39;Y&#39;&quot;</span>
     <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_lds</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">with_lds</span>
     <span class="k">if</span> <span class="s2">&quot;oversample_mul&quot;</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="p">:</span>
         <span class="n">oversample_mul</span> <span class="o">=</span> <span class="n">kwargs</span><span class="p">[</span><span class="s2">&quot;oversample_mul&quot;</span><span class="p">]</span>
         <span class="k">del</span> <span class="n">kwargs</span><span class="p">[</span><span class="s2">&quot;oversample_mul&quot;</span><span class="p">]</span>
@@ -1721,9 +1694,9 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
         <span class="n">dataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">num_workers</span><span class="o">=</span><span class="n">num_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -1739,8 +1712,7 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -1816,7 +1788,7 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -1834,7 +1806,7 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/losses.html b/mkdocs/site/pytorch-widedeep/losses.html
index bf88a090..60bebe62 100644
--- a/mkdocs/site/pytorch-widedeep/losses.html
+++ b/mkdocs/site/pytorch-widedeep/losses.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1198,8 +1198,6 @@
         
       
         
-      
-        
       
         
       
@@ -1537,32 +1535,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1579,11 +1556,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1600,11 +1577,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1621,11 +1598,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1642,11 +1619,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1663,11 +1640,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1684,11 +1661,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1705,11 +1682,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1789,35 +1766,31 @@ <h1 id="losses">Losses<a class="headerlink" href="#losses" title="Permanent link
 
 
 <h2 id="pytorch_widedeep.losses.MSELoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">MSELoss</span>
+            <span class="doc doc-object-name doc-class-name">MSELoss</span>
 
 
 <a href="#pytorch_widedeep.losses.MSELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">MSELoss</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">MSELoss</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
 
-  
-      <p>Mean square error loss with the option of using Label Smooth
-Distribution (LDS)</p>
-<p>LDS is based on
-<a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">25</span>
-<span class="normal">26</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+      <p>Mean square error loss</p>
+
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">23</span>
+<span class="normal">24</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1830,65 +1803,53 @@ <h2 id="pytorch_widedeep.losses.MSELoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.MSELoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.MSELoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-
+    <div class="doc doc-contents ">
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual values</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lds_weight</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Tensor of weights that will multiply the loss value.</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual values</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">MSELoss</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mf">1.2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">lds_weight</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">MSELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">MSELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">28</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">26</span>
+<span class="normal">27</span>
+<span class="normal">28</span>
 <span class="normal">29</span>
 <span class="normal">30</span>
 <span class="normal">31</span>
@@ -1909,13 +1870,10 @@ <h3 id="pytorch_widedeep.losses.MSELoss.forward" class="doc doc-heading">
 <span class="normal">46</span>
 <span class="normal">47</span>
 <span class="normal">48</span>
-<span class="normal">49</span>
-<span class="normal">50</span>
-<span class="normal">51</span>
-<span class="normal">52</span>
-<span class="normal">53</span>
-<span class="normal">54</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">lds_weight</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+<span class="normal">49</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+    <span class="bp">self</span><span class="p">,</span>
+    <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+    <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
@@ -1924,8 +1882,6 @@ <h3 id="pytorch_widedeep.losses.MSELoss.forward" class="doc doc-heading">
 <span class="sd">        Input tensor with predictions</span>
 <span class="sd">    target: Tensor</span>
 <span class="sd">        Target tensor with the actual values</span>
-<span class="sd">    lds_weight: Tensor, Optional</span>
-<span class="sd">        Tensor of weights that will multiply the loss value.</span>
 
 <span class="sd">    Examples</span>
 <span class="sd">    --------</span>
@@ -1934,16 +1890,13 @@ <h3 id="pytorch_widedeep.losses.MSELoss.forward" class="doc doc-heading">
 <span class="sd">    &gt;&gt;&gt;</span>
 <span class="sd">    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
 <span class="sd">    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
-<span class="sd">    &gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)</span>
-<span class="sd">    &gt;&gt;&gt; loss = MSELoss()(input, target, lds_weight)</span>
+<span class="sd">    &gt;&gt;&gt; loss = MSELoss()(input, target)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">loss</span> <span class="o">=</span> <span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span>
-    <span class="k">if</span> <span class="n">lds_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">loss</span> <span class="o">*=</span> <span class="n">lds_weight</span>
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -1951,8 +1904,7 @@ <h3 id="pytorch_widedeep.losses.MSELoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -1961,35 +1913,31 @@ <h3 id="pytorch_widedeep.losses.MSELoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.MSLELoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">MSLELoss</span>
+            <span class="doc doc-object-name doc-class-name">MSLELoss</span>
 
 
 <a href="#pytorch_widedeep.losses.MSLELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">MSLELoss</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">MSLELoss</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
 
-  
-      <p>Mean square log error loss with the option of using Label Smooth
-Distribution (LDS)</p>
-<p>LDS is based on
-<a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">65</span>
-<span class="normal">66</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+      <p>Mean square log error loss</p>
+
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">55</span>
+<span class="normal">56</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2002,99 +1950,84 @@ <h2 id="pytorch_widedeep.losses.MSLELoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.MSLELoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.MSLELoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-
+    <div class="doc doc-contents ">
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions (not probabilities)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual classes</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lds_weight</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Tensor of weights that will multiply the loss value.</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions (not probabilities)</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual classes</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">MSLELoss</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mf">1.2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">lds_weight</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">MSLELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">MSLELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 68</span>
-<span class="normal"> 69</span>
-<span class="normal"> 70</span>
-<span class="normal"> 71</span>
-<span class="normal"> 72</span>
-<span class="normal"> 73</span>
-<span class="normal"> 74</span>
-<span class="normal"> 75</span>
-<span class="normal"> 76</span>
-<span class="normal"> 77</span>
-<span class="normal"> 78</span>
-<span class="normal"> 79</span>
-<span class="normal"> 80</span>
-<span class="normal"> 81</span>
-<span class="normal"> 82</span>
-<span class="normal"> 83</span>
-<span class="normal"> 84</span>
-<span class="normal"> 85</span>
-<span class="normal"> 86</span>
-<span class="normal"> 87</span>
-<span class="normal"> 88</span>
-<span class="normal"> 89</span>
-<span class="normal"> 90</span>
-<span class="normal"> 91</span>
-<span class="normal"> 92</span>
-<span class="normal"> 93</span>
-<span class="normal"> 94</span>
-<span class="normal"> 95</span>
-<span class="normal"> 96</span>
-<span class="normal"> 97</span>
-<span class="normal"> 98</span>
-<span class="normal"> 99</span>
-<span class="normal">100</span>
-<span class="normal">101</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">lds_weight</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span>
+<span class="normal">62</span>
+<span class="normal">63</span>
+<span class="normal">64</span>
+<span class="normal">65</span>
+<span class="normal">66</span>
+<span class="normal">67</span>
+<span class="normal">68</span>
+<span class="normal">69</span>
+<span class="normal">70</span>
+<span class="normal">71</span>
+<span class="normal">72</span>
+<span class="normal">73</span>
+<span class="normal">74</span>
+<span class="normal">75</span>
+<span class="normal">76</span>
+<span class="normal">77</span>
+<span class="normal">78</span>
+<span class="normal">79</span>
+<span class="normal">80</span>
+<span class="normal">81</span>
+<span class="normal">82</span>
+<span class="normal">83</span>
+<span class="normal">84</span>
+<span class="normal">85</span>
+<span class="normal">86</span>
+<span class="normal">87</span>
+<span class="normal">88</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+    <span class="bp">self</span><span class="p">,</span>
+    <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+    <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
@@ -2103,8 +2036,6 @@ <h3 id="pytorch_widedeep.losses.MSLELoss.forward" class="doc doc-heading">
 <span class="sd">        Input tensor with predictions (not probabilities)</span>
 <span class="sd">    target: Tensor</span>
 <span class="sd">        Target tensor with the actual classes</span>
-<span class="sd">    lds_weight: Tensor, Optional</span>
-<span class="sd">        Tensor of weights that will multiply the loss value.</span>
 
 <span class="sd">    Examples</span>
 <span class="sd">    --------</span>
@@ -2113,8 +2044,7 @@ <h3 id="pytorch_widedeep.losses.MSLELoss.forward" class="doc doc-heading">
 <span class="sd">    &gt;&gt;&gt;</span>
 <span class="sd">    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
 <span class="sd">    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
-<span class="sd">    &gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)</span>
-<span class="sd">    &gt;&gt;&gt; loss = MSLELoss()(input, target, lds_weight)</span>
+<span class="sd">    &gt;&gt;&gt; loss = MSLELoss()(input, target)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">assert</span> <span class="p">(</span>
         <span class="nb">input</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="mi">0</span>
@@ -2124,12 +2054,10 @@ <h3 id="pytorch_widedeep.losses.MSLELoss.forward" class="doc doc-heading">
     <span class="k">assert</span> <span class="n">target</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">&quot;All target values must be &gt;=0&quot;</span>
 
     <span class="n">loss</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="nb">input</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="n">torch</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">target</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span> <span class="o">**</span> <span class="mi">2</span>
-    <span class="k">if</span> <span class="n">lds_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">loss</span> <span class="o">*=</span> <span class="n">lds_weight</span>
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2137,8 +2065,7 @@ <h3 id="pytorch_widedeep.losses.MSLELoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2147,35 +2074,31 @@ <h3 id="pytorch_widedeep.losses.MSLELoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.RMSELoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">RMSELoss</span>
+            <span class="doc doc-object-name doc-class-name">RMSELoss</span>
 
 
 <a href="#pytorch_widedeep.losses.RMSELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">RMSELoss</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">RMSELoss</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
 
-  
-      <p>Root mean square error loss adjusted for the possibility of using Label
-Smooth Distribution (LDS)</p>
-<p>LDS is based on
-<a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">112</span>
-<span class="normal">113</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+      <p>Root mean square error loss</p>
+
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">97</span>
+<span class="normal">98</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2188,93 +2111,70 @@ <h2 id="pytorch_widedeep.losses.RMSELoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.RMSELoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.RMSELoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-
+    <div class="doc doc-contents ">
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions (not probabilities)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual classes</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lds_weight</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Tensor of weights that will multiply the loss value.</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions (not probabilities)</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual classes</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">RMSELoss</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mf">1.2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">lds_weight</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">RMSELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">RMSELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">115</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
 <span class="normal">116</span>
 <span class="normal">117</span>
 <span class="normal">118</span>
-<span class="normal">119</span>
-<span class="normal">120</span>
-<span class="normal">121</span>
-<span class="normal">122</span>
-<span class="normal">123</span>
-<span class="normal">124</span>
-<span class="normal">125</span>
-<span class="normal">126</span>
-<span class="normal">127</span>
-<span class="normal">128</span>
-<span class="normal">129</span>
-<span class="normal">130</span>
-<span class="normal">131</span>
-<span class="normal">132</span>
-<span class="normal">133</span>
-<span class="normal">134</span>
-<span class="normal">135</span>
-<span class="normal">136</span>
-<span class="normal">137</span>
-<span class="normal">138</span>
-<span class="normal">139</span>
-<span class="normal">140</span>
-<span class="normal">141</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">lds_weight</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
-<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="normal">119</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -2282,8 +2182,6 @@ <h3 id="pytorch_widedeep.losses.RMSELoss.forward" class="doc doc-heading">
 <span class="sd">        Input tensor with predictions (not probabilities)</span>
 <span class="sd">    target: Tensor</span>
 <span class="sd">        Target tensor with the actual classes</span>
-<span class="sd">    lds_weight: Tensor, Optional</span>
-<span class="sd">        Tensor of weights that will multiply the loss value.</span>
 
 <span class="sd">    Examples</span>
 <span class="sd">    --------</span>
@@ -2292,16 +2190,13 @@ <h3 id="pytorch_widedeep.losses.RMSELoss.forward" class="doc doc-heading">
 <span class="sd">    &gt;&gt;&gt;</span>
 <span class="sd">    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
 <span class="sd">    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
-<span class="sd">    &gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)</span>
-<span class="sd">    &gt;&gt;&gt; loss = RMSELoss()(input, target, lds_weight)</span>
+<span class="sd">    &gt;&gt;&gt; loss = RMSELoss()(input, target)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">loss</span> <span class="o">=</span> <span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span>
-    <span class="k">if</span> <span class="n">lds_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">loss</span> <span class="o">*=</span> <span class="n">lds_weight</span>
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">))</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2309,8 +2204,7 @@ <h3 id="pytorch_widedeep.losses.RMSELoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2319,35 +2213,31 @@ <h3 id="pytorch_widedeep.losses.RMSELoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.RMSLELoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">RMSLELoss</span>
+            <span class="doc doc-object-name doc-class-name">RMSLELoss</span>
 
 
 <a href="#pytorch_widedeep.losses.RMSLELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">RMSLELoss</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">RMSLELoss</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
 
-  
-      <p>Root mean square log error loss adjusted for the possibility of using Label
-Smooth Distribution (LDS)</p>
-<p>LDS is based on
-<a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">152</span>
-<span class="normal">153</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+      <p>Root mean square log error loss</p>
+
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">125</span>
+<span class="normal">126</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2360,100 +2250,77 @@ <h2 id="pytorch_widedeep.losses.RMSLELoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.RMSLELoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.RMSLELoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-
+    <div class="doc doc-contents ">
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions (not probabilities)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual classes</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lds_weight</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Tensor of weights that will multiply the loss value.</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions (not probabilities)</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual classes</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">RMSLELoss</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mf">1.2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">lds_weight</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">RMSLELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">RMSLELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">155</span>
-<span class="normal">156</span>
-<span class="normal">157</span>
-<span class="normal">158</span>
-<span class="normal">159</span>
-<span class="normal">160</span>
-<span class="normal">161</span>
-<span class="normal">162</span>
-<span class="normal">163</span>
-<span class="normal">164</span>
-<span class="normal">165</span>
-<span class="normal">166</span>
-<span class="normal">167</span>
-<span class="normal">168</span>
-<span class="normal">169</span>
-<span class="normal">170</span>
-<span class="normal">171</span>
-<span class="normal">172</span>
-<span class="normal">173</span>
-<span class="normal">174</span>
-<span class="normal">175</span>
-<span class="normal">176</span>
-<span class="normal">177</span>
-<span class="normal">178</span>
-<span class="normal">179</span>
-<span class="normal">180</span>
-<span class="normal">181</span>
-<span class="normal">182</span>
-<span class="normal">183</span>
-<span class="normal">184</span>
-<span class="normal">185</span>
-<span class="normal">186</span>
-<span class="normal">187</span>
-<span class="normal">188</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">lds_weight</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
-<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -2461,8 +2328,6 @@ <h3 id="pytorch_widedeep.losses.RMSLELoss.forward" class="doc doc-heading">
 <span class="sd">        Input tensor with predictions (not probabilities)</span>
 <span class="sd">    target: Tensor</span>
 <span class="sd">        Target tensor with the actual classes</span>
-<span class="sd">    lds_weight: Tensor, Optional</span>
-<span class="sd">        Tensor of weights that will multiply the loss value.</span>
 
 <span class="sd">    Examples</span>
 <span class="sd">    --------</span>
@@ -2471,8 +2336,7 @@ <h3 id="pytorch_widedeep.losses.RMSLELoss.forward" class="doc doc-heading">
 <span class="sd">    &gt;&gt;&gt;</span>
 <span class="sd">    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
 <span class="sd">    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
-<span class="sd">    &gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)</span>
-<span class="sd">    &gt;&gt;&gt; loss = RMSLELoss()(input, target, lds_weight)</span>
+<span class="sd">    &gt;&gt;&gt; loss = RMSLELoss()(input, target)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">assert</span> <span class="p">(</span>
         <span class="nb">input</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="mi">0</span>
@@ -2482,12 +2346,10 @@ <h3 id="pytorch_widedeep.losses.RMSLELoss.forward" class="doc doc-heading">
     <span class="k">assert</span> <span class="n">target</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">&quot;All target values must be &gt;=0&quot;</span>
 
     <span class="n">loss</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="nb">input</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="n">torch</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">target</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span> <span class="o">**</span> <span class="mi">2</span>
-    <span class="k">if</span> <span class="n">lds_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">loss</span> <span class="o">*=</span> <span class="n">lds_weight</span>
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">))</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2495,8 +2357,7 @@ <h3 id="pytorch_widedeep.losses.RMSLELoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2505,20 +2366,20 @@ <h3 id="pytorch_widedeep.losses.RMSLELoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.QuantileLoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">QuantileLoss</span>
+            <span class="doc doc-object-name doc-class-name">QuantileLoss</span>
 
 
 <a href="#pytorch_widedeep.losses.QuantileLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">QuantileLoss</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">QuantileLoss</span><span class="p">(</span>
     <span class="n">quantiles</span><span class="o">=</span><span class="p">[</span><span class="mf">0.02</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.25</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.98</span><span class="p">]</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Quantile loss defined as:</p>
 <div class="arithmatex">\[
 Loss = max(q \times (y-y_{pred}), (1-q) \times (y_{pred}-y))
@@ -2527,41 +2388,39 @@ <h2 id="pytorch_widedeep.losses.QuantileLoss" class="doc doc-heading">
 <a href="https://pytorch-forecasting.readthedocs.io/en/latest/_modules/pytorch_forecasting/metrics.html#QuantileLoss">pytorch-forecasting</a>.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>quantiles</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[float]</code>, default:
-                <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>quantiles</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[float]</code>, default:
+                  <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of quantiles</p>
-        </div>
-      </li>
-  </ul>
+          –
+          <div class="doc-md-description">
+            <p>List of quantiles</p>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">207</span>
-<span class="normal">208</span>
-<span class="normal">209</span>
-<span class="normal">210</span>
-<span class="normal">211</span>
-<span class="normal">212</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">quantiles</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mf">0.02</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.25</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.98</span><span class="p">],</span>
 <span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">quantiles</span> <span class="o">=</span> <span class="n">quantiles</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2574,43 +2433,40 @@ <h2 id="pytorch_widedeep.losses.QuantileLoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.QuantileLoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.QuantileLoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-
+    <div class="doc doc-contents ">
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual values</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual values</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">QuantileLoss</span>
@@ -2622,44 +2478,44 @@ <h3 id="pytorch_widedeep.losses.QuantileLoss.forward" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">qloss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">214</span>
-<span class="normal">215</span>
-<span class="normal">216</span>
-<span class="normal">217</span>
-<span class="normal">218</span>
-<span class="normal">219</span>
-<span class="normal">220</span>
-<span class="normal">221</span>
-<span class="normal">222</span>
-<span class="normal">223</span>
-<span class="normal">224</span>
-<span class="normal">225</span>
-<span class="normal">226</span>
-<span class="normal">227</span>
-<span class="normal">228</span>
-<span class="normal">229</span>
-<span class="normal">230</span>
-<span class="normal">231</span>
-<span class="normal">232</span>
-<span class="normal">233</span>
-<span class="normal">234</span>
-<span class="normal">235</span>
-<span class="normal">236</span>
-<span class="normal">237</span>
-<span class="normal">238</span>
-<span class="normal">239</span>
-<span class="normal">240</span>
-<span class="normal">241</span>
-<span class="normal">242</span>
-<span class="normal">243</span>
-<span class="normal">244</span>
-<span class="normal">245</span>
-<span class="normal">246</span>
-<span class="normal">247</span>
-<span class="normal">248</span>
-<span class="normal">249</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -2696,8 +2552,8 @@ <h3 id="pytorch_widedeep.losses.QuantileLoss.forward" class="doc doc-heading">
 
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2705,8 +2561,7 @@ <h3 id="pytorch_widedeep.losses.QuantileLoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2715,18 +2570,18 @@ <h3 id="pytorch_widedeep.losses.QuantileLoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.FocalLoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">FocalLoss</span>
+            <span class="doc doc-object-name doc-class-name">FocalLoss</span>
 
 
 <a href="#pytorch_widedeep.losses.FocalLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">FocalLoss</span><span class="p">(</span><span class="n">alpha</span><span class="o">=</span><span class="mf">0.25</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.0</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FocalLoss</span><span class="p">(</span><span class="n">alpha</span><span class="o">=</span><span class="mf">0.25</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.0</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Implementation of the <a href="https://arxiv.org/pdf/1708.02002.pdf">Focal loss</a>
 for both binary and multiclass classification:</p>
 <div class="arithmatex">\[
@@ -2738,47 +2593,45 @@ <h2 id="pytorch_widedeep.losses.FocalLoss" class="doc doc-heading">
 \]</div>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>alpha</code></b>
-            (<code>float</code>, default:
-                <code>0.25</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>alpha</code></b>
+              (<code>float</code>, default:
+                  <code>0.25</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Focal Loss <code>alpha</code> parameter</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>gamma</code></b>
-            (<code>float</code>, default:
-                <code>1.0</code>
+          –
+          <div class="doc-md-description">
+            <p>Focal Loss <code>alpha</code> parameter</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>gamma</code></b>
+              (<code>float</code>, default:
+                  <code>1.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Focal Loss <code>gamma</code> parameter</p>
-        </div>
-      </li>
-  </ul>
-
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">274</span>
-<span class="normal">275</span>
-<span class="normal">276</span>
-<span class="normal">277</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">alpha</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.25</span><span class="p">,</span> <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">):</span>
+          –
+          <div class="doc-md-description">
+            <p>Focal Loss <code>gamma</code> parameter</p>
+          </div>
+        </li>
+    </ul>
+
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">alpha</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.25</span><span class="p">,</span> <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">alpha</span> <span class="o">=</span> <span class="n">alpha</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2791,43 +2644,40 @@ <h2 id="pytorch_widedeep.losses.FocalLoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.FocalLoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.FocalLoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-
+    <div class="doc doc-contents ">
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions (not probabilities)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual classes</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions (not probabilities)</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual classes</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">FocalLoss</span>
@@ -2843,48 +2693,48 @@ <h3 id="pytorch_widedeep.losses.FocalLoss.forward" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">FocalLoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">284</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
 <span class="normal">285</span>
 <span class="normal">286</span>
 <span class="normal">287</span>
 <span class="normal">288</span>
-<span class="normal">289</span>
-<span class="normal">290</span>
-<span class="normal">291</span>
-<span class="normal">292</span>
-<span class="normal">293</span>
-<span class="normal">294</span>
-<span class="normal">295</span>
-<span class="normal">296</span>
-<span class="normal">297</span>
-<span class="normal">298</span>
-<span class="normal">299</span>
-<span class="normal">300</span>
-<span class="normal">301</span>
-<span class="normal">302</span>
-<span class="normal">303</span>
-<span class="normal">304</span>
-<span class="normal">305</span>
-<span class="normal">306</span>
-<span class="normal">307</span>
-<span class="normal">308</span>
-<span class="normal">309</span>
-<span class="normal">310</span>
-<span class="normal">311</span>
-<span class="normal">312</span>
-<span class="normal">313</span>
-<span class="normal">314</span>
-<span class="normal">315</span>
-<span class="normal">316</span>
-<span class="normal">317</span>
-<span class="normal">318</span>
-<span class="normal">319</span>
-<span class="normal">320</span>
-<span class="normal">321</span>
-<span class="normal">322</span>
-<span class="normal">323</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="normal">289</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -2925,8 +2775,8 @@ <h3 id="pytorch_widedeep.losses.FocalLoss.forward" class="doc doc-heading">
         <span class="n">input_prob</span><span class="p">,</span> <span class="n">binary_target</span><span class="p">,</span> <span class="n">weight</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;mean&quot;</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2934,8 +2784,7 @@ <h3 id="pytorch_widedeep.losses.FocalLoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2944,34 +2793,33 @@ <h3 id="pytorch_widedeep.losses.FocalLoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.BayesianSELoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">BayesianSELoss</span>
+            <span class="doc doc-object-name doc-class-name">BayesianSELoss</span>
 
 
 <a href="#pytorch_widedeep.losses.BayesianSELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">BayesianSELoss</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">BayesianSELoss</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Squared Loss (log Gaussian) for the case of a regression as specified in
 the original publication
 <a href="https://arxiv.org/abs/1505.05424">Weight Uncertainty in Neural Networks</a>.</p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">352</span>
-<span class="normal">353</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">318</span>
+<span class="normal">319</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2984,43 +2832,40 @@ <h2 id="pytorch_widedeep.losses.BayesianSELoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.BayesianSELoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.BayesianSELoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-
+    <div class="doc doc-contents ">
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions (not probabilities)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual classes</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions (not probabilities)</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual classes</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">BayesianSELoss</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mf">1.2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
@@ -3028,26 +2873,26 @@ <h3 id="pytorch_widedeep.losses.BayesianSELoss.forward" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">BayesianSELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">355</span>
-<span class="normal">356</span>
-<span class="normal">357</span>
-<span class="normal">358</span>
-<span class="normal">359</span>
-<span class="normal">360</span>
-<span class="normal">361</span>
-<span class="normal">362</span>
-<span class="normal">363</span>
-<span class="normal">364</span>
-<span class="normal">365</span>
-<span class="normal">366</span>
-<span class="normal">367</span>
-<span class="normal">368</span>
-<span class="normal">369</span>
-<span class="normal">370</span>
-<span class="normal">371</span>
-<span class="normal">372</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -3066,8 +2911,8 @@ <h3 id="pytorch_widedeep.losses.BayesianSELoss.forward" class="doc doc-heading">
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="p">(</span><span class="mf">0.5</span> <span class="o">*</span> <span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -3075,8 +2920,7 @@ <h3 id="pytorch_widedeep.losses.BayesianSELoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -3085,35 +2929,34 @@ <h3 id="pytorch_widedeep.losses.BayesianSELoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.TweedieLoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TweedieLoss</span>
+            <span class="doc doc-object-name doc-class-name">TweedieLoss</span>
 
 
 <a href="#pytorch_widedeep.losses.TweedieLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TweedieLoss</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TweedieLoss</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Tweedie loss for extremely unbalanced zero-inflated data</p>
 <p>All credits go to Wenbo Shi. See
 <a href="https://towardsdatascience.com/tweedie-loss-function-for-right-skewed-data-2c5ca470678f">this post</a>
 and the <a href="https://arxiv.org/abs/1811.10192">original publication</a> for details.</p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">384</span>
-<span class="normal">385</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">350</span>
+<span class="normal">351</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -3126,125 +2969,101 @@ <h2 id="pytorch_widedeep.losses.TweedieLoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.TweedieLoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">p</span><span class="o">=</span><span class="mf">1.5</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">p</span><span class="o">=</span><span class="mf">1.5</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
 
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual values</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lds_weight</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If we choose to use LDS this is the tensor of weights that will
-multiply the loss value.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>p</code></b>
-            (<code>float</code>, default:
-                <code>1.5</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual values</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>p</code></b>
+              (<code>float</code>, default:
+                  <code>1.5</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>the power to be used to compute the loss. See the original
+          –
+          <div class="doc-md-description">
+            <p>the power to be used to compute the loss. See the original
 publication for details</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">TweedieLoss</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mf">1.2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">lds_weight</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">])</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">TweedieLoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">TweedieLoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">387</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
 <span class="normal">388</span>
-<span class="normal">389</span>
-<span class="normal">390</span>
-<span class="normal">391</span>
-<span class="normal">392</span>
-<span class="normal">393</span>
-<span class="normal">394</span>
-<span class="normal">395</span>
-<span class="normal">396</span>
-<span class="normal">397</span>
-<span class="normal">398</span>
-<span class="normal">399</span>
-<span class="normal">400</span>
-<span class="normal">401</span>
-<span class="normal">402</span>
-<span class="normal">403</span>
-<span class="normal">404</span>
-<span class="normal">405</span>
-<span class="normal">406</span>
-<span class="normal">407</span>
-<span class="normal">408</span>
-<span class="normal">409</span>
-<span class="normal">410</span>
-<span class="normal">411</span>
-<span class="normal">412</span>
-<span class="normal">413</span>
-<span class="normal">414</span>
-<span class="normal">415</span>
-<span class="normal">416</span>
-<span class="normal">417</span>
-<span class="normal">418</span>
-<span class="normal">419</span>
-<span class="normal">420</span>
-<span class="normal">421</span>
-<span class="normal">422</span>
-<span class="normal">423</span>
-<span class="normal">424</span>
-<span class="normal">425</span>
-<span class="normal">426</span>
-<span class="normal">427</span>
-<span class="normal">428</span>
-<span class="normal">429</span>
-<span class="normal">430</span>
-<span class="normal">431</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+<span class="normal">389</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
-    <span class="n">lds_weight</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">p</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.5</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
@@ -3254,9 +3073,6 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
 <span class="sd">        Input tensor with predictions</span>
 <span class="sd">    target: Tensor</span>
 <span class="sd">        Target tensor with the actual values</span>
-<span class="sd">    lds_weight: Tensor, Optional</span>
-<span class="sd">        If we choose to use LDS this is the tensor of weights that will</span>
-<span class="sd">        multiply the loss value.</span>
 <span class="sd">    p: float, default = 1.5</span>
 <span class="sd">        the power to be used to compute the loss. See the original</span>
 <span class="sd">        publication for details</span>
@@ -3268,8 +3084,7 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
 <span class="sd">    &gt;&gt;&gt;</span>
 <span class="sd">    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
 <span class="sd">    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
-<span class="sd">    &gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)</span>
-<span class="sd">    &gt;&gt;&gt; loss = TweedieLoss()(input, target, lds_weight)</span>
+<span class="sd">    &gt;&gt;&gt; loss = TweedieLoss()(input, target)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="k">assert</span> <span class="p">(</span>
@@ -3281,13 +3096,10 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
     <span class="n">loss</span> <span class="o">=</span> <span class="o">-</span><span class="n">target</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">p</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">p</span><span class="p">)</span> <span class="o">+</span> <span class="n">torch</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="mi">2</span> <span class="o">-</span> <span class="n">p</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span>
         <span class="mi">2</span> <span class="o">-</span> <span class="n">p</span>
     <span class="p">)</span>
-    <span class="k">if</span> <span class="n">lds_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">loss</span> <span class="o">*=</span> <span class="n">lds_weight</span>
-
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -3295,8 +3107,7 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -3305,35 +3116,34 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.ZILNLoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">ZILNLoss</span>
+            <span class="doc doc-object-name doc-class-name">ZILNLoss</span>
 
 
 <a href="#pytorch_widedeep.losses.ZILNLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">ZILNLoss</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ZILNLoss</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Adjusted implementation of the Zero Inflated LogNormal Loss</p>
 <p>See <a href="https://arxiv.org/pdf/1912.07753.pdf">A Deep Probabilistic Model for Customer Lifetime Value Prediction</a>
 and the corresponding
 <a href="https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py">code</a>.</p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">442</span>
-<span class="normal">443</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">400</span>
+<span class="normal">401</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -3346,43 +3156,40 @@ <h2 id="pytorch_widedeep.losses.ZILNLoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.ZILNLoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-
+    <div class="doc doc-contents ">
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions with spape (N,3), where N is the batch size</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual target values</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions with spape (N,3), where N is the batch size</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual target values</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">ZILNLoss</span>
 <span class="gp">&gt;&gt;&gt;</span>
@@ -3391,9 +3198,51 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">ZILNLoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">445</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span>
+<span class="normal">434</span>
+<span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
 <span class="normal">446</span>
 <span class="normal">447</span>
 <span class="normal">448</span>
@@ -3402,49 +3251,7 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
 <span class="normal">451</span>
 <span class="normal">452</span>
 <span class="normal">453</span>
-<span class="normal">454</span>
-<span class="normal">455</span>
-<span class="normal">456</span>
-<span class="normal">457</span>
-<span class="normal">458</span>
-<span class="normal">459</span>
-<span class="normal">460</span>
-<span class="normal">461</span>
-<span class="normal">462</span>
-<span class="normal">463</span>
-<span class="normal">464</span>
-<span class="normal">465</span>
-<span class="normal">466</span>
-<span class="normal">467</span>
-<span class="normal">468</span>
-<span class="normal">469</span>
-<span class="normal">470</span>
-<span class="normal">471</span>
-<span class="normal">472</span>
-<span class="normal">473</span>
-<span class="normal">474</span>
-<span class="normal">475</span>
-<span class="normal">476</span>
-<span class="normal">477</span>
-<span class="normal">478</span>
-<span class="normal">479</span>
-<span class="normal">480</span>
-<span class="normal">481</span>
-<span class="normal">482</span>
-<span class="normal">483</span>
-<span class="normal">484</span>
-<span class="normal">485</span>
-<span class="normal">486</span>
-<span class="normal">487</span>
-<span class="normal">488</span>
-<span class="normal">489</span>
-<span class="normal">490</span>
-<span class="normal">491</span>
-<span class="normal">492</span>
-<span class="normal">493</span>
-<span class="normal">494</span>
-<span class="normal">495</span>
-<span class="normal">496</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="normal">454</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -3497,8 +3304,8 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
 
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">classification_loss</span> <span class="o">+</span> <span class="n">regression_loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -3506,8 +3313,7 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -3516,35 +3322,31 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.L1Loss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">L1Loss</span>
+            <span class="doc doc-object-name doc-class-name">L1Loss</span>
 
 
 <a href="#pytorch_widedeep.losses.L1Loss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">L1Loss</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">L1Loss</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
 
-  
-      <p>L1 loss adjusted for the possibility of using Label Smooth
-Distribution (LDS)</p>
-<p>LDS is based on
-<a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">507</span>
-<span class="normal">508</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+      <p>L1 loss</p>
+
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">463</span>
+<span class="normal">464</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -3557,54 +3359,40 @@ <h2 id="pytorch_widedeep.losses.L1Loss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.L1Loss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.L1Loss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-
+    <div class="doc doc-contents ">
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual values</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lds_weight</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If we choose to use LDS this is the tensor of weights that will
-multiply the loss value.</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual values</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">L1Loss</span>
@@ -3614,38 +3402,29 @@ <h3 id="pytorch_widedeep.losses.L1Loss.forward" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">L1Loss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">510</span>
-<span class="normal">511</span>
-<span class="normal">512</span>
-<span class="normal">513</span>
-<span class="normal">514</span>
-<span class="normal">515</span>
-<span class="normal">516</span>
-<span class="normal">517</span>
-<span class="normal">518</span>
-<span class="normal">519</span>
-<span class="normal">520</span>
-<span class="normal">521</span>
-<span class="normal">522</span>
-<span class="normal">523</span>
-<span class="normal">524</span>
-<span class="normal">525</span>
-<span class="normal">526</span>
-<span class="normal">527</span>
-<span class="normal">528</span>
-<span class="normal">529</span>
-<span class="normal">530</span>
-<span class="normal">531</span>
-<span class="normal">532</span>
-<span class="normal">533</span>
-<span class="normal">534</span>
-<span class="normal">535</span>
-<span class="normal">536</span>
-<span class="normal">537</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">lds_weight</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
-<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span>
+<span class="normal">475</span>
+<span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -3653,9 +3432,6 @@ <h3 id="pytorch_widedeep.losses.L1Loss.forward" class="doc doc-heading">
 <span class="sd">        Input tensor with predictions</span>
 <span class="sd">    target: Tensor</span>
 <span class="sd">        Target tensor with the actual values</span>
-<span class="sd">    lds_weight: Tensor, Optional</span>
-<span class="sd">        If we choose to use LDS this is the tensor of weights that will</span>
-<span class="sd">        multiply the loss value.</span>
 
 <span class="sd">    Examples</span>
 <span class="sd">    --------</span>
@@ -3668,12 +3444,10 @@ <h3 id="pytorch_widedeep.losses.L1Loss.forward" class="doc doc-heading">
 <span class="sd">    &gt;&gt;&gt; loss = L1Loss()(input, target)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">l1_loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;none&quot;</span><span class="p">)</span>
-    <span class="k">if</span> <span class="n">lds_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">loss</span> <span class="o">*=</span> <span class="n">lds_weight</span>
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -3681,8 +3455,7 @@ <h3 id="pytorch_widedeep.losses.L1Loss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -3691,71 +3464,70 @@ <h3 id="pytorch_widedeep.losses.L1Loss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">FocalR_L1Loss</span>
+            <span class="doc doc-object-name doc-class-name">FocalR_L1Loss</span>
 
 
 <a href="#pytorch_widedeep.losses.FocalR_L1Loss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">FocalR_L1Loss</span><span class="p">(</span><span class="n">beta</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">activation_fn</span><span class="o">=</span><span class="s1">&#39;sigmoid&#39;</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FocalR_L1Loss</span><span class="p">(</span><span class="n">beta</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">activation_fn</span><span class="o">=</span><span class="s1">&#39;sigmoid&#39;</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Focal-R L1 loss</p>
 <p>Based on <a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>beta</code></b>
-            (<code>float</code>, default:
-                <code>0.2</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>beta</code></b>
+              (<code>float</code>, default:
+                  <code>0.2</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Focal Loss <code>beta</code> parameter in their implementation</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>gamma</code></b>
-            (<code>float</code>, default:
-                <code>1.0</code>
+          –
+          <div class="doc-md-description">
+            <p>Focal Loss <code>beta</code> parameter in their implementation</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>gamma</code></b>
+              (<code>float</code>, default:
+                  <code>1.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Focal Loss <code>gamma</code> parameter</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>activation_fn</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>, default:
-                <code>&#39;sigmoid&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>Focal Loss <code>gamma</code> parameter</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>activation_fn</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>, default:
+                  <code>&#39;sigmoid&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Activation function to be used during the computation of the loss.
+          –
+          <div class="doc-md-description">
+            <p>Activation function to be used during the computation of the loss.
 Possible values are <em>'sigmoid'</em> and <em>'tanh'</em>. See the original
 publication for details.</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">557</span>
-<span class="normal">558</span>
-<span class="normal">559</span>
-<span class="normal">560</span>
-<span class="normal">561</span>
-<span class="normal">562</span>
-<span class="normal">563</span>
-<span class="normal">564</span>
-<span class="normal">565</span>
-<span class="normal">566</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span>
+<span class="normal">514</span>
+<span class="normal">515</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
     <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
@@ -3766,12 +3538,11 @@ <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">=</span> <span class="n">activation_fn</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -3784,54 +3555,40 @@ <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.FocalR_L1Loss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
 
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions (not probabilities)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual classes</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lds_weight</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If we choose to use LDS this is the tensor of weights that will
-multiply the loss value.</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions (not probabilities)</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual classes</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">FocalR_L1Loss</span>
@@ -3841,53 +3598,46 @@ <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">FocalR_L1Loss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">568</span>
-<span class="normal">569</span>
-<span class="normal">570</span>
-<span class="normal">571</span>
-<span class="normal">572</span>
-<span class="normal">573</span>
-<span class="normal">574</span>
-<span class="normal">575</span>
-<span class="normal">576</span>
-<span class="normal">577</span>
-<span class="normal">578</span>
-<span class="normal">579</span>
-<span class="normal">580</span>
-<span class="normal">581</span>
-<span class="normal">582</span>
-<span class="normal">583</span>
-<span class="normal">584</span>
-<span class="normal">585</span>
-<span class="normal">586</span>
-<span class="normal">587</span>
-<span class="normal">588</span>
-<span class="normal">589</span>
-<span class="normal">590</span>
-<span class="normal">591</span>
-<span class="normal">592</span>
-<span class="normal">593</span>
-<span class="normal">594</span>
-<span class="normal">595</span>
-<span class="normal">596</span>
-<span class="normal">597</span>
-<span class="normal">598</span>
-<span class="normal">599</span>
-<span class="normal">600</span>
-<span class="normal">601</span>
-<span class="normal">602</span>
-<span class="normal">603</span>
-<span class="normal">604</span>
-<span class="normal">605</span>
-<span class="normal">606</span>
-<span class="normal">607</span>
-<span class="normal">608</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">517</span>
+<span class="normal">518</span>
+<span class="normal">519</span>
+<span class="normal">520</span>
+<span class="normal">521</span>
+<span class="normal">522</span>
+<span class="normal">523</span>
+<span class="normal">524</span>
+<span class="normal">525</span>
+<span class="normal">526</span>
+<span class="normal">527</span>
+<span class="normal">528</span>
+<span class="normal">529</span>
+<span class="normal">530</span>
+<span class="normal">531</span>
+<span class="normal">532</span>
+<span class="normal">533</span>
+<span class="normal">534</span>
+<span class="normal">535</span>
+<span class="normal">536</span>
+<span class="normal">537</span>
+<span class="normal">538</span>
+<span class="normal">539</span>
+<span class="normal">540</span>
+<span class="normal">541</span>
+<span class="normal">542</span>
+<span class="normal">543</span>
+<span class="normal">544</span>
+<span class="normal">545</span>
+<span class="normal">546</span>
+<span class="normal">547</span>
+<span class="normal">548</span>
+<span class="normal">549</span>
+<span class="normal">550</span>
+<span class="normal">551</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
-    <span class="n">lds_weight</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
@@ -3896,9 +3646,6 @@ <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
 <span class="sd">        Input tensor with predictions (not probabilities)</span>
 <span class="sd">    target: Tensor</span>
 <span class="sd">        Target tensor with the actual classes</span>
-<span class="sd">    lds_weight: Tensor, Optional</span>
-<span class="sd">        If we choose to use LDS this is the tensor of weights that will</span>
-<span class="sd">        multiply the loss value.</span>
 
 <span class="sd">    Examples</span>
 <span class="sd">    --------</span>
@@ -3921,12 +3668,10 @@ <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
         <span class="ne">ValueError</span><span class="p">(</span>
             <span class="s2">&quot;Incorrect activation function value - must be in [&#39;sigmoid&#39;, &#39;tanh&#39;]&quot;</span>
         <span class="p">)</span>
-    <span class="k">if</span> <span class="n">lds_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">loss</span> <span class="o">*=</span> <span class="n">lds_weight</span>
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -3934,8 +3679,7 @@ <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -3944,73 +3688,72 @@ <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">FocalR_MSELoss</span>
+            <span class="doc doc-object-name doc-class-name">FocalR_MSELoss</span>
 
 
 <a href="#pytorch_widedeep.losses.FocalR_MSELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">FocalR_MSELoss</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FocalR_MSELoss</span><span class="p">(</span>
     <span class="n">beta</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">activation_fn</span><span class="o">=</span><span class="s2">&quot;sigmoid&quot;</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Focal-R MSE loss</p>
 <p>Based on <a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>beta</code></b>
-            (<code>float</code>, default:
-                <code>0.2</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>beta</code></b>
+              (<code>float</code>, default:
+                  <code>0.2</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Focal Loss <code>beta</code> parameter in their implementation</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>gamma</code></b>
-            (<code>float</code>, default:
-                <code>1.0</code>
+          –
+          <div class="doc-md-description">
+            <p>Focal Loss <code>beta</code> parameter in their implementation</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>gamma</code></b>
+              (<code>float</code>, default:
+                  <code>1.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Focal Loss <code>gamma</code> parameter</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>activation_fn</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>, default:
-                <code>&#39;sigmoid&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>Focal Loss <code>gamma</code> parameter</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>activation_fn</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>, default:
+                  <code>&#39;sigmoid&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Activation function to be used during the computation of the loss.
+          –
+          <div class="doc-md-description">
+            <p>Activation function to be used during the computation of the loss.
 Possible values are <em>'sigmoid'</em> and <em>'tanh'</em>. See the original
 publication for details.</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">628</span>
-<span class="normal">629</span>
-<span class="normal">630</span>
-<span class="normal">631</span>
-<span class="normal">632</span>
-<span class="normal">633</span>
-<span class="normal">634</span>
-<span class="normal">635</span>
-<span class="normal">636</span>
-<span class="normal">637</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">571</span>
+<span class="normal">572</span>
+<span class="normal">573</span>
+<span class="normal">574</span>
+<span class="normal">575</span>
+<span class="normal">576</span>
+<span class="normal">577</span>
+<span class="normal">578</span>
+<span class="normal">579</span>
+<span class="normal">580</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
     <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
@@ -4021,12 +3764,11 @@ <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">=</span> <span class="n">activation_fn</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -4039,54 +3781,40 @@ <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.FocalR_MSELoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
 
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions (not probabilities)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual classes</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lds_weight</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If we choose to use LDS this is the tensor of weights that will
-multiply the loss value.</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions (not probabilities)</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual classes</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">FocalR_MSELoss</span>
@@ -4096,53 +3824,46 @@ <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">FocalR_MSELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">639</span>
-<span class="normal">640</span>
-<span class="normal">641</span>
-<span class="normal">642</span>
-<span class="normal">643</span>
-<span class="normal">644</span>
-<span class="normal">645</span>
-<span class="normal">646</span>
-<span class="normal">647</span>
-<span class="normal">648</span>
-<span class="normal">649</span>
-<span class="normal">650</span>
-<span class="normal">651</span>
-<span class="normal">652</span>
-<span class="normal">653</span>
-<span class="normal">654</span>
-<span class="normal">655</span>
-<span class="normal">656</span>
-<span class="normal">657</span>
-<span class="normal">658</span>
-<span class="normal">659</span>
-<span class="normal">660</span>
-<span class="normal">661</span>
-<span class="normal">662</span>
-<span class="normal">663</span>
-<span class="normal">664</span>
-<span class="normal">665</span>
-<span class="normal">666</span>
-<span class="normal">667</span>
-<span class="normal">668</span>
-<span class="normal">669</span>
-<span class="normal">670</span>
-<span class="normal">671</span>
-<span class="normal">672</span>
-<span class="normal">673</span>
-<span class="normal">674</span>
-<span class="normal">675</span>
-<span class="normal">676</span>
-<span class="normal">677</span>
-<span class="normal">678</span>
-<span class="normal">679</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">582</span>
+<span class="normal">583</span>
+<span class="normal">584</span>
+<span class="normal">585</span>
+<span class="normal">586</span>
+<span class="normal">587</span>
+<span class="normal">588</span>
+<span class="normal">589</span>
+<span class="normal">590</span>
+<span class="normal">591</span>
+<span class="normal">592</span>
+<span class="normal">593</span>
+<span class="normal">594</span>
+<span class="normal">595</span>
+<span class="normal">596</span>
+<span class="normal">597</span>
+<span class="normal">598</span>
+<span class="normal">599</span>
+<span class="normal">600</span>
+<span class="normal">601</span>
+<span class="normal">602</span>
+<span class="normal">603</span>
+<span class="normal">604</span>
+<span class="normal">605</span>
+<span class="normal">606</span>
+<span class="normal">607</span>
+<span class="normal">608</span>
+<span class="normal">609</span>
+<span class="normal">610</span>
+<span class="normal">611</span>
+<span class="normal">612</span>
+<span class="normal">613</span>
+<span class="normal">614</span>
+<span class="normal">615</span>
+<span class="normal">616</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
-    <span class="n">lds_weight</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
@@ -4151,9 +3872,6 @@ <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
 <span class="sd">        Input tensor with predictions (not probabilities)</span>
 <span class="sd">    target: Tensor</span>
 <span class="sd">        Target tensor with the actual classes</span>
-<span class="sd">    lds_weight: Tensor, Optional</span>
-<span class="sd">        If we choose to use LDS this is the tensor of weights that will</span>
-<span class="sd">        multiply the loss value.</span>
 
 <span class="sd">    Examples</span>
 <span class="sd">    --------</span>
@@ -4176,12 +3894,10 @@ <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
         <span class="ne">ValueError</span><span class="p">(</span>
             <span class="s2">&quot;Incorrect activation function value - must be in [&#39;sigmoid&#39;, &#39;tanh&#39;]&quot;</span>
         <span class="p">)</span>
-    <span class="k">if</span> <span class="n">lds_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">loss</span> <span class="o">*=</span> <span class="n">lds_weight</span>
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -4189,8 +3905,7 @@ <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -4199,73 +3914,72 @@ <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">FocalR_RMSELoss</span>
+            <span class="doc doc-object-name doc-class-name">FocalR_RMSELoss</span>
 
 
 <a href="#pytorch_widedeep.losses.FocalR_RMSELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">FocalR_RMSELoss</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FocalR_RMSELoss</span><span class="p">(</span>
     <span class="n">beta</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">activation_fn</span><span class="o">=</span><span class="s2">&quot;sigmoid&quot;</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Focal-R RMSE loss</p>
 <p>Based on <a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>beta</code></b>
-            (<code>float</code>, default:
-                <code>0.2</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>beta</code></b>
+              (<code>float</code>, default:
+                  <code>0.2</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Focal Loss <code>beta</code> parameter in their implementation</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>gamma</code></b>
-            (<code>float</code>, default:
-                <code>1.0</code>
+          –
+          <div class="doc-md-description">
+            <p>Focal Loss <code>beta</code> parameter in their implementation</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>gamma</code></b>
+              (<code>float</code>, default:
+                  <code>1.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Focal Loss <code>gamma</code> parameter</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>activation_fn</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>, default:
-                <code>&#39;sigmoid&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>Focal Loss <code>gamma</code> parameter</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>activation_fn</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>, default:
+                  <code>&#39;sigmoid&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Activation function to be used during the computation of the loss.
+          –
+          <div class="doc-md-description">
+            <p>Activation function to be used during the computation of the loss.
 Possible values are <em>'sigmoid'</em> and <em>'tanh'</em>. See the original
 publication for details.</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">699</span>
-<span class="normal">700</span>
-<span class="normal">701</span>
-<span class="normal">702</span>
-<span class="normal">703</span>
-<span class="normal">704</span>
-<span class="normal">705</span>
-<span class="normal">706</span>
-<span class="normal">707</span>
-<span class="normal">708</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">636</span>
+<span class="normal">637</span>
+<span class="normal">638</span>
+<span class="normal">639</span>
+<span class="normal">640</span>
+<span class="normal">641</span>
+<span class="normal">642</span>
+<span class="normal">643</span>
+<span class="normal">644</span>
+<span class="normal">645</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
     <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
@@ -4276,12 +3990,11 @@ <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">=</span> <span class="n">activation_fn</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -4294,54 +4007,40 @@ <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
 
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions (not probabilities)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual classes</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lds_weight</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If we choose to use LDS this is the tensor of weights that will
-multiply the loss value.</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions (not probabilities)</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual classes</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">FocalR_RMSELoss</span>
@@ -4351,53 +4050,46 @@ <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">FocalR_RMSELoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">710</span>
-<span class="normal">711</span>
-<span class="normal">712</span>
-<span class="normal">713</span>
-<span class="normal">714</span>
-<span class="normal">715</span>
-<span class="normal">716</span>
-<span class="normal">717</span>
-<span class="normal">718</span>
-<span class="normal">719</span>
-<span class="normal">720</span>
-<span class="normal">721</span>
-<span class="normal">722</span>
-<span class="normal">723</span>
-<span class="normal">724</span>
-<span class="normal">725</span>
-<span class="normal">726</span>
-<span class="normal">727</span>
-<span class="normal">728</span>
-<span class="normal">729</span>
-<span class="normal">730</span>
-<span class="normal">731</span>
-<span class="normal">732</span>
-<span class="normal">733</span>
-<span class="normal">734</span>
-<span class="normal">735</span>
-<span class="normal">736</span>
-<span class="normal">737</span>
-<span class="normal">738</span>
-<span class="normal">739</span>
-<span class="normal">740</span>
-<span class="normal">741</span>
-<span class="normal">742</span>
-<span class="normal">743</span>
-<span class="normal">744</span>
-<span class="normal">745</span>
-<span class="normal">746</span>
-<span class="normal">747</span>
-<span class="normal">748</span>
-<span class="normal">749</span>
-<span class="normal">750</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">647</span>
+<span class="normal">648</span>
+<span class="normal">649</span>
+<span class="normal">650</span>
+<span class="normal">651</span>
+<span class="normal">652</span>
+<span class="normal">653</span>
+<span class="normal">654</span>
+<span class="normal">655</span>
+<span class="normal">656</span>
+<span class="normal">657</span>
+<span class="normal">658</span>
+<span class="normal">659</span>
+<span class="normal">660</span>
+<span class="normal">661</span>
+<span class="normal">662</span>
+<span class="normal">663</span>
+<span class="normal">664</span>
+<span class="normal">665</span>
+<span class="normal">666</span>
+<span class="normal">667</span>
+<span class="normal">668</span>
+<span class="normal">669</span>
+<span class="normal">670</span>
+<span class="normal">671</span>
+<span class="normal">672</span>
+<span class="normal">673</span>
+<span class="normal">674</span>
+<span class="normal">675</span>
+<span class="normal">676</span>
+<span class="normal">677</span>
+<span class="normal">678</span>
+<span class="normal">679</span>
+<span class="normal">680</span>
+<span class="normal">681</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
-    <span class="n">lds_weight</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
@@ -4406,9 +4098,6 @@ <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading"
 <span class="sd">        Input tensor with predictions (not probabilities)</span>
 <span class="sd">    target: Tensor</span>
 <span class="sd">        Target tensor with the actual classes</span>
-<span class="sd">    lds_weight: Tensor, Optional</span>
-<span class="sd">        If we choose to use LDS this is the tensor of weights that will</span>
-<span class="sd">        multiply the loss value.</span>
 
 <span class="sd">    Examples</span>
 <span class="sd">    --------</span>
@@ -4431,12 +4120,10 @@ <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading"
         <span class="ne">ValueError</span><span class="p">(</span>
             <span class="s2">&quot;Incorrect activation function value - must be in [&#39;sigmoid&#39;, &#39;tanh&#39;]&quot;</span>
         <span class="p">)</span>
-    <span class="k">if</span> <span class="n">lds_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">loss</span> <span class="o">*=</span> <span class="n">lds_weight</span>
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">))</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -4444,8 +4131,7 @@ <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading"
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -4454,35 +4140,34 @@ <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading"
 
 
 <h2 id="pytorch_widedeep.losses.HuberLoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">HuberLoss</span>
+            <span class="doc doc-object-name doc-class-name">HuberLoss</span>
 
 
 <a href="#pytorch_widedeep.losses.HuberLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">HuberLoss</span><span class="p">(</span><span class="n">beta</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">HuberLoss</span><span class="p">(</span><span class="n">beta</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Hubbler Loss</p>
 <p>Based on <a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">759</span>
-<span class="normal">760</span>
-<span class="normal">761</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">):</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">690</span>
+<span class="normal">691</span>
+<span class="normal">692</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -4495,54 +4180,40 @@ <h2 id="pytorch_widedeep.losses.HuberLoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.HuberLoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">lds_weight</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
 
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input tensor with predictions (not probabilities)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Target tensor with the actual classes</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lds_weight</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If we choose to use LDS this is the tensor of weights that will
-multiply the loss value.</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input tensor with predictions (not probabilities)</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Target tensor with the actual classes</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">HuberLoss</span>
@@ -4552,47 +4223,40 @@ <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">HuberLoss</span><span class="p">()(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">763</span>
-<span class="normal">764</span>
-<span class="normal">765</span>
-<span class="normal">766</span>
-<span class="normal">767</span>
-<span class="normal">768</span>
-<span class="normal">769</span>
-<span class="normal">770</span>
-<span class="normal">771</span>
-<span class="normal">772</span>
-<span class="normal">773</span>
-<span class="normal">774</span>
-<span class="normal">775</span>
-<span class="normal">776</span>
-<span class="normal">777</span>
-<span class="normal">778</span>
-<span class="normal">779</span>
-<span class="normal">780</span>
-<span class="normal">781</span>
-<span class="normal">782</span>
-<span class="normal">783</span>
-<span class="normal">784</span>
-<span class="normal">785</span>
-<span class="normal">786</span>
-<span class="normal">787</span>
-<span class="normal">788</span>
-<span class="normal">789</span>
-<span class="normal">790</span>
-<span class="normal">791</span>
-<span class="normal">792</span>
-<span class="normal">793</span>
-<span class="normal">794</span>
-<span class="normal">795</span>
-<span class="normal">796</span>
-<span class="normal">797</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">694</span>
+<span class="normal">695</span>
+<span class="normal">696</span>
+<span class="normal">697</span>
+<span class="normal">698</span>
+<span class="normal">699</span>
+<span class="normal">700</span>
+<span class="normal">701</span>
+<span class="normal">702</span>
+<span class="normal">703</span>
+<span class="normal">704</span>
+<span class="normal">705</span>
+<span class="normal">706</span>
+<span class="normal">707</span>
+<span class="normal">708</span>
+<span class="normal">709</span>
+<span class="normal">710</span>
+<span class="normal">711</span>
+<span class="normal">712</span>
+<span class="normal">713</span>
+<span class="normal">714</span>
+<span class="normal">715</span>
+<span class="normal">716</span>
+<span class="normal">717</span>
+<span class="normal">718</span>
+<span class="normal">719</span>
+<span class="normal">720</span>
+<span class="normal">721</span>
+<span class="normal">722</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
-    <span class="n">lds_weight</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
@@ -4601,9 +4265,6 @@ <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
 <span class="sd">        Input tensor with predictions (not probabilities)</span>
 <span class="sd">    target: Tensor</span>
 <span class="sd">        Target tensor with the actual classes</span>
-<span class="sd">    lds_weight: Tensor, Optional</span>
-<span class="sd">        If we choose to use LDS this is the tensor of weights that will</span>
-<span class="sd">        multiply the loss value.</span>
 
 <span class="sd">    Examples</span>
 <span class="sd">    --------</span>
@@ -4620,12 +4281,10 @@ <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
     <span class="n">loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">where</span><span class="p">(</span>
         <span class="n">cond</span><span class="p">,</span> <span class="mf">0.5</span> <span class="o">*</span> <span class="n">l1_loss</span><span class="o">**</span><span class="mi">2</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">,</span> <span class="n">l1_loss</span> <span class="o">-</span> <span class="mf">0.5</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span>
     <span class="p">)</span>
-    <span class="k">if</span> <span class="n">lds_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">loss</span> <span class="o">*=</span> <span class="n">lds_weight</span>
     <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -4633,8 +4292,7 @@ <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -4643,18 +4301,18 @@ <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.InfoNCELoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">InfoNCELoss</span>
+            <span class="doc doc-object-name doc-class-name">InfoNCELoss</span>
 
 
 <a href="#pytorch_widedeep.losses.InfoNCELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">InfoNCELoss</span><span class="p">(</span><span class="n">temperature</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s1">&#39;mean&#39;</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">InfoNCELoss</span><span class="p">(</span><span class="n">temperature</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s1">&#39;mean&#39;</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>InfoNCE Loss. Loss applied during the Contrastive Denoising Self
 Supervised Pre-training routine available in this library</p>
 <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: This loss is in principle not exposed to
@@ -4666,49 +4324,47 @@ <h2 id="pytorch_widedeep.losses.InfoNCELoss" class="doc doc-heading">
 <p>Partially inspired by the code in this <a href="https://github.com/RElbers/info-nce-pytorch">repo</a></p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>temperature</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>temperature</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The logits are divided by the temperature before computing the loss value</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>reduction</code></b>
-            (<code>str</code>, default:
-                <code>&#39;mean&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>The logits are divided by the temperature before computing the loss value</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>reduction</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;mean&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Loss reduction method</p>
-        </div>
-      </li>
-  </ul>
+          –
+          <div class="doc-md-description">
+            <p>Loss reduction method</p>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">822</span>
-<span class="normal">823</span>
-<span class="normal">824</span>
-<span class="normal">825</span>
-<span class="normal">826</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">temperature</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">reduction</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">):</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">747</span>
+<span class="normal">748</span>
+<span class="normal">749</span>
+<span class="normal">750</span>
+<span class="normal">751</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">temperature</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">reduction</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">InfoNCELoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
     <span class="bp">self</span><span class="o">.</span><span class="n">temperature</span> <span class="o">=</span> <span class="n">temperature</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">=</span> <span class="n">reduction</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -4721,37 +4377,34 @@ <h2 id="pytorch_widedeep.losses.InfoNCELoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.InfoNCELoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.InfoNCELoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="n">g_projs</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="n">g_projs</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
 
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>g_projs</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Tuple with the two tensors corresponding to the output of the two
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>g_projs</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Tuple with the two tensors corresponding to the output of the two
 projection heads, as described 'SAINT: Improved Neural Networks
 for Tabular Data via Row Attention and Contrastive Pre-Training'.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">InfoNCELoss</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">g_projs</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">16</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">16</span><span class="p">))</span>
@@ -4759,40 +4412,40 @@ <h3 id="pytorch_widedeep.losses.InfoNCELoss.forward" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">res</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="n">g_projs</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">828</span>
-<span class="normal">829</span>
-<span class="normal">830</span>
-<span class="normal">831</span>
-<span class="normal">832</span>
-<span class="normal">833</span>
-<span class="normal">834</span>
-<span class="normal">835</span>
-<span class="normal">836</span>
-<span class="normal">837</span>
-<span class="normal">838</span>
-<span class="normal">839</span>
-<span class="normal">840</span>
-<span class="normal">841</span>
-<span class="normal">842</span>
-<span class="normal">843</span>
-<span class="normal">844</span>
-<span class="normal">845</span>
-<span class="normal">846</span>
-<span class="normal">847</span>
-<span class="normal">848</span>
-<span class="normal">849</span>
-<span class="normal">850</span>
-<span class="normal">851</span>
-<span class="normal">852</span>
-<span class="normal">853</span>
-<span class="normal">854</span>
-<span class="normal">855</span>
-<span class="normal">856</span>
-<span class="normal">857</span>
-<span class="normal">858</span>
-<span class="normal">859</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">g_projs</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">753</span>
+<span class="normal">754</span>
+<span class="normal">755</span>
+<span class="normal">756</span>
+<span class="normal">757</span>
+<span class="normal">758</span>
+<span class="normal">759</span>
+<span class="normal">760</span>
+<span class="normal">761</span>
+<span class="normal">762</span>
+<span class="normal">763</span>
+<span class="normal">764</span>
+<span class="normal">765</span>
+<span class="normal">766</span>
+<span class="normal">767</span>
+<span class="normal">768</span>
+<span class="normal">769</span>
+<span class="normal">770</span>
+<span class="normal">771</span>
+<span class="normal">772</span>
+<span class="normal">773</span>
+<span class="normal">774</span>
+<span class="normal">775</span>
+<span class="normal">776</span>
+<span class="normal">777</span>
+<span class="normal">778</span>
+<span class="normal">779</span>
+<span class="normal">780</span>
+<span class="normal">781</span>
+<span class="normal">782</span>
+<span class="normal">783</span>
+<span class="normal">784</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">g_projs</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -4825,8 +4478,8 @@ <h3 id="pytorch_widedeep.losses.InfoNCELoss.forward" class="doc doc-heading">
 
     <span class="k">return</span> <span class="p">(</span><span class="n">loss</span> <span class="o">+</span> <span class="n">loss_</span><span class="p">)</span> <span class="o">/</span> <span class="mf">2.0</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -4834,8 +4487,7 @@ <h3 id="pytorch_widedeep.losses.InfoNCELoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -4844,20 +4496,20 @@ <h3 id="pytorch_widedeep.losses.InfoNCELoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">DenoisingLoss</span>
+            <span class="doc doc-object-name doc-class-name">DenoisingLoss</span>
 
 
 <a href="#pytorch_widedeep.losses.DenoisingLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">DenoisingLoss</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">DenoisingLoss</span><span class="p">(</span>
     <span class="n">lambda_cat</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">lambda_cont</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;mean&quot;</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Denoising Loss. Loss applied during the Contrastive Denoising Self
 Supervised Pre-training routine available in this library</p>
 <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: This loss is in principle not exposed to
@@ -4868,53 +4520,52 @@ <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
 references therein</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>lambda_cat</code></b>
-            (<code>float</code>, default:
-                <code>1.0</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>lambda_cat</code></b>
+              (<code>float</code>, default:
+                  <code>1.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Multiplicative factor that will be applied to loss associated to the
+          –
+          <div class="doc-md-description">
+            <p>Multiplicative factor that will be applied to loss associated to the
 categorical features</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lambda_cont</code></b>
-            (<code>float</code>, default:
-                <code>1.0</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>lambda_cont</code></b>
+              (<code>float</code>, default:
+                  <code>1.0</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Multiplicative factor that will be applied to loss associated to the
+          –
+          <div class="doc-md-description">
+            <p>Multiplicative factor that will be applied to loss associated to the
 continuous features</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>reduction</code></b>
-            (<code>str</code>, default:
-                <code>&#39;mean&#39;</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>reduction</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;mean&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Loss reduction method</p>
-        </div>
-      </li>
-  </ul>
+          –
+          <div class="doc-md-description">
+            <p>Loss reduction method</p>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">886</span>
-<span class="normal">887</span>
-<span class="normal">888</span>
-<span class="normal">889</span>
-<span class="normal">890</span>
-<span class="normal">891</span>
-<span class="normal">892</span>
-<span class="normal">893</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">811</span>
+<span class="normal">812</span>
+<span class="normal">813</span>
+<span class="normal">814</span>
+<span class="normal">815</span>
+<span class="normal">816</span>
+<span class="normal">817</span>
+<span class="normal">818</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span> <span class="n">lambda_cat</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span> <span class="n">lambda_cont</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span> <span class="n">reduction</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span>
 <span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">DenoisingLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
@@ -4923,12 +4574,11 @@ <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">lambda_cont</span> <span class="o">=</span> <span class="n">lambda_cont</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">=</span> <span class="n">reduction</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -4941,47 +4591,44 @@ <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.DenoisingLoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.DenoisingLoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="n">x_cat_and_cat_</span><span class="p">,</span> <span class="n">x_cont_and_cont_</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="n">x_cat_and_cat_</span><span class="p">,</span> <span class="n">x_cont_and_cont_</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
 
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>x_cat_and_cat_</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]]]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Tuple of tensors containing the raw input features and their
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>x_cat_and_cat_</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]]]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Tuple of tensors containing the raw input features and their
 encodings, referred in the SAINT paper as <span class="arithmatex">\(x\)</span> and <span class="arithmatex">\(x''\)</span>
 respectively. If one denoising MLP is used per categorical
 feature <code>x_cat_and_cat_</code> will be a list of tuples, one per
 categorical feature</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>x_cont_and_cont_</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]]]</code>)
-        –
-        <div class="doc-md-description">
-          <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>x_cont_and_cont_</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]]]</code>)
+          –
+          <div class="doc-md-description">
+            <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">DenoisingLoss</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">x_cat_and_cat_</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">long</span><span class="p">(),</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
@@ -4990,51 +4637,51 @@ <h3 id="pytorch_widedeep.losses.DenoisingLoss.forward" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">res</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="n">x_cat_and_cat_</span><span class="p">,</span> <span class="n">x_cont_and_cont_</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">895</span>
-<span class="normal">896</span>
-<span class="normal">897</span>
-<span class="normal">898</span>
-<span class="normal">899</span>
-<span class="normal">900</span>
-<span class="normal">901</span>
-<span class="normal">902</span>
-<span class="normal">903</span>
-<span class="normal">904</span>
-<span class="normal">905</span>
-<span class="normal">906</span>
-<span class="normal">907</span>
-<span class="normal">908</span>
-<span class="normal">909</span>
-<span class="normal">910</span>
-<span class="normal">911</span>
-<span class="normal">912</span>
-<span class="normal">913</span>
-<span class="normal">914</span>
-<span class="normal">915</span>
-<span class="normal">916</span>
-<span class="normal">917</span>
-<span class="normal">918</span>
-<span class="normal">919</span>
-<span class="normal">920</span>
-<span class="normal">921</span>
-<span class="normal">922</span>
-<span class="normal">923</span>
-<span class="normal">924</span>
-<span class="normal">925</span>
-<span class="normal">926</span>
-<span class="normal">927</span>
-<span class="normal">928</span>
-<span class="normal">929</span>
-<span class="normal">930</span>
-<span class="normal">931</span>
-<span class="normal">932</span>
-<span class="normal">933</span>
-<span class="normal">934</span>
-<span class="normal">935</span>
-<span class="normal">936</span>
-<span class="normal">937</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">820</span>
+<span class="normal">821</span>
+<span class="normal">822</span>
+<span class="normal">823</span>
+<span class="normal">824</span>
+<span class="normal">825</span>
+<span class="normal">826</span>
+<span class="normal">827</span>
+<span class="normal">828</span>
+<span class="normal">829</span>
+<span class="normal">830</span>
+<span class="normal">831</span>
+<span class="normal">832</span>
+<span class="normal">833</span>
+<span class="normal">834</span>
+<span class="normal">835</span>
+<span class="normal">836</span>
+<span class="normal">837</span>
+<span class="normal">838</span>
+<span class="normal">839</span>
+<span class="normal">840</span>
+<span class="normal">841</span>
+<span class="normal">842</span>
+<span class="normal">843</span>
+<span class="normal">844</span>
+<span class="normal">845</span>
+<span class="normal">846</span>
+<span class="normal">847</span>
+<span class="normal">848</span>
+<span class="normal">849</span>
+<span class="normal">850</span>
+<span class="normal">851</span>
+<span class="normal">852</span>
+<span class="normal">853</span>
+<span class="normal">854</span>
+<span class="normal">855</span>
+<span class="normal">856</span>
+<span class="normal">857</span>
+<span class="normal">858</span>
+<span class="normal">859</span>
+<span class="normal">860</span>
+<span class="normal">861</span>
+<span class="normal">862</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">x_cat_and_cat_</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
         <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]],</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]</span>
@@ -5078,8 +4725,8 @@ <h3 id="pytorch_widedeep.losses.DenoisingLoss.forward" class="doc doc-heading">
 
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">lambda_cat</span> <span class="o">*</span> <span class="n">loss_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">lambda_cont</span> <span class="o">*</span> <span class="n">loss_cont</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -5087,8 +4734,7 @@ <h3 id="pytorch_widedeep.losses.DenoisingLoss.forward" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -5097,18 +4743,18 @@ <h3 id="pytorch_widedeep.losses.DenoisingLoss.forward" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.losses.EncoderDecoderLoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">EncoderDecoderLoss</span>
+            <span class="doc doc-object-name doc-class-name">EncoderDecoderLoss</span>
 
 
 <a href="#pytorch_widedeep.losses.EncoderDecoderLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">EncoderDecoderLoss</span><span class="p">(</span><span class="n">eps</span><span class="o">=</span><span class="mf">1e-09</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">EncoderDecoderLoss</span><span class="p">(</span><span class="n">eps</span><span class="o">=</span><span class="mf">1e-09</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>'<em>Standard</em>' Encoder Decoder Loss. Loss applied during the Endoder-Decoder
  Self-Supervised Pre-Training routine available in this library</p>
 <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: This loss is in principle not exposed to
@@ -5120,35 +4766,33 @@ <h2 id="pytorch_widedeep.losses.EncoderDecoderLoss" class="doc doc-heading">
 Interpretable Tabular Learning</a>.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>eps</code></b>
-            (<code>float</code>, default:
-                <code>1e-09</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>eps</code></b>
+              (<code>float</code>, default:
+                  <code>1e-09</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Simply a small number to avoid dividing by zero</p>
-        </div>
-      </li>
-  </ul>
+          –
+          <div class="doc-md-description">
+            <p>Simply a small number to avoid dividing by zero</p>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">993</span>
-<span class="normal">994</span>
-<span class="normal">995</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">eps</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1e-9</span><span class="p">):</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">918</span>
+<span class="normal">919</span>
+<span class="normal">920</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">eps</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1e-9</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">EncoderDecoderLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="n">eps</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -5161,52 +4805,49 @@ <h2 id="pytorch_widedeep.losses.EncoderDecoderLoss" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="n">x_true</span><span class="p">,</span> <span class="n">x_pred</span><span class="p">,</span> <span class="n">mask</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="n">x_true</span><span class="p">,</span> <span class="n">x_pred</span><span class="p">,</span> <span class="n">mask</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
 
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>x_true</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Embeddings of the input data</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>x_pred</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Reconstructed embeddings</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mask</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Mask with 1s indicated that the reconstruction, and therefore the
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>x_true</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Embeddings of the input data</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>x_pred</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Reconstructed embeddings</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mask</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Mask with 1s indicated that the reconstruction, and therefore the
 loss, is based on those features.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses</span> <span class="kn">import</span> <span class="n">EncoderDecoderLoss</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">x_true</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
@@ -5216,48 +4857,48 @@ <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-headi
 <span class="gp">&gt;&gt;&gt; </span><span class="n">res</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="n">x_true</span><span class="p">,</span> <span class="n">x_pred</span><span class="p">,</span> <span class="n">mask</span><span class="p">)</span>
 </code></pre></div>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 997</span>
-<span class="normal"> 998</span>
-<span class="normal"> 999</span>
-<span class="normal">1000</span>
-<span class="normal">1001</span>
-<span class="normal">1002</span>
-<span class="normal">1003</span>
-<span class="normal">1004</span>
-<span class="normal">1005</span>
-<span class="normal">1006</span>
-<span class="normal">1007</span>
-<span class="normal">1008</span>
-<span class="normal">1009</span>
-<span class="normal">1010</span>
-<span class="normal">1011</span>
-<span class="normal">1012</span>
-<span class="normal">1013</span>
-<span class="normal">1014</span>
-<span class="normal">1015</span>
-<span class="normal">1016</span>
-<span class="normal">1017</span>
-<span class="normal">1018</span>
-<span class="normal">1019</span>
-<span class="normal">1020</span>
-<span class="normal">1021</span>
-<span class="normal">1022</span>
-<span class="normal">1023</span>
-<span class="normal">1024</span>
-<span class="normal">1025</span>
-<span class="normal">1026</span>
-<span class="normal">1027</span>
-<span class="normal">1028</span>
-<span class="normal">1029</span>
-<span class="normal">1030</span>
-<span class="normal">1031</span>
-<span class="normal">1032</span>
-<span class="normal">1033</span>
-<span class="normal">1034</span>
-<span class="normal">1035</span>
-<span class="normal">1036</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">x_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">mask</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">922</span>
+<span class="normal">923</span>
+<span class="normal">924</span>
+<span class="normal">925</span>
+<span class="normal">926</span>
+<span class="normal">927</span>
+<span class="normal">928</span>
+<span class="normal">929</span>
+<span class="normal">930</span>
+<span class="normal">931</span>
+<span class="normal">932</span>
+<span class="normal">933</span>
+<span class="normal">934</span>
+<span class="normal">935</span>
+<span class="normal">936</span>
+<span class="normal">937</span>
+<span class="normal">938</span>
+<span class="normal">939</span>
+<span class="normal">940</span>
+<span class="normal">941</span>
+<span class="normal">942</span>
+<span class="normal">943</span>
+<span class="normal">944</span>
+<span class="normal">945</span>
+<span class="normal">946</span>
+<span class="normal">947</span>
+<span class="normal">948</span>
+<span class="normal">949</span>
+<span class="normal">950</span>
+<span class="normal">951</span>
+<span class="normal">952</span>
+<span class="normal">953</span>
+<span class="normal">954</span>
+<span class="normal">955</span>
+<span class="normal">956</span>
+<span class="normal">957</span>
+<span class="normal">958</span>
+<span class="normal">959</span>
+<span class="normal">960</span>
+<span class="normal">961</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">x_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">mask</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -5298,8 +4939,8 @@ <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-headi
 
     <span class="k">return</span> <span class="n">loss</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -5307,8 +4948,7 @@ <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-headi
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -5317,56 +4957,54 @@ <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-headi
 
 
 <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">MultiTargetRegressionLoss</span>
+            <span class="doc doc-object-name doc-class-name">MultiTargetRegressionLoss</span>
 
 
 <a href="#pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">MultiTargetRegressionLoss</span><span class="p">(</span><span class="n">weights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s1">&#39;mean&#39;</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">MultiTargetRegressionLoss</span><span class="p">(</span><span class="n">weights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s1">&#39;mean&#39;</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>This class is a wrapper around the Pytorch MSELoss. It allows for multi-target
 regression problems. The user can provide a list of weights to apply to each
 target. The loss can be either the sum or the mean of the individual losses</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>weights</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>weights</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of weights to apply to the loss associated to each target. The
+          –
+          <div class="doc-md-description">
+            <p>List of weights to apply to the loss associated to each target. The
 length of the list must match the number of targets.
 Alias: 'target_weights'</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>reduction</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>, default:
-                <code>&#39;mean&#39;</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>reduction</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>, default:
+                  <code>&#39;mean&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Specifies the reduction to apply to the loss associated to each
+          –
+          <div class="doc-md-description">
+            <p>Specifies the reduction to apply to the loss associated to each
 target: 'mean' | 'sum'. Note that this is NOT the same as the
 reduction in the MSELoss. This reduction is applied after the loss
 for each target has been computed. Alias: 'target_reduction'</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses_multitarget</span> <span class="kn">import</span> <span class="n">MultiTargetRegressionLoss</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
@@ -5375,9 +5013,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
 <span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">46</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">46</span>
 <span class="normal">47</span>
 <span class="normal">48</span>
 <span class="normal">49</span>
@@ -5405,12 +5043,11 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
     <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]:</span>
         <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;reduction must be either &#39;mean&#39; or &#39;sum&#39;&quot;</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -5421,10 +5058,10 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -5433,11 +5070,11 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
 
 
 <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">MultiTargetClassificationLoss</span>
+            <span class="doc doc-object-name doc-class-name">MultiTargetClassificationLoss</span>
 
 
 <a href="#pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">MultiTargetClassificationLoss</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">MultiTargetClassificationLoss</span><span class="p">(</span>
     <span class="n">binary_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">multiclass_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">weights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -5446,80 +5083,79 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and
 cross_entropy losses. It allows for multi-target classification problems. The
 user can provide a list of weights to apply to each target. The loss can be
 either the sum or the mean of the individual losses</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>binary_config</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, float]]]]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>binary_config</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, float]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of integers with the index of the target for binary
+          –
+          <div class="doc-md-description">
+            <p>List of integers with the index of the target for binary
 classification or tuples with two elements: the index of the targets
 or binary classification and the positive weight for binary
 classification</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>multiclass_config</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>multiclass_config</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of tuples with two or three elements: the index of the target and the
+          –
+          <div class="doc-md-description">
+            <p>List of tuples with two or three elements: the index of the target and the
 number of classes for multiclass classification, or a tuple with the index of
 the target, the number of classes and a list of weights to apply to each class
 (i.e. the 'weight' parameter in the cross_entropy loss)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>weights</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>weights</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of weights to apply to the loss associated to each target. The
+          –
+          <div class="doc-md-description">
+            <p>List of weights to apply to the loss associated to each target. The
 length of the list must match the number of targets.
 Alias: 'target_weights'</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>reduction</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>, default:
-                <code>&#39;mean&#39;</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>reduction</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>, default:
+                  <code>&#39;mean&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Specifies the reduction to apply to the loss associated to each
+          –
+          <div class="doc-md-description">
+            <p>Specifies the reduction to apply to the loss associated to each
 target: 'mean' | 'sum'. Note that this is NOT the same as the
 reduction in the cross_entropy loss or the
 binary_cross_entropy_with_logits. This reduction is applied after the
 loss for each target has been computed. Alias: 'target_reduction'</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>binary_trick</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>binary_trick</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>If True, each target will be considered independently and the loss
+          –
+          <div class="doc-md-description">
+            <p>If True, each target will be considered independently and the loss
 will be computed as binary_cross_entropy_with_logits. This is a
 faster implementation. Note that the 'weights' parameter is not
 compatible with binary_trick=True. Also note that if
@@ -5533,13 +5169,12 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
   WideDeep class should be adjusted accordingly (adding 2 to per
   binary target). For example, in a problem with a binary target and
   a 4 class multiclassification target, the pred_dim should be 6.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses_multitarget</span> <span class="kn">import</span> <span class="n">MultiTargetClassificationLoss</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
@@ -5554,9 +5189,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
 <span class="gp">&gt;&gt;&gt; </span><span class="n">output_3</span> <span class="o">=</span> <span class="n">loss_3</span><span class="p">(</span><span class="n">input_binary_trick</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
 <span class="normal">145</span>
 <span class="normal">146</span>
 <span class="normal">147</span>
@@ -5652,12 +5287,11 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
             <span class="k">else</span> <span class="kc">None</span>
         <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -5668,10 +5302,10 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -5680,11 +5314,11 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
 
 
 <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">MutilTargetRegressionAndClassificationLoss</span>
+            <span class="doc doc-object-name doc-class-name">MutilTargetRegressionAndClassificationLoss</span>
 
 
 <a href="#pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">MutilTargetRegressionAndClassificationLoss</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">MutilTargetRegressionAndClassificationLoss</span><span class="p">(</span>
     <span class="n">regression_config</span><span class="o">=</span><span class="p">[],</span>
     <span class="n">binary_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">multiclass_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -5694,89 +5328,88 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>This class is a wrapper around the MultiTargetRegressionLoss and the
 MultiTargetClassificationLoss. It allows for multi-target regression and
 classification problems. The user can provide a list of weights to apply to
 each target. The loss can be either the sum or the mean of the individual losses</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>regression_config</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
-                <code>[]</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>regression_config</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+                  <code>[]</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of integers with the indices of the regression targets</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>binary_config</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, float]]]]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>List of integers with the indices of the regression targets</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>binary_config</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, float]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of integers with the index of the target for binary
+          –
+          <div class="doc-md-description">
+            <p>List of integers with the index of the target for binary
 classification or tuples with two elements: the index of the targets
 or binary classification and the positive weight for binary
 classification</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>multiclass_config</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>multiclass_config</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of tuples with two or three elements: the index of the target and the
+          –
+          <div class="doc-md-description">
+            <p>List of tuples with two or three elements: the index of the target and the
 number of classes for multiclass classification, or a tuple with the index of
 the target, the number of classes and a list of weights to apply to each class
 (i.e. the 'weight' parameter in the cross_entropy loss)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>weights</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>weights</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of weights to apply to the loss associated to each target. The
+          –
+          <div class="doc-md-description">
+            <p>List of weights to apply to the loss associated to each target. The
 length of the list must match the number of targets.
 Alias: 'target_weights'</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>reduction</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>, default:
-                <code>&#39;mean&#39;</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>reduction</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>, default:
+                  <code>&#39;mean&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note
+          –
+          <div class="doc-md-description">
+            <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note
 that this is NOT the same as the reduction in the cross_entropy loss,
 the binary_cross_entropy_with_logits or the MSELoss. This reduction
 is applied after each target has been computed. Alias: 'target_reduction'</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>binary_trick</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>binary_trick</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>If True, each target will be considered independently and the loss
+          –
+          <div class="doc-md-description">
+            <p>If True, each target will be considered independently and the loss
 will be computed as binary_cross_entropy_with_logits. This is a
 faster implementation. Note that the 'weights' parameter is not
 compatible with binary_trick=True. Also note that if
@@ -5790,13 +5423,12 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
   WideDeep class should be adjusted accordingly (adding 2 to per
   binary target). For example, in a problem with a binary target and
   a 4 class multiclassification target, the pred_dim should be 6.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.losses_multitarget</span> <span class="kn">import</span> <span class="n">MutilTargetRegressionAndClassificationLoss</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
@@ -5806,9 +5438,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
 <span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">359</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">359</span>
 <span class="normal">360</span>
 <span class="normal">361</span>
 <span class="normal">362</span>
@@ -5954,12 +5586,11 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
         <span class="n">binary_trick</span><span class="o">=</span><span class="n">binary_trick</span><span class="p">,</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -5970,10 +5601,10 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -6049,7 +5680,7 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -6067,7 +5698,7 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/metrics.html b/mkdocs/site/pytorch-widedeep/metrics.html
index 523cbc6c..33102abf 100644
--- a/mkdocs/site/pytorch-widedeep/metrics.html
+++ b/mkdocs/site/pytorch-widedeep/metrics.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1072,8 +1072,6 @@
         
       
         
-      
-        
       
         
       
@@ -1411,11 +1409,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1432,11 +1430,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1453,11 +1451,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1474,11 +1472,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1495,11 +1493,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1516,11 +1514,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1537,11 +1535,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1558,32 +1556,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1670,40 +1647,38 @@ <h1 id="metrics">Metrics<a class="headerlink" href="#metrics" title="Permanent l
 
 
 <h2 id="pytorch_widedeep.metrics.Accuracy" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Accuracy</span>
+            <span class="doc doc-object-name doc-class-name">Accuracy</span>
 
 
 <a href="#pytorch_widedeep.metrics.Accuracy" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Accuracy</span><span class="p">(</span><span class="n">top_k</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Accuracy</span><span class="p">(</span><span class="n">top_k</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
 
-  
-      <p>Class to calculate the accuracy for both binary and categorical problems</p>
 
+      <p>Class to calculate the accuracy for both binary and categorical problems</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>top_k</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>top_k</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Accuracy will be computed using the top k most likely classes in
+          –
+          <div class="doc-md-description">
+            <p>Accuracy will be computed using the top k most likely classes in
 multiclass problems</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.metrics</span> <span class="kn">import</span> <span class="n">Accuracy</span>
@@ -1721,9 +1696,9 @@ <h2 id="pytorch_widedeep.metrics.Accuracy" class="doc doc-heading">
 <span class="go">array(0.66666667)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">75</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">75</span>
 <span class="normal">76</span>
 <span class="normal">77</span>
 <span class="normal">78</span>
@@ -1737,12 +1712,11 @@ <h2 id="pytorch_widedeep.metrics.Accuracy" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">total_count</span> <span class="o">=</span> <span class="mi">0</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;acc&quot;</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1755,22 +1729,21 @@ <h2 id="pytorch_widedeep.metrics.Accuracy" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.metrics.Accuracy.reset" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">reset</span>
+            <span class="doc doc-object-name doc-function-name">reset</span>
 
 
 <a href="#pytorch_widedeep.metrics.Accuracy.reset" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">reset</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">reset</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>resets counters to 0</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">83</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">83</span>
 <span class="normal">84</span>
 <span class="normal">85</span>
 <span class="normal">86</span>
@@ -1782,8 +1755,8 @@ <h3 id="pytorch_widedeep.metrics.Accuracy.reset" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">correct_count</span> <span class="o">=</span> <span class="mi">0</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">total_count</span> <span class="o">=</span> <span class="mi">0</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -1791,8 +1764,7 @@ <h3 id="pytorch_widedeep.metrics.Accuracy.reset" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -1801,40 +1773,38 @@ <h3 id="pytorch_widedeep.metrics.Accuracy.reset" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.metrics.Precision" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Precision</span>
+            <span class="doc doc-object-name doc-class-name">Precision</span>
 
 
 <a href="#pytorch_widedeep.metrics.Precision" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Precision</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Precision</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
 
-  
-      <p>Class to calculate the precision for both binary and categorical problems</p>
 
+      <p>Class to calculate the precision for both binary and categorical problems</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>average</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>average</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>This applies only to multiclass problems. if <code>True</code> calculate
+          –
+          <div class="doc-md-description">
+            <p>This applies only to multiclass problems. if <code>True</code> calculate
 precision for each label, and finds their unweighted mean.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.metrics</span> <span class="kn">import</span> <span class="n">Precision</span>
@@ -1852,9 +1822,9 @@ <h2 id="pytorch_widedeep.metrics.Precision" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">134</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">134</span>
 <span class="normal">135</span>
 <span class="normal">136</span>
 <span class="normal">137</span>
@@ -1870,12 +1840,11 @@ <h2 id="pytorch_widedeep.metrics.Precision" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="mf">1e-20</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;prec&quot;</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1888,22 +1857,21 @@ <h2 id="pytorch_widedeep.metrics.Precision" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.metrics.Precision.reset" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">reset</span>
+            <span class="doc doc-object-name doc-function-name">reset</span>
 
 
 <a href="#pytorch_widedeep.metrics.Precision.reset" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">reset</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">reset</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>resets counters to 0</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">143</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">143</span>
 <span class="normal">144</span>
 <span class="normal">145</span>
 <span class="normal">146</span>
@@ -1915,8 +1883,8 @@ <h3 id="pytorch_widedeep.metrics.Precision.reset" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">=</span> <span class="mi">0</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">all_positives</span> <span class="o">=</span> <span class="mi">0</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -1924,8 +1892,7 @@ <h3 id="pytorch_widedeep.metrics.Precision.reset" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -1934,40 +1901,38 @@ <h3 id="pytorch_widedeep.metrics.Precision.reset" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.metrics.Recall" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Recall</span>
+            <span class="doc doc-object-name doc-class-name">Recall</span>
 
 
 <a href="#pytorch_widedeep.metrics.Recall" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Recall</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Recall</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
 
-  
-      <p>Class to calculate the recall for both binary and categorical problems</p>
 
+      <p>Class to calculate the recall for both binary and categorical problems</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>average</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>average</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>This applies only to multiclass problems. if <code>True</code> calculate recall
+          –
+          <div class="doc-md-description">
+            <p>This applies only to multiclass problems. if <code>True</code> calculate recall
 for each label, and finds their unweighted mean.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.metrics</span> <span class="kn">import</span> <span class="n">Recall</span>
@@ -1985,9 +1950,9 @@ <h2 id="pytorch_widedeep.metrics.Recall" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">200</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">200</span>
 <span class="normal">201</span>
 <span class="normal">202</span>
 <span class="normal">203</span>
@@ -2003,12 +1968,11 @@ <h2 id="pytorch_widedeep.metrics.Recall" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="mf">1e-20</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;rec&quot;</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2021,22 +1985,21 @@ <h2 id="pytorch_widedeep.metrics.Recall" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.metrics.Recall.reset" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">reset</span>
+            <span class="doc doc-object-name doc-function-name">reset</span>
 
 
 <a href="#pytorch_widedeep.metrics.Recall.reset" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">reset</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">reset</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>resets counters to 0</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">209</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">209</span>
 <span class="normal">210</span>
 <span class="normal">211</span>
 <span class="normal">212</span>
@@ -2048,8 +2011,8 @@ <h3 id="pytorch_widedeep.metrics.Recall.reset" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">=</span> <span class="mi">0</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">actual_positives</span> <span class="o">=</span> <span class="mi">0</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2057,8 +2020,7 @@ <h3 id="pytorch_widedeep.metrics.Recall.reset" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2067,51 +2029,49 @@ <h3 id="pytorch_widedeep.metrics.Recall.reset" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">FBetaScore</span>
+            <span class="doc doc-object-name doc-class-name">FBetaScore</span>
 
 
 <a href="#pytorch_widedeep.metrics.FBetaScore" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">FBetaScore</span><span class="p">(</span><span class="n">beta</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FBetaScore</span><span class="p">(</span><span class="n">beta</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
+
 
-  
       <p>Class to calculate the fbeta score for both binary and categorical problems</p>
 <div class="arithmatex">\[
 F_{\beta} = ((1 + {\beta}^2) * \frac{(precision * recall)}{({\beta}^2 * precision + recall)}
 \]</div>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>beta</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Coefficient to control the balance between precision and recall</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>average</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>beta</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Coefficient to control the balance between precision and recall</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>average</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta
+          –
+          <div class="doc-md-description">
+            <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta
 for each label, and find their unweighted mean.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.metrics</span> <span class="kn">import</span> <span class="n">FBetaScore</span>
@@ -2129,9 +2089,9 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">272</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">272</span>
 <span class="normal">273</span>
 <span class="normal">274</span>
 <span class="normal">275</span>
@@ -2149,12 +2109,11 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="mf">1e-20</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="s2">&quot;f&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">)])</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2167,22 +2126,21 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.metrics.FBetaScore.reset" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">reset</span>
+            <span class="doc doc-object-name doc-function-name">reset</span>
 
 
 <a href="#pytorch_widedeep.metrics.FBetaScore.reset" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">reset</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">reset</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>resets precision and recall</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">282</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">282</span>
 <span class="normal">283</span>
 <span class="normal">284</span>
 <span class="normal">285</span>
@@ -2194,8 +2152,8 @@ <h3 id="pytorch_widedeep.metrics.FBetaScore.reset" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">precision</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">recall</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2203,8 +2161,7 @@ <h3 id="pytorch_widedeep.metrics.FBetaScore.reset" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2213,40 +2170,38 @@ <h3 id="pytorch_widedeep.metrics.FBetaScore.reset" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.metrics.F1Score" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">F1Score</span>
+            <span class="doc doc-object-name doc-class-name">F1Score</span>
 
 
 <a href="#pytorch_widedeep.metrics.F1Score" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">F1Score</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">F1Score</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
 
-  
-      <p>Class to calculate the f1 score for both binary and categorical problems</p>
 
+      <p>Class to calculate the f1 score for both binary and categorical problems</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>average</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>average</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for
+          –
+          <div class="doc-md-description">
+            <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for
 each label, and find their unweighted mean.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.metrics</span> <span class="kn">import</span> <span class="n">F1Score</span>
@@ -2264,9 +2219,9 @@ <h2 id="pytorch_widedeep.metrics.F1Score" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">330</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">330</span>
 <span class="normal">331</span>
 <span class="normal">332</span>
 <span class="normal">333</span>
@@ -2278,12 +2233,11 @@ <h2 id="pytorch_widedeep.metrics.F1Score" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">f1</span> <span class="o">=</span> <span class="n">FBetaScore</span><span class="p">(</span><span class="n">beta</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">average</span><span class="p">)</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">f1</span><span class="o">.</span><span class="n">_name</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2296,22 +2250,21 @@ <h2 id="pytorch_widedeep.metrics.F1Score" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.metrics.F1Score.reset" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">reset</span>
+            <span class="doc doc-object-name doc-function-name">reset</span>
 
 
 <a href="#pytorch_widedeep.metrics.F1Score.reset" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">reset</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">reset</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>resets counters to 0</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">337</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">337</span>
 <span class="normal">338</span>
 <span class="normal">339</span>
 <span class="normal">340</span>
@@ -2321,8 +2274,8 @@ <h3 id="pytorch_widedeep.metrics.F1Score.reset" class="doc doc-heading">
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">f1</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2330,8 +2283,7 @@ <h3 id="pytorch_widedeep.metrics.F1Score.reset" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2340,18 +2292,18 @@ <h3 id="pytorch_widedeep.metrics.F1Score.reset" class="doc doc-heading">
 
 
 <h2 id="pytorch_widedeep.metrics.R2Score" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">R2Score</span>
+            <span class="doc doc-object-name doc-class-name">R2Score</span>
 
 
 <a href="#pytorch_widedeep.metrics.R2Score" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">R2Score</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">R2Score</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.metrics.Metric">Metric</span></code></p>
+
 
-  
       <p>Calculates R-Squared, the
 <a href="https://en.wikipedia.org/wiki/Coefficient_of_determination&gt;">coefficient of determination</a>:</p>
 <div class="arithmatex">\[
@@ -2361,8 +2313,7 @@ <h2 id="pytorch_widedeep.metrics.R2Score" class="doc doc-heading">
 <span class="arithmatex">\(\bar{y}\)</span> is the mean of the ground truth.</p>
 
 
-
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.metrics</span> <span class="kn">import</span> <span class="n">R2Score</span>
@@ -2374,9 +2325,9 @@ <h2 id="pytorch_widedeep.metrics.R2Score" class="doc doc-heading">
 <span class="go">array(0.94860814)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">372</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">372</span>
 <span class="normal">373</span>
 <span class="normal">374</span>
 <span class="normal">375</span>
@@ -2390,12 +2341,11 @@ <h2 id="pytorch_widedeep.metrics.R2Score" class="doc doc-heading">
 
     <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;r2&quot;</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2408,22 +2358,21 @@ <h2 id="pytorch_widedeep.metrics.R2Score" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.metrics.R2Score.reset" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">reset</span>
+            <span class="doc doc-object-name doc-function-name">reset</span>
 
 
 <a href="#pytorch_widedeep.metrics.R2Score.reset" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">reset</span><span class="p">()</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">reset</span><span class="p">()</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>resets counters to 0</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">380</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">380</span>
 <span class="normal">381</span>
 <span class="normal">382</span>
 <span class="normal">383</span>
@@ -2439,8 +2388,8 @@ <h3 id="pytorch_widedeep.metrics.R2Score.reset" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">num_examples</span> <span class="o">=</span> <span class="mi">0</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">y_true_sum</span> <span class="o">=</span> <span class="mi">0</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2448,8 +2397,7 @@ <h3 id="pytorch_widedeep.metrics.R2Score.reset" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2525,7 +2473,7 @@ <h3 id="pytorch_widedeep.metrics.R2Score.reset" class="doc doc-heading">
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -2543,7 +2491,7 @@ <h3 id="pytorch_widedeep.metrics.R2Score.reset" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/model_components.html b/mkdocs/site/pytorch-widedeep/model_components.html
index 4d58ed48..7b693485 100644
--- a/mkdocs/site/pytorch-widedeep/model_components.html
+++ b/mkdocs/site/pytorch-widedeep/model_components.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -959,15 +959,6 @@
     </span>
   </a>
   
-</li>
-      
-        <li class="md-nav__item">
-  <a href="#pytorch_widedeep.models.fds_layer.FDSLayer" class="md-nav__link">
-    <span class="md-ellipsis">
-      FDSLayer
-    </span>
-  </a>
-  
 </li>
       
     </ul>
@@ -1225,8 +1216,6 @@
         
       
         
-      
-        
       
         
       
@@ -1564,32 +1553,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1606,11 +1574,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1627,11 +1595,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1648,11 +1616,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1669,11 +1637,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1690,11 +1658,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1711,11 +1679,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1732,11 +1700,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1805,70 +1773,67 @@ <h1 id="the-models-module">The <code>models</code> module<a class="headerlink" h
 
 
 <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Wide</span>
+            <span class="doc doc-object-name doc-class-name">Wide</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.linear.wide.Wide" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Wide</span><span class="p">(</span><span class="n">input_dim</span><span class="p">,</span> <span class="n">pred_dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Wide</span><span class="p">(</span><span class="n">input_dim</span><span class="p">,</span> <span class="n">pred_dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Defines a <code>Wide</code> (linear) model where the non-linearities are
 captured via the so-called crossed-columns. This can be used as the
 <code>wide</code> component of a Wide &amp; Deep model.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input_dim</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>size of the Linear layer (implemented via an Embedding layer).
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input_dim</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>size of the Linear layer (implemented via an Embedding layer).
 <code>input_dim</code> is the summation of all the individual values for all the
 features that go through the wide model. For example, if the wide
 model receives 2 features with 5 individual values each, <code>input_dim =
 10</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pred_dim</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>size of the ouput tensor containing the predictions. Note that unlike
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pred_dim</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>size of the ouput tensor containing the predictions. Note that unlike
 all the other models, the wide model is connected directly to the
 output neuron(s) when used to build a Wide and Deep model. Therefore,
 it requires the <code>pred_dim</code> parameter.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.linear.wide.Wide.wide_linear">wide_linear</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>the linear layer that comprises the wide branch of the model</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.linear.wide.Wide.wide_linear">wide_linear</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>the linear layer that comprises the wide branch of the model</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">Wide</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
@@ -1876,9 +1841,9 @@ <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">wide</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">43</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">43</span>
 <span class="normal">44</span>
 <span class="normal">45</span>
 <span class="normal">46</span>
@@ -1902,12 +1867,11 @@ <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading
     <span class="bp">self</span><span class="o">.</span><span class="n">bias</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">pred_dim</span><span class="p">))</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">_reset_parameters</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1920,23 +1884,22 @@ <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.models.tabular.linear.wide.Wide.forward" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">forward</span>
+            <span class="doc doc-object-name doc-function-name">forward</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.linear.wide.Wide.forward" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">forward</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">forward</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Forward pass. Simply connecting the Embedding layer with the ouput
 neuron(s)</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">65</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">65</span>
 <span class="normal">66</span>
 <span class="normal">67</span>
 <span class="normal">68</span>
@@ -1946,8 +1909,8 @@ <h3 id="pytorch_widedeep.models.tabular.linear.wide.Wide.forward" class="doc doc
     <span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">wide_linear</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">long</span><span class="p">())</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bias</span>
     <span class="k">return</span> <span class="n">out</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -1955,8 +1918,7 @@ <h3 id="pytorch_widedeep.models.tabular.linear.wide.Wide.forward" class="doc doc
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -1965,11 +1927,11 @@ <h3 id="pytorch_widedeep.models.tabular.linear.wide.Wide.forward" class="doc doc
 
 
 <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TabMlp</span>
+            <span class="doc doc-object-name doc-class-name">TabMlp</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TabMlp</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabMlp</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -1997,11 +1959,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithoutAttention">BaseTabularModelWithoutAttention</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithoutAttention">BaseTabularModelWithoutAttention</span></code></p>
+
 
-  
       <p>Defines a <code>TabMlp</code> model that can be used as the <code>deeptabular</code>
 component of a Wide &amp; Deep model or independently by itself.</p>
 <p>This class combines embedding representations of the categorical features
@@ -2012,108 +1974,107 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 categorical features only, continuous features only or both).</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name, number of unique values and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name, number of unique values and
 embedding dimension. e.g. <em>[(education, 11, 32), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Categorical embeddings dropout. If <code>None</code>, it will default
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cat_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the categorical embeddings.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cat_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the continuous columns will be embedded using
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the continuous columns will be embedded using
 one of the available methods: <em>'standard'</em>, <em>'periodic'</em>
 or <em>'piecewise'</em>. If <code>None</code>, it will default to 'False'.<br/>
 <img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: This parameter is deprecated and it
  will be removed in future releases. Please, use the
  <code>embed_continuous_method</code> parameter instead.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Method to use to embed the continuous features. Options are:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -2121,194 +2082,192 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dim</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Size of the continuous embeddings. If the continuous columns are
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dim</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Size of the continuous embeddings. If the continuous columns are
 embedded, <code>cont_embed_dim</code> must be passed.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_frequencies</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_frequencies</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the sigma parameter in the paper mentioned when describing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>sigma</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_last_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is not present in the before mentioned paper but it is implemented in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_last_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>full_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>full_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
-                <code>[200, 100]</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the mlp.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers of the MLP. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+                  <code>[200, 100]</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the mlp.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>float or List of floats with the dropout between the dense layers.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>float or List of floats with the dropout between the dense layers.
 e.g: <em>[0.5,0.5]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm_last</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm_last</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_linear_first</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_linear_first</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>mlp model that will receive the concatenation of the embeddings and
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>mlp model that will receive the concatenation of the embeddings and
 the continuous columns</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabMlp</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -2320,9 +2279,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
 <span class="normal">145</span>
 <span class="normal">146</span>
 <span class="normal">147</span>
@@ -2458,9 +2417,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
         <span class="n">mlp_linear_first</span><span class="p">,</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -2475,9 +2434,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 
 
 <h3 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -2486,11 +2445,11 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim" class="do
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -2500,8 +2459,7 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim" class="do
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2510,11 +2468,11 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim" class="do
 
 
 <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TabMlpDecoder</span>
+            <span class="doc doc-object-name doc-class-name">TabMlpDecoder</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TabMlpDecoder</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabMlpDecoder</span><span class="p">(</span>
     <span class="n">embed_dim</span><span class="p">,</span>
     <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="p">[</span><span class="mi">100</span><span class="p">,</span> <span class="mi">200</span><span class="p">],</span>
     <span class="n">mlp_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
@@ -2525,11 +2483,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Companion decoder model for the <code>TabMlp</code> model (which can be considered
 an encoder itself).</p>
 <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when
@@ -2538,102 +2496,99 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
 and '<em>reconstruct</em>' the embeddings.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>embed_dim</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Size of the embeddings tensor that needs to be reconstructed.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
-                <code>[100, 200]</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the mlp.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers of the MLP. Currently
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>embed_dim</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Size of the embeddings tensor that needs to be reconstructed.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+                  <code>[100, 200]</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the mlp.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>float or List of floats with the dropout between the dense layers.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>float or List of floats with the dropout between the dense layers.
 e.g: <em>[0.5,0.5]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm_last</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm_last</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_linear_first</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_linear_first</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder.decoder">decoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>mlp model that will receive the output of the encoder</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder.decoder">decoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>mlp model that will receive the output of the encoder</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabMlpDecoder</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">x_inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">8</span><span class="p">)</span>
@@ -2643,9 +2598,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
 <span class="go">torch.Size([3, 32])</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">279</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">279</span>
 <span class="normal">280</span>
 <span class="normal">281</span>
 <span class="normal">282</span>
@@ -2703,12 +2658,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
         <span class="n">mlp_linear_first</span><span class="p">,</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2719,10 +2673,10 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -2731,11 +2685,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
 
 
 <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TabResnet</span>
+            <span class="doc doc-object-name doc-class-name">TabResnet</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TabResnet</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabResnet</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -2766,11 +2720,11 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithoutAttention">BaseTabularModelWithoutAttention</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithoutAttention">BaseTabularModelWithoutAttention</span></code></p>
+
 
-  
       <p>Defines a <code>TabResnet</code> model that can be used as the <code>deeptabular</code>
 component of a Wide &amp; Deep model or independently by itself.</p>
 <p>This class combines embedding representations of the categorical features
@@ -2783,108 +2737,107 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 categorical features only, continuous features only or both).</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name, number of unique values and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name, number of unique values and
 embedding dimension. e.g. <em>[(education, 11, 32), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Categorical embeddings dropout. If <code>None</code>, it will default
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cat_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the categorical embeddings.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cat_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the continuous columns will be embedded using
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the continuous columns will be embedded using
 one of the available methods: <em>'standard'</em>, <em>'periodic'</em>
 or <em>'piecewise'</em>. If <code>None</code>, it will default to 'False'.<br/>
 <img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: This parameter is deprecated and it
  will be removed in future releases. Please, use the
  <code>embed_continuous_method</code> parameter instead.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Method to use to embed the continuous features. Options are:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -2892,252 +2845,250 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dim</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Size of the continuous embeddings. If the continuous columns are
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dim</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Size of the continuous embeddings. If the continuous columns are
 embedded, <code>cont_embed_dim</code> must be passed.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_frequencies</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_frequencies</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the sigma parameter in the paper mentioned when describing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>sigma</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_last_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is not present in the before mentioned paper but it is implemented in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_last_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>full_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>full_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>blocks_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
-                <code>[200, 100, 100]</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of integers that define the input and output units of each block.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>blocks_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+                  <code>[200, 100, 100]</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of integers that define the input and output units of each block.
 For example: <em>[200, 100, 100]</em> will generate 2 blocks. The first will
 receive a tensor of size 200 and output a tensor of size 100, and the
 second will receive a tensor of size 100 and output a tensor of size
 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for
 details on the structure of each block.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>blocks_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Block's internal dropout.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>simplify_blocks</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>blocks_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Block's internal dropout.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>simplify_blocks</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [
 [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one
 (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the MLP. e.g:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If <code>None</code> the  output of the Resnet Blocks will be
 connected directly to the output neuron(s).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers of the MLP. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>float with the dropout between the dense layers of the MLP.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm_last</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm_last</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_linear_first</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_linear_first</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>deep dense Resnet model that will receive the concatenation of the
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>deep dense Resnet model that will receive the concatenation of the
 embeddings and the continuous columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.mlp">mlp</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.mlp">mlp</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp
 model that will receive the results of the concatenation of the
 embeddings and the continuous columns -- if present --.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabResnet</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_deep</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -3149,9 +3100,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_deep</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">174</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">174</span>
 <span class="normal">175</span>
 <span class="normal">176</span>
 <span class="normal">177</span>
@@ -3345,9 +3296,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
     <span class="k">else</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -3362,9 +3313,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 
 
 <h3 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -3373,11 +3324,11 @@ <h3 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim"
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -3387,8 +3338,7 @@ <h3 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim"
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -3397,11 +3347,11 @@ <h3 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim"
 
 
 <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TabResnetDecoder</span>
+            <span class="doc doc-object-name doc-class-name">TabResnetDecoder</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TabResnetDecoder</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabResnetDecoder</span><span class="p">(</span>
     <span class="n">embed_dim</span><span class="p">,</span>
     <span class="n">blocks_dims</span><span class="o">=</span><span class="p">[</span><span class="mi">100</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">200</span><span class="p">],</span>
     <span class="n">blocks_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
@@ -3415,11 +3365,11 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Companion decoder model for the <code>TabResnet</code> model (which can be
 considered an encoder itself)</p>
 <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when
@@ -3428,161 +3378,158 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 MLP(if present) and '<em>reconstruct</em>' the embeddings.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>embed_dim</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Size of the embeddings tensor to be reconstructed.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>blocks_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
-                <code>[100, 100, 200]</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of integers that define the input and output units of each block.
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>embed_dim</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Size of the embeddings tensor to be reconstructed.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>blocks_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+                  <code>[100, 100, 200]</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of integers that define the input and output units of each block.
 For example: <em>[200, 100, 100]</em> will generate 2 blocks. The first will
 receive a tensor of size 200 and output a tensor of size 100, and the
 second will receive a tensor of size 100 and output a tensor of size
 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for
 details on the structure of each block.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>blocks_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Block's internal dropout.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>simplify_blocks</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>blocks_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Block's internal dropout.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>simplify_blocks</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [
 [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one
 (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the MLP. e.g:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If <code>None</code> the  output of the Resnet Blocks will be
 connected directly to the output neuron(s).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers of the MLP. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>float with the dropout between the dense layers of the MLP.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm_last</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm_last</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_linear_first</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_linear_first</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder.decoder">decoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>deep dense Resnet model that will receive the output of the encoder IF
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder.decoder">decoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>deep dense Resnet model that will receive the output of the encoder IF
 <code>mlp_hidden_dims</code> is None</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder.mlp">mlp</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder.mlp">mlp</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist
 in an MLP that will receive the output of the encoder followed by the
 deep dense Resnet.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabResnetDecoder</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">x_inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">8</span><span class="p">)</span>
@@ -3592,9 +3539,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 <span class="go">torch.Size([3, 32])</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">368</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">368</span>
 <span class="normal">369</span>
 <span class="normal">370</span>
 <span class="normal">371</span>
@@ -3720,12 +3667,11 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 
     <span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">blocks_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">embed_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -3736,10 +3682,10 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -3748,11 +3694,11 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 
 
 <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TabNet</span>
+            <span class="doc doc-object-name doc-class-name">TabNet</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TabNet</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabNet</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -3786,11 +3732,11 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithoutAttention">BaseTabularModelWithoutAttention</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithoutAttention">BaseTabularModelWithoutAttention</span></code></p>
+
 
-  
       <p>Defines a <a href="https://arxiv.org/abs/1908.07442">TabNet model</a> that
 can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or
 independently by itself.</p>
@@ -3800,108 +3746,107 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 Therefore, <strong>ALL CREDIT TO THE DREAMQUARK-AI TEAM</strong>.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name, number of unique values and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name, number of unique values and
 embedding dimension. e.g. <em>[(education, 11, 32), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Categorical embeddings dropout. If <code>None</code>, it will default
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cat_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the categorical embeddings.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cat_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the continuous columns will be embedded using
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the continuous columns will be embedded using
 one of the available methods: <em>'standard'</em>, <em>'periodic'</em>
 or <em>'piecewise'</em>. If <code>None</code>, it will default to 'False'.<br/>
 <img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: This parameter is deprecated and it
  will be removed in future releases. Please, use the
  <code>embed_continuous_method</code> parameter instead.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Method to use to embed the continuous features. Options are:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -3909,257 +3854,255 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dim</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Size of the continuous embeddings. If the continuous columns are
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dim</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Size of the continuous embeddings. If the continuous columns are
 embedded, <code>cont_embed_dim</code> must be passed.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_frequencies</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_frequencies</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the sigma parameter in the paper mentioned when describing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>sigma</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_last_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is not present in the before mentioned paper but it is implemented in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_last_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>full_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>full_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_steps</code></b>
-            (<code>int</code>, default:
-                <code>3</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>number of decision steps. For a better understanding of the function
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_steps</code></b>
+              (<code>int</code>, default:
+                  <code>3</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>number of decision steps. For a better understanding of the function
 of <code>n_steps</code> and the upcoming parameters, please see the
 <a href="https://arxiv.org/abs/1908.07442">paper</a>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>step_dim</code></b>
-            (<code>int</code>, default:
-                <code>8</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Step's output dimension. This is the output dimension that
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>step_dim</code></b>
+              (<code>int</code>, default:
+                  <code>8</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Step's output dimension. This is the output dimension that
 <code>WideDeep</code> will collect and connect to the output neuron(s).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dim</code></b>
-            (<code>int</code>, default:
-                <code>8</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Attention dimension</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.0</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>GLU block's internal dropout</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_glu_step_dependent</code></b>
-            (<code>int</code>, default:
-                <code>2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_glu_shared</code></b>
-            (<code>int</code>, default:
-                <code>2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dim</code></b>
+              (<code>int</code>, default:
+                  <code>8</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Attention dimension</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.0</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>GLU block's internal dropout</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_glu_step_dependent</code></b>
+              (<code>int</code>, default:
+                  <code>2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_glu_shared</code></b>
+              (<code>int</code>, default:
+                  <code>2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared
 across decision steps</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ghost_bn</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if <a href="https://arxiv.org/abs/1705.08741">Ghost Batch Normalization</a>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ghost_bn</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if <a href="https://arxiv.org/abs/1705.08741">Ghost Batch Normalization</a>
 will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>virtual_batch_size</code></b>
-            (<code>int</code>, default:
-                <code>128</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Batch size when using Ghost Batch Normalization</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>momentum</code></b>
-            (<code>float</code>, default:
-                <code>0.02</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>virtual_batch_size</code></b>
+              (<code>int</code>, default:
+                  <code>128</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Batch size when using Ghost Batch Normalization</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>momentum</code></b>
+              (<code>float</code>, default:
+                  <code>0.02</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for
 very low values. However high values are used in the original
 publication. During our tests higher values lead to better results</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>gamma</code></b>
-            (<code>float</code>, default:
-                <code>1.3</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Relaxation parameter in the paper. When gamma = 1, a feature is
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>gamma</code></b>
+              (<code>float</code>, default:
+                  <code>1.3</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Relaxation parameter in the paper. When gamma = 1, a feature is
 enforced to be used only at one decision step. As gamma increases,
 more flexibility is provided to use a feature at multiple decision
 steps</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>epsilon</code></b>
-            (<code>float</code>, default:
-                <code>1e-15</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Float to avoid log(0). Always keep low</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mask_type</code></b>
-            (<code>str</code>, default:
-                <code>&#39;sparsemax&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Mask function to use. Either <em>'sparsemax'</em> or <em>'entmax'</em></p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>epsilon</code></b>
+              (<code>float</code>, default:
+                  <code>1e-15</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Float to avoid log(0). Always keep low</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mask_type</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;sparsemax&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Mask function to use. Either <em>'sparsemax'</em> or <em>'entmax'</em></p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>the TabNet encoder. For details see the <a href="https://arxiv.org/abs/1908.07442">original publication</a>.</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>the TabNet encoder. For details see the <a href="https://arxiv.org/abs/1908.07442">original publication</a>.</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">colnames</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;a&quot;</span><span class="p">,</span> <span class="s2">&quot;b&quot;</span><span class="p">,</span> <span class="s2">&quot;c&quot;</span><span class="p">,</span> <span class="s2">&quot;d&quot;</span><span class="p">,</span> <span class="s2">&quot;e&quot;</span><span class="p">]</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">cat_embed_input</span> <span class="o">=</span> <span class="p">[(</span><span class="n">u</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">)</span> <span class="k">for</span> <span class="n">u</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">colnames</span><span class="p">[:</span><span class="mi">4</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">]</span> <span class="o">*</span> <span class="mi">4</span><span class="p">,</span> <span class="p">[</span><span class="mi">8</span><span class="p">]</span> <span class="o">*</span> <span class="mi">4</span><span class="p">)]</span>
@@ -4168,9 +4111,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">158</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">158</span>
 <span class="normal">159</span>
 <span class="normal">160</span>
 <span class="normal">161</span>
@@ -4344,9 +4287,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
         <span class="n">mask_type</span><span class="p">,</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -4361,9 +4304,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 
 
 <h3 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -4372,11 +4315,11 @@ <h3 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim" class=
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -4386,8 +4329,7 @@ <h3 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim" class=
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -4396,11 +4338,11 @@ <h3 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim" class=
 
 
 <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TabNetDecoder</span>
+            <span class="doc doc-object-name doc-class-name">TabNetDecoder</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TabNetDecoder</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabNetDecoder</span><span class="p">(</span>
     <span class="n">embed_dim</span><span class="p">,</span>
     <span class="n">n_steps</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
     <span class="n">step_dim</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span>
@@ -4413,11 +4355,11 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Companion decoder model for the <code>TabNet</code> model (which can be
 considered an encoder itself)</p>
 <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when
@@ -4427,124 +4369,121 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
 embeddings.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>embed_dim</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Size of the embeddings tensor to be reconstructed.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_steps</code></b>
-            (<code>int</code>, default:
-                <code>3</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>number of decision steps. For a better understanding of the function
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>embed_dim</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Size of the embeddings tensor to be reconstructed.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_steps</code></b>
+              (<code>int</code>, default:
+                  <code>3</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>number of decision steps. For a better understanding of the function
 of <code>n_steps</code> and the upcoming parameters, please see the
 <a href="https://arxiv.org/abs/1908.07442">paper</a>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>step_dim</code></b>
-            (<code>int</code>, default:
-                <code>8</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Step's output dimension. This is the output dimension that
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>step_dim</code></b>
+              (<code>int</code>, default:
+                  <code>8</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Step's output dimension. This is the output dimension that
 <code>WideDeep</code> will collect and connect to the output neuron(s).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.0</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>GLU block's internal dropout</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_glu_step_dependent</code></b>
-            (<code>int</code>, default:
-                <code>2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_glu_shared</code></b>
-            (<code>int</code>, default:
-                <code>2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.0</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>GLU block's internal dropout</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_glu_step_dependent</code></b>
+              (<code>int</code>, default:
+                  <code>2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_glu_shared</code></b>
+              (<code>int</code>, default:
+                  <code>2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared
 across decision steps</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ghost_bn</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if <a href="https://arxiv.org/abs/1705.08741">Ghost Batch Normalization</a>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ghost_bn</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if <a href="https://arxiv.org/abs/1705.08741">Ghost Batch Normalization</a>
 will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>virtual_batch_size</code></b>
-            (<code>int</code>, default:
-                <code>128</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Batch size when using Ghost Batch Normalization</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>momentum</code></b>
-            (<code>float</code>, default:
-                <code>0.02</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>virtual_batch_size</code></b>
+              (<code>int</code>, default:
+                  <code>128</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Batch size when using Ghost Batch Normalization</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>momentum</code></b>
+              (<code>float</code>, default:
+                  <code>0.02</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for
 very low values. However high values are used in the original
 publication. During our tests higher values lead to better results</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder.decoder">decoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>decoder that will receive the output from the encoder's steps and will
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder.decoder">decoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>decoder that will receive the output from the encoder's steps and will
 reconstruct the embeddings</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabNetDecoder</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">x_inp</span> <span class="o">=</span> <span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">8</span><span class="p">)]</span>
@@ -4554,9 +4493,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
 <span class="go">torch.Size([3, 32])</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">342</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">342</span>
 <span class="normal">343</span>
 <span class="normal">344</span>
 <span class="normal">345</span>
@@ -4648,12 +4587,11 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
     <span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">step_dim</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
     <span class="n">initialize_non_glu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span><span class="p">,</span> <span class="n">step_dim</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -4664,10 +4602,10 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -4676,11 +4614,11 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
 
 
 <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">ContextAttentionMLP</span>
+            <span class="doc doc-object-name doc-class-name">ContextAttentionMLP</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">ContextAttentionMLP</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ContextAttentionMLP</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -4708,11 +4646,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+
 
-  
       <p>Defines a <code>ContextAttentionMLP</code> model that can be used as the
 <code>deeptabular</code> component of a Wide &amp; Deep model or independently by
 itself.</p>
@@ -4729,134 +4667,133 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 categorical features only, continuous features only or both).</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name and number of unique values and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Categorical embeddings dropout. If <code>None</code>, it will default
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cat_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the categorical embeddings.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cat_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>add_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The two embedding sharing strategies are: 1) add the shared embeddings
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>add_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>frac_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>frac_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                <code>&#39;standard&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Method to use to embed the continuous features. Options are:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
+                  <code>&#39;standard&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -4864,170 +4801,168 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_frequencies</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_frequencies</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the sigma parameter in the paper mentioned when describing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>sigma</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_last_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is not present in the before mentioned paper but it is implemented in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_last_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>full_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>full_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>input_dim</code></b>
-            (<code>int</code>, default:
-                <code>32</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The so-called <em>dimension of the model</em>. Is the number of embeddings
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>input_dim</code></b>
+              (<code>int</code>, default:
+                  <code>32</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The so-called <em>dimension of the model</em>. Is the number of embeddings
 used to encode the categorical and/or continuous columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for each attention block</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_addnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if residual connections will be used in the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for each attention block</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>with_addnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if residual connections will be used in the
 attention blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;leaky_relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>String indicating the activation function to be applied to the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;leaky_relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>String indicating the activation function to be applied to the dense
 layer in each attention encoder. <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em>
 and <em>'gelu'</em> are supported.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_blocks</code></b>
-            (<code>int</code>, default:
-                <code>3</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention blocks</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_blocks</code></b>
+              (<code>int</code>, default:
+                  <code>3</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention blocks</p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Sequence of attention encoders.</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Sequence of attention encoders.</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">ContextAttentionMLP</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -5038,9 +4973,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">152</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">152</span>
 <span class="normal">153</span>
 <span class="normal">154</span>
 <span class="normal">155</span>
@@ -5188,9 +5123,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             <span class="p">),</span>
         <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -5205,9 +5140,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 
 
 <h3 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -5216,11 +5151,11 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -5229,9 +5164,9 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 
 
 <h3 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.attention_weights" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+            <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -5240,12 +5175,12 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weights</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>List with the attention weights per block</p>
 <p>The shape of the attention weights is <span class="arithmatex">\((N, F)\)</span>, where <span class="arithmatex">\(N\)</span> is the batch
 size and <span class="arithmatex">\(F\)</span> is the number of features/columns in the dataset</p>
-  </div>
+    </div>
 
 </div>
 
@@ -5255,8 +5190,7 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -5265,11 +5199,11 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 
 
 <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">SelfAttentionMLP</span>
+            <span class="doc doc-object-name doc-class-name">SelfAttentionMLP</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">SelfAttentionMLP</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">SelfAttentionMLP</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -5299,11 +5233,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+
 
-  
       <p>Defines a <code>SelfAttentionMLP</code> model that can be used as the
 deeptabular component of a Wide &amp; Deep model or independently by
 itself.</p>
@@ -5332,134 +5266,133 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 categorical features only, continuous features only or both).</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name and number of unique values and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Categorical embeddings dropout. If <code>None</code>, it will default
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cat_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the categorical embeddings.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cat_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>add_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The two embedding sharing strategies are: 1) add the shared embeddings
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>add_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>frac_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>frac_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                <code>&#39;standard&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Method to use to embed the continuous features. Options are:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
+                  <code>&#39;standard&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -5467,198 +5400,196 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_frequencies</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_frequencies</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the sigma parameter in the paper mentioned when describing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>sigma</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_last_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is not present in the before mentioned paper but it is implemented in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_last_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>full_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>full_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>input_dim</code></b>
-            (<code>int</code>, default:
-                <code>32</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The so-called <em>dimension of the model</em>. Is the number of
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>input_dim</code></b>
+              (<code>int</code>, default:
+                  <code>32</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The so-called <em>dimension of the model</em>. Is the number of
 embeddings used to encode the categorical and/or continuous columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for each attention block</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_heads</code></b>
-            (<code>int</code>, default:
-                <code>8</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention heads per attention block.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_bias</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to use bias in the Q, K projection
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for each attention block</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_heads</code></b>
+              (<code>int</code>, default:
+                  <code>8</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention heads per attention block.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_bias</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to use bias in the Q, K projection
 layers.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_addnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if residual connections will be used in the attention blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;leaky_relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>String indicating the activation function to be applied to the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>with_addnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if residual connections will be used in the attention blocks</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;leaky_relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>String indicating the activation function to be applied to the dense
 layer in each attention encoder. <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em>
 and <em>'gelu'</em> are supported.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_blocks</code></b>
-            (<code>int</code>, default:
-                <code>3</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention blocks</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_blocks</code></b>
+              (<code>int</code>, default:
+                  <code>3</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention blocks</p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.cat_and_cont_embed">cat_and_cont_embed</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>This is the module that processes the categorical and continuous columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Sequence of attention encoders.</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.cat_and_cont_embed">cat_and_cont_embed</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>This is the module that processes the categorical and continuous columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Sequence of attention encoders.</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">SelfAttentionMLP</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -5669,9 +5600,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">170</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">170</span>
 <span class="normal">171</span>
 <span class="normal">172</span>
 <span class="normal">173</span>
@@ -5831,9 +5762,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             <span class="p">),</span>
         <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -5848,9 +5779,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 
 
 <h3 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -5859,11 +5790,11 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the WideDeep class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -5872,9 +5803,9 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.
 
 
 <h3 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.attention_weights" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+            <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -5883,13 +5814,13 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weights</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>List with the attention weights per block</p>
 <p>The shape of the attention weights is <span class="arithmatex">\((N, H, F, F)\)</span>, where <span class="arithmatex">\(N\)</span> is the
 batch size, <span class="arithmatex">\(H\)</span> is the number of attention heads and <span class="arithmatex">\(F\)</span> is the
 number of features/columns in the dataset</p>
-  </div>
+    </div>
 
 </div>
 
@@ -5899,8 +5830,7 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -5909,11 +5839,11 @@ <h3 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.
 
 
 <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TabTransformer</span>
+            <span class="doc doc-object-name doc-class-name">TabTransformer</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TabTransformer</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabTransformer</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -5953,11 +5883,11 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+
 
-  
       <p>Defines our adptation of the
 <a href="https://arxiv.org/abs/2012.06678">TabTransformer model</a>
 that can be used as the <code>deeptabular</code> component of a
@@ -5972,134 +5902,133 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 original publication(and some other models in the library)</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name and number of unique values and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Categorical embeddings dropout. If <code>None</code>, it will default
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cat_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the categorical embeddings.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cat_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>add_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The two embedding sharing strategies are: 1) add the shared embeddings
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>add_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>frac_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>frac_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Method to use to embed the continuous features. Options are:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -6107,312 +6036,310 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_frequencies</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_frequencies</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the sigma parameter in the paper mentioned when describing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>sigma</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_last_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is not present in the before mentioned paper but it is implemented in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_last_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>full_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>full_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>input_dim</code></b>
-            (<code>int</code>, default:
-                <code>32</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The so-called <em>dimension of the model</em>. Is the number of
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>input_dim</code></b>
+              (<code>int</code>, default:
+                  <code>32</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The so-called <em>dimension of the model</em>. Is the number of
 embeddings used to encode the categorical and/or continuous columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_heads</code></b>
-            (<code>int</code>, default:
-                <code>8</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention heads per Transformer block</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_qkv_bias</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to use bias in the Q, K, and V
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_heads</code></b>
+              (<code>int</code>, default:
+                  <code>8</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention heads per Transformer block</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_qkv_bias</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to use bias in the Q, K, and V
 projection layers.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_blocks</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of Transformer blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the Multi-Head Attention layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the FeedForward network</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_factor</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Multiplicative factor applied to the first layer of the FF network in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_blocks</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of Transformer blocks</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the Multi-Head Attention layers</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the FeedForward network</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_factor</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>transformer_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;gelu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>transformer_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;gelu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_linear_attention</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if Linear Attention (from <a href="https://arxiv.org/abs/2006.16236">Transformers are RNNs:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_linear_attention</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if Linear Attention (from <a href="https://arxiv.org/abs/2006.16236">Transformers are RNNs:
 Fast Autoregressive Transformers with Linear Attention</a>)
 will be used. The inclusing of this mode of attention is inspired by
 <a href="https://www.uber.com/en-GB/blog/deepeta-how-uber-predicts-arrival-times/">this post</a>,
 where the Uber team finds that this attention mechanism leads to the
 best results for their tabular data.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_flash_attention</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_flash_attention</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if
 <a href="https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html">Flash Attention</a>
 will be used. <br/></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the MLP. e.g:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If not provided no MLP on top of the final
 Transformer block will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers of the MLP. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>float with the dropout between the dense layers of the MLP.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm_last</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm_last</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_linear_first</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_linear_first</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Sequence of Transformer blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.mlp">mlp</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>MLP component in the model</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Sequence of Transformer blocks</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.mlp">mlp</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>MLP component in the model</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabTransformer</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -6424,9 +6351,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">201</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">201</span>
 <span class="normal">202</span>
 <span class="normal">203</span>
 <span class="normal">204</span>
@@ -6686,9 +6613,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
     <span class="k">else</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -6703,9 +6630,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 
 
 <h3 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -6714,11 +6641,11 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -6727,9 +6654,9 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 
 
 <h3 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.attention_weights" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+            <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -6738,8 +6665,8 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weights</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>List with the attention weights per block</p>
 <p>The shape of the attention weights is <span class="arithmatex">\((N, H, F, F)\)</span>, where <span class="arithmatex">\(N\)</span> is the
 batch size, <span class="arithmatex">\(H\)</span> is the number of attention heads and <span class="arithmatex">\(F\)</span> is the
@@ -6748,7 +6675,7 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 if flash attention or linear attention
 are used, no attention weights are saved during the training process
 and calling this property will throw a ValueError</p>
-  </div>
+    </div>
 
 </div>
 
@@ -6758,8 +6685,7 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -6768,11 +6694,11 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 
 
 <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">SAINT</span>
+            <span class="doc doc-object-name doc-class-name">SAINT</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">SAINT</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">SAINT</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -6809,11 +6735,11 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+
 
-  
       <p>Defines a <a href="https://arxiv.org/abs/2106.01342">SAINT model</a> that
 can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or
 independently by itself.</p>
@@ -6824,134 +6750,133 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
  version of the model described in the paper,</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name and number of unique values and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Categorical embeddings dropout. If <code>None</code>, it will default
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cat_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the categorical embeddings.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cat_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>add_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The two embedding sharing strategies are: 1) add the shared embeddings
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>add_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>frac_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>frac_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                <code>&#39;standard&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Method to use to embed the continuous features. Options are:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
+                  <code>&#39;standard&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -6959,286 +6884,284 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_frequencies</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_frequencies</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the sigma parameter in the paper mentioned when describing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>sigma</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_last_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is not present in the before mentioned paper but it is implemented in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_last_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>full_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>full_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>input_dim</code></b>
-            (<code>int</code>, default:
-                <code>32</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The so-called <em>dimension of the model</em>. Is the number of
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>input_dim</code></b>
+              (<code>int</code>, default:
+                  <code>32</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The so-called <em>dimension of the model</em>. Is the number of
 embeddings used to encode the categorical and/or continuous columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_heads</code></b>
-            (<code>int</code>, default:
-                <code>8</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention heads per Transformer block</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_qkv_bias</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to use bias in the Q, K, and V
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_heads</code></b>
+              (<code>int</code>, default:
+                  <code>8</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention heads per Transformer block</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_qkv_bias</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to use bias in the Q, K, and V
 projection layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_blocks</code></b>
-            (<code>int</code>, default:
-                <code>2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of SAINT-Transformer blocks.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the Multi-Head Attention column and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_blocks</code></b>
+              (<code>int</code>, default:
+                  <code>2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of SAINT-Transformer blocks.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the Multi-Head Attention column and
 row layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the FeedForward network</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_factor</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Multiplicative factor applied to the first layer of the FF network in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the FeedForward network</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_factor</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>transformer_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;gelu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>transformer_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;gelu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the MLP. e.g:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If not provided no MLP on top of the final
 Transformer block will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers of the MLP. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>float with the dropout between the dense layers of the MLP.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm_last</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm_last</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_linear_first</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_linear_first</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.saint.SAINT.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Sequence of SAINT-Transformer blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.saint.SAINT.mlp">mlp</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>MLP component in the model</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.saint.SAINT.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Sequence of SAINT-Transformer blocks</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.saint.SAINT.mlp">mlp</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>MLP component in the model</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">SAINT</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -7250,9 +7173,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">184</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">184</span>
 <span class="normal">185</span>
 <span class="normal">186</span>
 <span class="normal">187</span>
@@ -7500,9 +7423,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
     <span class="k">else</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -7517,9 +7440,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
 
 
 <h3 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -7528,11 +7451,11 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim" cla
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -7541,9 +7464,9 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim" cla
 
 
 <h3 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weights" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+            <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -7552,8 +7475,8 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weigh
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weights</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>List with the attention weights. Each element of the list is a tuple
 where the first and the second elements are the column and row
 attention weights respectively</p>
@@ -7568,7 +7491,7 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weigh
 </ul>
 <p>where <span class="arithmatex">\(N\)</span> is the batch size, <span class="arithmatex">\(H\)</span> is the number of heads and <span class="arithmatex">\(F\)</span> is the
 number of features/columns in the dataset</p>
-  </div>
+    </div>
 
 </div>
 
@@ -7578,8 +7501,7 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weigh
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -7588,11 +7510,11 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weigh
 
 
 <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">FTTransformer</span>
+            <span class="doc doc-object-name doc-class-name">FTTransformer</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">FTTransformer</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FTTransformer</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -7631,11 +7553,11 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+
 
-  
       <p>Defines a <a href="https://arxiv.org/abs/2106.11959">FTTransformer model</a> that
 can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or
 independently by itself.</p>
@@ -7644,134 +7566,133 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 categorical features only, continuous features only or both).</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name and number of unique values and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Categorical embeddings dropout. If <code>None</code>, it will default
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cat_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the categorical embeddings.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cat_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>add_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The two embedding sharing strategies are: 1) add the shared embeddings
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>add_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>frac_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>frac_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                <code>&#39;standard&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Method to use to embed the continuous features. Options are:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
+                  <code>&#39;standard&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -7779,313 +7700,311 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_frequencies</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_frequencies</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the sigma parameter in the paper mentioned when describing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>sigma</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_last_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is not present in the before mentioned paper but it is implemented in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_last_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>full_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>full_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>input_dim</code></b>
-            (<code>int</code>, default:
-                <code>64</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The so-called <em>dimension of the model</em>. Is the number of embeddings used to encode
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>input_dim</code></b>
+              (<code>int</code>, default:
+                  <code>64</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The so-called <em>dimension of the model</em>. Is the number of embeddings used to encode
 the categorical and/or continuous columns.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>kv_compression_factor</code></b>
-            (<code>float</code>, default:
-                <code>0.5</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>By default, the FTTransformer uses Linear Attention
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>kv_compression_factor</code></b>
+              (<code>float</code>, default:
+                  <code>0.5</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>By default, the FTTransformer uses Linear Attention
 (See <a href="https://arxiv.org/abs/2006.04768&gt;">Linformer: Self-Attention with Linear Complexity</a> ).
 The compression factor that will be used to reduce the input sequence
 length. If we denote the resulting sequence length as
 <span class="arithmatex">\(k = int(kv_{compression \space factor} \times s)\)</span>
 where <span class="arithmatex">\(s\)</span> is the input sequence length.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>kv_sharing</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the <span class="arithmatex">\(E\)</span> and <span class="arithmatex">\(F\)</span> projection matrices
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>kv_sharing</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the <span class="arithmatex">\(E\)</span> and <span class="arithmatex">\(F\)</span> projection matrices
 will share weights.  See <a href="https://arxiv.org/abs/2006.04768">Linformer: Self-Attention with Linear
 Complexity</a> for details</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_heads</code></b>
-            (<code>int</code>, default:
-                <code>8</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention heads per FTTransformer block</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_qkv_bias</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to use bias in the Q, K, and V
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_heads</code></b>
+              (<code>int</code>, default:
+                  <code>8</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention heads per FTTransformer block</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_qkv_bias</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to use bias in the Q, K, and V
 projection layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_blocks</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of FTTransformer blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the Linear-Attention layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the FeedForward network</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_factor</code></b>
-            (<code>float</code>, default:
-                <code>1.33</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Multiplicative factor applied to the first layer of the FF network in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_blocks</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of FTTransformer blocks</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the Linear-Attention layers</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the FeedForward network</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_factor</code></b>
+              (<code>float</code>, default:
+                  <code>1.33</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4, but they use 4/3
 in the paper.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>transformer_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;reglu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>transformer_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;reglu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the MLP. e.g:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If not provided no MLP on top of the final
 FTTransformer block will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers of the MLP. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>float with the dropout between the dense layers of the MLP.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm_last</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm_last</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_linear_first</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_linear_first</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Sequence of FTTransformer blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.mlp">mlp</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>MLP component in the model</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Sequence of FTTransformer blocks</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.mlp">mlp</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>MLP component in the model</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">FTTransformer</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -8097,9 +8016,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">194</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">194</span>
 <span class="normal">195</span>
 <span class="normal">196</span>
 <span class="normal">197</span>
@@ -8365,9 +8284,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
     <span class="k">else</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -8382,9 +8301,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 
 
 <h3 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -8393,11 +8312,11 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -8406,9 +8325,9 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 
 
 <h3 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.attention_weights" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+            <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -8417,14 +8336,14 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weights</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>List with the attention weights per block</p>
 <p>The shape of the attention weights is: <span class="arithmatex">\((N, H, F, k)\)</span>, where <span class="arithmatex">\(N\)</span> is
 the batch size, <span class="arithmatex">\(H\)</span> is the number of attention heads, <span class="arithmatex">\(F\)</span> is the
 number of features/columns and <span class="arithmatex">\(k\)</span> is the reduced sequence length or
 dimension, i.e. <span class="arithmatex">\(k = int(kv_{compression \space factor} \times s)\)</span></p>
-  </div>
+    </div>
 
 </div>
 
@@ -8434,8 +8353,7 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -8444,11 +8362,11 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 
 
 <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TabPerceiver</span>
+            <span class="doc doc-object-name doc-class-name">TabPerceiver</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TabPerceiver</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabPerceiver</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -8490,11 +8408,11 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+
 
-  
       <p>Defines an adaptation of a <a href="https://arxiv.org/abs/2103.03206">Perceiver</a>
  that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model
  or independently by itself.</p>
@@ -8508,134 +8426,133 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
  <a href="https://arxiv.org/abs/2108.09084">FastFormer</a> for tabular data.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name and number of unique values and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Categorical embeddings dropout. If <code>None</code>, it will default
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cat_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the categorical embeddings.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cat_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>add_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The two embedding sharing strategies are: 1) add the shared embeddings
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>add_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>frac_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>frac_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                <code>&#39;standard&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Method to use to embed the continuous features. Options are:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
+                  <code>&#39;standard&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -8643,355 +8560,353 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_frequencies</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_frequencies</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the sigma parameter in the paper mentioned when describing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>sigma</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_last_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is not present in the before mentioned paper but it is implemented in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_last_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>full_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>full_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>input_dim</code></b>
-            (<code>int</code>, default:
-                <code>32</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The so-called <em>dimension of the model</em>. Is the number of embeddings
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>input_dim</code></b>
+              (<code>int</code>, default:
+                  <code>32</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The so-called <em>dimension of the model</em>. Is the number of embeddings
 used to encode the categorical and/or continuous columns.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_cross_attns</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of times each perceiver block will cross attend to the input
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_cross_attns</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of times each perceiver block will cross attend to the input
 data (i.e. number of cross attention components per perceiver block).
 This should normally be 1. However, in the paper they describe some
 architectures (normally computer vision-related problems) where the
 Perceiver attends multiple times to the input array. Therefore, maybe
 multiple cross attention to the input array is also useful in some
 cases for tabular data <img alt="🤷" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/1f937.png" title=":shrug:" /> .</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_cross_attn_heads</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention heads for the cross attention component</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_latents</code></b>
-            (<code>int</code>, default:
-                <code>16</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of latents. This is the <span class="arithmatex">\(N\)</span> parameter in the paper. As
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_cross_attn_heads</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention heads for the cross attention component</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_latents</code></b>
+              (<code>int</code>, default:
+                  <code>16</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of latents. This is the <span class="arithmatex">\(N\)</span> parameter in the paper. As
 indicated in the paper, this number should be significantly lower
 than <span class="arithmatex">\(M\)</span> (the number of columns in the dataset). Setting <span class="arithmatex">\(N\)</span> closer
 to <span class="arithmatex">\(M\)</span> defies the main purpose of the Perceiver, which is to overcome
 the transformer quadratic bottleneck</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>latent_dim</code></b>
-            (<code>int</code>, default:
-                <code>128</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Latent dimension.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_latent_heads</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention heads per Latent Transformer</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_latent_blocks</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of transformer encoder blocks (normalised MHA + normalised FF)
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>latent_dim</code></b>
+              (<code>int</code>, default:
+                  <code>128</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Latent dimension.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_latent_heads</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention heads per Latent Transformer</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_latent_blocks</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of transformer encoder blocks (normalised MHA + normalised FF)
 per Latent Transformer</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_perceiver_blocks</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of Perceiver blocks defined as [Cross Attention + Latent
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_perceiver_blocks</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of Perceiver blocks defined as [Cross Attention + Latent
 Transformer]</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_weights</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the weights will be shared between Perceiver
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_weights</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the weights will be shared between Perceiver
 blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the Multi-Head Attention layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the FeedForward network</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_factor</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Multiplicative factor applied to the first layer of the FF network in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the Multi-Head Attention layers</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the FeedForward network</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_factor</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>transformer_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;geglu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>transformer_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;geglu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the MLP. e.g:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If not provided no MLP on top of the final
 Transformer block will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers of the MLP. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>float with the dropout between the dense layers of the MLP.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm_last</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm_last</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_linear_first</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_linear_first</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.ModuleDict">ModuleDict</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>ModuleDict with the Perceiver blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.latents">latents</span></code></b>
-            (<code><span title="torch.nn.Parameter">Parameter</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Latents that will be used for prediction</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.mlp">mlp</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>MLP component in the model</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.ModuleDict">ModuleDict</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>ModuleDict with the Perceiver blocks</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.latents">latents</span></code></b>
+              (<code><span title="torch.nn.Parameter">Parameter</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Latents that will be used for prediction</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.mlp">mlp</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>MLP component in the model</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabPerceiver</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -9005,9 +8920,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">216</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">216</span>
 <span class="normal">217</span>
 <span class="normal">218</span>
 <span class="normal">219</span>
@@ -9257,9 +9172,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
     <span class="k">else</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -9274,9 +9189,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
 
 
 <h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -9285,11 +9200,11 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -9298,9 +9213,9 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.
 
 
 <h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+            <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -9309,8 +9224,8 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weights</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>List with the attention weights. If the weights are not shared
 between perceiver blocks each element of the list will be a list
 itself containing the Cross Attention and Latent Transformer
@@ -9328,7 +9243,7 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.
 heads, <span class="arithmatex">\(L\)</span> is the number of Latents, <span class="arithmatex">\(F\)</span> is the number of
 features/columns in the dataset and <span class="arithmatex">\(T\)</span> is the number of Latent
 Attention heads</p>
-  </div>
+    </div>
 
 </div>
 
@@ -9338,8 +9253,7 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -9348,11 +9262,11 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.
 
 
 <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TabFastFormer</span>
+            <span class="doc doc-object-name doc-class-name">TabFastFormer</span>
 
 
 <a href="#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TabFastFormer</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabFastFormer</span><span class="p">(</span>
     <span class="n">column_idx</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -9391,11 +9305,11 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models.tabular._base_tabular_model.BaseTabularModelWithAttention">BaseTabularModelWithAttention</span></code></p>
+
 
-  
       <p>Defines an adaptation of a <a href="https://arxiv.org/abs/2108.09084">FastFormer</a>
 that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model
 or independently by itself.</p>
@@ -9409,134 +9323,133 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
  <a href="https://arxiv.org/abs/2108.09084">FastFormer</a> for tabular data.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>column_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the index of the columns that will be passed through
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>column_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_input</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name and number of unique values and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_input</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Categorical embeddings dropout. If <code>None</code>, it will default
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cat_bias</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if bias will be used for the categorical embeddings.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cat_bias</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the categorical embeddings, if any. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>add_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The two embedding sharing strategies are: 1) add the shared embeddings
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>add_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>frac_shared_embed</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>frac_shared_embed</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the numeric (aka continuous) columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_norm_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Type of normalization layer applied to the continuous features.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the numeric (aka continuous) columns</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_norm_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_continuous_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                <code>&#39;standard&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Method to use to embed the continuous features. Options are:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_continuous_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
+                  <code>&#39;standard&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -9544,321 +9457,319 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_embed_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the continuous embeddings if any. Currently
-<em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_embed_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the continuous embeddings if any. Currently
+<em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_frequencies</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_frequencies</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This is the sigma parameter in the paper mentioned when describing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>sigma</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_last_layer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This parameter is not present in the before mentioned paper but it is implemented in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_last_layer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>full_embed_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>full_embed_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>input_dim</code></b>
-            (<code>int</code>, default:
-                <code>32</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>The so-called <em>dimension of the model</em>. Is the number of
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>input_dim</code></b>
+              (<code>int</code>, default:
+                  <code>32</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>The so-called <em>dimension of the model</em>. Is the number of
 embeddings used to encode the categorical and/or continuous columns</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_heads</code></b>
-            (<code>int</code>, default:
-                <code>8</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention heads per FastFormer block</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_bias</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to use bias in the Q, K, and V
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_heads</code></b>
+              (<code>int</code>, default:
+                  <code>8</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention heads per FastFormer block</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_bias</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to use bias in the Q, K, and V
 projection layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_blocks</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of FastFormer blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the Additive Attention layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the FeedForward network</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_factor</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Multiplicative factor applied to the first layer of the FF network in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_blocks</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of FastFormer blocks</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the Additive Attention layers</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.2</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the FeedForward network</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_factor</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_qv_weights</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Following the paper, this is a boolean indicating if the Value (<span class="arithmatex">\(V\)</span>) and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_qv_weights</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Following the paper, this is a boolean indicating if the Value (<span class="arithmatex">\(V\)</span>) and
 the Query (<span class="arithmatex">\(Q\)</span>) transformation parameters will be shared.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>share_weights</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>In addition to sharing the <span class="arithmatex">\(V\)</span> and <span class="arithmatex">\(Q\)</span> transformation parameters, the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>share_weights</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>In addition to sharing the <span class="arithmatex">\(V\)</span> and <span class="arithmatex">\(Q\)</span> transformation parameters, the
 parameters across different Fastformer layers can also be shared.
 Please, see
 <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> for
 details</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>transformer_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>transformer_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>MLP hidden dimensions. If not provided no MLP on top of the final
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>MLP hidden dimensions. If not provided no MLP on top of the final
 FTTransformer block will be used</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the MLP. e.g:
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If not provided no MLP on top of the final
 Transformer block will be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers of the MLP. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>float with the dropout between the dense layers of the MLP.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_batchnorm_last</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_batchnorm_last</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>mlp_linear_first</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>mlp_linear_first</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Sequence of FasFormer blocks.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.mlp">mlp</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>MLP component in the model</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Sequence of FasFormer blocks.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.mlp">mlp</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>MLP component in the model</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabFastFormer</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">4</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -9870,9 +9781,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">200</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">200</span>
 <span class="normal">201</span>
 <span class="normal">202</span>
 <span class="normal">203</span>
@@ -10160,9 +10071,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
     <span class="k">else</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -10177,9 +10088,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 
 
 <h3 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -10188,11 +10099,11 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -10201,9 +10112,9 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 
 
 <h3 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.attention_weights" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+            <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -10212,15 +10123,15 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weights</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>List with the attention weights. Each element of the list is a
 tuple where the first and second elements are the <span class="arithmatex">\(\alpha\)</span>
 and <span class="arithmatex">\(\beta\)</span> attention weights in the paper.</p>
 <p>The shape of the attention weights is <span class="arithmatex">\((N, H, F)\)</span> where <span class="arithmatex">\(N\)</span> is the
 batch size, <span class="arithmatex">\(H\)</span> is the number of attention heads and <span class="arithmatex">\(F\)</span> is the
 number of features/columns in the dataset</p>
-  </div>
+    </div>
 
 </div>
 
@@ -10230,8 +10141,7 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -10240,11 +10150,11 @@ <h3 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 
 
 <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">BasicRNN</span>
+            <span class="doc doc-object-name doc-class-name">BasicRNN</span>
 
 
 <a href="#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">BasicRNN</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">BasicRNN</span><span class="p">(</span>
     <span class="n">vocab_size</span><span class="p">,</span>
     <span class="n">embed_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">embed_matrix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -10265,11 +10175,11 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
+
 
-  
       <p>Standard text classifier/regressor comprised by a stack of RNNs
 (LSTMs or GRUs) that can be used as the <code>deeptext</code> component of a Wide &amp;
 Deep model or independently by itself.</p>
@@ -10277,223 +10187,220 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
 dense layers on top of the stack of RNNs</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>vocab_size</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Number of words in the vocabulary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_dim</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dimension of the word embeddings if non-pretained word vectors are
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>vocab_size</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Number of words in the vocabulary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_dim</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dimension of the word embeddings if non-pretained word vectors are
 used</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_matrix</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Pretrained word embeddings</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_trainable</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the pretrained embeddings are trainable</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>rnn_type</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[<span title="torch.lstm">lstm</span>, gru]</code>, default:
-                <code>&#39;lstm&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>String indicating the type of RNN to use. One of <em>'lstm'</em> or <em>'gru'</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>hidden_dim</code></b>
-            (<code>int</code>, default:
-                <code>64</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Hidden dim of the RNN</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_layers</code></b>
-            (<code>int</code>, default:
-                <code>3</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of recurrent layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>rnn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.0</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for each RNN layer except the last layer</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>bidirectional</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the staked RNNs are bidirectional</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_hidden_state</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether to use the final hidden state or the RNN's
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_matrix</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Pretrained word embeddings</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_trainable</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the pretrained embeddings are trainable</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>rnn_type</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[<span title="torch.lstm">lstm</span>, gru]</code>, default:
+                  <code>&#39;lstm&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>String indicating the type of RNN to use. One of <em>'lstm'</em> or <em>'gru'</em></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>hidden_dim</code></b>
+              (<code>int</code>, default:
+                  <code>64</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Hidden dim of the RNN</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_layers</code></b>
+              (<code>int</code>, default:
+                  <code>3</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of recurrent layers</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>rnn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.0</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for each RNN layer except the last layer</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>bidirectional</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the staked RNNs are bidirectional</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_hidden_state</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether to use the final hidden state or the RNN's
 output as predicting features. Typically the former is used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>padding_idx</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>index of the padding token in the padded-tokenised sequences. The
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>padding_idx</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>index of the padding token in the padded-tokenised sequences. The
 <code>TextPreprocessor</code> class within this library uses fastai's tokenizer
 where the token index 0 is reserved for the <em>'unknown'</em> word token.
 Therefore, the default value is set to 1.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers in the head. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers in the head. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout of the dense layers in the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to include batch normalization in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout of the dense layers in the head</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to include batch normalization in
 the dense layers that form the <em>'rnn_mlp'</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm_last</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to apply batch normalization to the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm_last</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to apply batch normalization to the
 last of the dense layers in the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_linear_first</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_linear_first</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.word_embed">word_embed</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>word embedding matrix</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.rnn">rnn</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Stack of RNNs</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.rnn_mlp">rnn_mlp</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Stack of dense layers on top of the RNN. This will only exists if
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.word_embed">word_embed</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>word embedding matrix</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.rnn">rnn</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Stack of RNNs</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.rnn_mlp">rnn_mlp</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Stack of dense layers on top of the RNN. This will only exists if
 <code>head_layers_dim</code> is not None</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">BasicRNN</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_text</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span><span class="mi">1</span><span class="p">]),</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">4</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -10501,9 +10408,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 95</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 95</span>
 <span class="normal"> 96</span>
 <span class="normal"> 97</span>
 <span class="normal"> 98</span>
@@ -10721,9 +10628,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
         <span class="c1"># simple hack to add readability in the forward pass</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -10738,9 +10645,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
 
 
 <h3 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -10749,11 +10656,11 @@ <h3 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim" class="
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -10763,8 +10670,7 @@ <h3 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim" class="
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -10773,11 +10679,11 @@ <h3 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim" class="
 
 
 <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">AttentiveRNN</span>
+            <span class="doc doc-object-name doc-class-name">AttentiveRNN</span>
 
 
 <a href="#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">AttentiveRNN</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">AttentiveRNN</span><span class="p">(</span>
     <span class="n">vocab_size</span><span class="p">,</span>
     <span class="n">embed_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">embed_matrix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -10800,11 +10706,11 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" href="#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN">BasicRNN</a></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" href="#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN">BasicRNN</a></code></p>
+
 
-  
       <p>Text classifier/regressor comprised by a stack of RNNs
 (LSTMs or GRUs) plus an attention layer. This model can be used as the
 <code>deeptext</code> component of a Wide &amp; Deep model or independently by
@@ -10813,245 +10719,242 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
 layers on top of attention layer</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>vocab_size</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Number of words in the vocabulary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_dim</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dimension of the word embeddings if non-pretained word vectors are
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>vocab_size</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Number of words in the vocabulary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_dim</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dimension of the word embeddings if non-pretained word vectors are
 used</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_matrix</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Pretrained word embeddings</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_trainable</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the pretrained embeddings are trainable</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>rnn_type</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>, default:
-                <code>&#39;lstm&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>String indicating the type of RNN to use. One of <em>'lstm'</em> or <em>'gru'</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>hidden_dim</code></b>
-            (<code>int</code>, default:
-                <code>64</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Hidden dim of the RNN</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_layers</code></b>
-            (<code>int</code>, default:
-                <code>3</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of recurrent layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>rnn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout for each RNN layer except the last layer</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>bidirectional</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the staked RNNs are bidirectional</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_hidden_state</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether to use the final hidden state or the RNN's
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_matrix</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Pretrained word embeddings</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_trainable</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the pretrained embeddings are trainable</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>rnn_type</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>, default:
+                  <code>&#39;lstm&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>String indicating the type of RNN to use. One of <em>'lstm'</em> or <em>'gru'</em></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>hidden_dim</code></b>
+              (<code>int</code>, default:
+                  <code>64</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Hidden dim of the RNN</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_layers</code></b>
+              (<code>int</code>, default:
+                  <code>3</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of recurrent layers</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>rnn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout for each RNN layer except the last layer</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>bidirectional</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the staked RNNs are bidirectional</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_hidden_state</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether to use the final hidden state or the RNN's
 output as predicting features. Typically the former is used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>padding_idx</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>index of the padding token in the padded-tokenised sequences. The
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>padding_idx</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>index of the padding token in the padded-tokenised sequences. The
 <code>TextPreprocessor</code> class within this library uses fastai's
 tokenizer where the token index 0 is reserved for the <em>'unknown'</em>
 word token. Therefore, the default value is set to 1.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_concatenate</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the input to the attention mechanism will be the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_concatenate</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the input to the attention mechanism will be the
 output of the RNN or the output of the RNN concatenated with the last
 hidden state.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Internal dropout for the attention mechanism</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers in the head. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Internal dropout for the attention mechanism</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers in the head. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout of the dense layers in the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to include batch normalization in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout of the dense layers in the head</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to include batch normalization in
 the dense layers that form the <em>'rnn_mlp'</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm_last</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to apply batch normalization to the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm_last</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to apply batch normalization to the
 last of the dense layers in the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_linear_first</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_linear_first</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.word_embed">word_embed</span></code></b>
-            (<code><span title="nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>word embedding matrix</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.rnn">rnn</span></code></b>
-            (<code><span title="nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Stack of RNNs</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.rnn_mlp">rnn_mlp</span></code></b>
-            (<code><span title="nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Stack of dense layers on top of the RNN. This will only exists if
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.word_embed">word_embed</span></code></b>
+              (<code><span title="nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>word embedding matrix</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.rnn">rnn</span></code></b>
+              (<code><span title="nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Stack of RNNs</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.rnn_mlp">rnn_mlp</span></code></b>
+              (<code><span title="nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Stack of dense layers on top of the RNN. This will only exists if
 <code>head_layers_dim</code> is not <code>None</code></p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">AttentiveRNN</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_text</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span><span class="mi">1</span><span class="p">]),</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">4</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -11059,9 +10962,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 93</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 93</span>
 <span class="normal"> 94</span>
 <span class="normal"> 95</span>
 <span class="normal"> 96</span>
@@ -11199,9 +11102,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             <span class="n">head_linear_first</span><span class="p">,</span>
         <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -11216,9 +11119,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
 
 
 <h3 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_weights" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+            <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -11227,12 +11130,12 @@ <h3 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_w
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weights</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>List with the attention weights</p>
 <p>The shape of the attention weights is <span class="arithmatex">\((N, S)\)</span>, where <span class="arithmatex">\(N\)</span> is the batch
 size and <span class="arithmatex">\(S\)</span> is the length of the sequence</p>
-  </div>
+    </div>
 
 </div>
 
@@ -11242,8 +11145,7 @@ <h3 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_w
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -11252,11 +11154,11 @@ <h3 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_w
 
 
 <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">StackedAttentiveRNN</span>
+            <span class="doc doc-object-name doc-class-name">StackedAttentiveRNN</span>
 
 
 <a href="#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">StackedAttentiveRNN</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">StackedAttentiveRNN</span><span class="p">(</span>
     <span class="n">vocab_size</span><span class="p">,</span>
     <span class="n">embed_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">embed_matrix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -11278,11 +11180,11 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
+
 
-  
       <p>Text classifier/regressor comprised by a stack of blocks:
 <code>[RNN + Attention]</code>. This can be used as the <code>deeptext</code> component of a
 Wide &amp; Deep model or independently by itself.</p>
@@ -11290,236 +11192,233 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 dense layers on top of the attentiob blocks</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>vocab_size</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Number of words in the vocabulary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_dim</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dimension of the word embeddings if non-pretained word vectors are
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>vocab_size</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Number of words in the vocabulary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_dim</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dimension of the word embeddings if non-pretained word vectors are
 used</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_matrix</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Pretrained word embeddings</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embed_trainable</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the pretrained embeddings are trainable</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>rnn_type</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>, default:
-                <code>&#39;lstm&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>hidden_dim</code></b>
-            (<code>int</code>, default:
-                <code>64</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Hidden dim of the RNN</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>bidirectional</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the staked RNNs are bidirectional</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>padding_idx</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>index of the padding token in the padded-tokenised sequences. The
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_matrix</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Pretrained word embeddings</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embed_trainable</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the pretrained embeddings are trainable</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>rnn_type</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>, default:
+                  <code>&#39;lstm&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>hidden_dim</code></b>
+              (<code>int</code>, default:
+                  <code>64</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Hidden dim of the RNN</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>bidirectional</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the staked RNNs are bidirectional</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>padding_idx</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>index of the padding token in the padded-tokenised sequences. The
 <code>TextPreprocessor</code> class within this library uses fastai's
 tokenizer where the token index 0 is reserved for the <em>'unknown'</em>
 word token. Therefore, the default value is set to 1.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_blocks</code></b>
-            (<code>int</code>, default:
-                <code>3</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention blocks. Each block is comprised by an RNN and a
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_blocks</code></b>
+              (<code>int</code>, default:
+                  <code>3</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention blocks. Each block is comprised by an RNN and a
 Context Attention Encoder</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_concatenate</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the input to the attention mechanism will be the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_concatenate</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the input to the attention mechanism will be the
 output of the RNN or the output of the RNN concatenated with the last
 hidden state or simply</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Internal dropout for the attention mechanism</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_addnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the output of each block will be added to the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Internal dropout for the attention mechanism</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>with_addnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the output of each block will be added to the
 input and normalised</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers in the head. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers in the head. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout of the dense layers in the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to include batch normalization in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout of the dense layers in the head</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to include batch normalization in
 the dense layers that form the <em>'rnn_mlp'</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm_last</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to apply batch normalization to the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm_last</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to apply batch normalization to the
 last of the dense layers in the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_linear_first</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_linear_first</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.word_embed">word_embed</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>word embedding matrix</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.rnn">rnn</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Stack of RNNs</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.rnn_mlp">rnn_mlp</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Stack of dense layers on top of the RNN. This will only exists if
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.word_embed">word_embed</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>word embedding matrix</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.rnn">rnn</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Stack of RNNs</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.rnn_mlp">rnn_mlp</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Stack of dense layers on top of the RNN. This will only exists if
 <code>head_layers_dim</code> is not <code>None</code></p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">StackedAttentiveRNN</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_text</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span><span class="mi">1</span><span class="p">]),</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">4</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -11527,9 +11426,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">102</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">102</span>
 <span class="normal">103</span>
 <span class="normal">104</span>
 <span class="normal">105</span>
@@ -11773,9 +11672,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
         <span class="c1"># simple hack to add readability in the forward pass</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -11790,9 +11689,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 
 
 <h3 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -11801,11 +11700,11 @@ <h3 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -11814,9 +11713,9 @@ <h3 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 
 
 <h3 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.attention_weights" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+            <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -11825,12 +11724,12 @@ <h3 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weights</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>List with the attention weights per block</p>
 <p>The shape of the attention weights is <span class="arithmatex">\((N, S)\)</span> Where <span class="arithmatex">\(N\)</span> is the batch
 size and <span class="arithmatex">\(S\)</span> is the length of the sequence</p>
-  </div>
+    </div>
 
 </div>
 
@@ -11840,8 +11739,7 @@ <h3 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -11850,11 +11748,11 @@ <h3 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 
 
 <h2 id="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Transformer</span>
+            <span class="doc doc-object-name doc-class-name">Transformer</span>
 
 
 <a href="#pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Transformer</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Transformer</span><span class="p">(</span>
     <span class="n">vocab_size</span><span class="p">,</span>
     <span class="n">seq_length</span><span class="p">,</span>
     <span class="n">input_dim</span><span class="p">,</span>
@@ -11875,11 +11773,11 @@ <h2 id="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Basic Encoder-Only Transformer Model for text classification/regression.
 As all other models in the library this model can be used as the
 <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p>
@@ -11891,183 +11789,180 @@ <h2 id="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer
 to integrate the library wit Huggingface in the (hopefully) near future</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>vocab_size</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Number of words in the vocabulary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>input_dim</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dimension of the token embeddings</p>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>vocab_size</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Number of words in the vocabulary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>input_dim</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dimension of the token embeddings</p>
 <p>Param aliases: <code>embed_dim</code>, <code>d_model</code>. <br/></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>seq_length</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Input sequence length</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_heads</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Number of attention heads per Transformer block</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_blocks</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Number of Transformer blocks</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>attn_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the Multi-Head Attention layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout that will be applied to the FeedForward network</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ff_factor</code></b>
-            (<code>int</code>, default:
-                <code>4</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Multiplicative factor applied to the first layer of the FF network in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>seq_length</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Input sequence length</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_heads</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Number of attention heads per Transformer block</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_blocks</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Number of Transformer blocks</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>attn_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the Multi-Head Attention layers</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout that will be applied to the FeedForward network</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>ff_factor</code></b>
+              (<code>int</code>, default:
+                  <code>4</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;gelu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;gelu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>padding_idx</code></b>
-            (<code>int</code>, default:
-                <code>0</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>index of the padding token in the padded-tokenised sequences.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_cls_token</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if a <code>'[CLS]'</code> token is included in the tokenized
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>padding_idx</code></b>
+              (<code>int</code>, default:
+                  <code>0</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>index of the padding token in the padded-tokenised sequences.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>with_cls_token</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if a <code>'[CLS]'</code> token is included in the tokenized
 sequences. If present, the final hidden state corresponding to this
 token is used as the aggregated representation for classification and
 regression tasks. <strong>NOTE</strong>: if included in the tokenized sequences it
 must be inserted as the first token in the sequences.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_pos_encoding</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if positional encoding will be used</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pos_encoding_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Positional encoding dropout</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pos_encoder</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>This model uses by default a standard positional encoding approach.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>with_pos_encoding</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if positional encoding will be used</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pos_encoding_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Positional encoding dropout</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pos_encoder</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>This model uses by default a standard positional encoding approach.
 However, any custom positional encoder can also be used and pass to
 the Transformer model via the 'pos_encoder' parameter</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer.embedding">embedding</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Standard token embedding layer</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer.pos_encoder">pos_encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Positional Encoder</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer.encoder">encoder</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Sequence of Transformer blocks</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer.embedding">embedding</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Standard token embedding layer</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer.pos_encoder">pos_encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Positional Encoder</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer.encoder">encoder</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Sequence of Transformer blocks</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">Transformer</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_text</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span><span class="mi">1</span><span class="p">]),</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">4</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -12075,9 +11970,9 @@ <h2 id="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/text/miscellaneous/basic_transformer.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 85</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/text/miscellaneous/basic_transformer.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 85</span>
 <span class="normal"> 86</span>
 <span class="normal"> 87</span>
 <span class="normal"> 88</span>
@@ -12219,12 +12114,11 @@ <h2 id="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer
             <span class="p">),</span>
         <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -12235,10 +12129,10 @@ <h2 id="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -12247,11 +12141,11 @@ <h2 id="pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer
 
 
 <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">HFModel</span>
+            <span class="doc doc-object-name doc-class-name">HFModel</span>
 
 
 <a href="#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">HFModel</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">HFModel</span><span class="p">(</span>
     <span class="n">model_name</span><span class="p">,</span>
     <span class="n">use_cls_token</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
     <span class="n">trainable_parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -12266,11 +12160,11 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
+
 
-  
       <p>This class is a wrapper around the Hugging Face transformers library. It
 can be used as the text component of a Wide &amp; Deep model or independently
 by itself.</p>
@@ -12281,143 +12175,140 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
 work best for these tasks.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>model_name</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>The model name from the transformers library e.g. 'bert-base-uncased'.
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>model_name</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>The model name from the transformers library e.g. 'bert-base-uncased'.
 Currently supported models are those from the families: BERT, RoBERTa,
 DistilBERT, ALBERT and ELECTRA.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_cls_token</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether to use the [CLS] token or the mean of the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_cls_token</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether to use the [CLS] token or the mean of the
 sequence of hidden states as the sentence embedding</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>trainable_parameters</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the names of the model parameters that will be trained. If
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>trainable_parameters</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the names of the model parameters that will be trained. If
 None, none of the parameters will be trainable</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers in the head. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers in the head. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout of the dense layers in the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to include batch normalization in the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout of the dense layers in the head</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to include batch normalization in the
 dense layers that form the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm_last</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to apply batch normalization to the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm_last</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to apply batch normalization to the
 last of the dense layers in the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_linear_first</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_linear_first</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If True, it will print information about the model</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p>Additional kwargs to be passed to the model</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If True, it will print information about the model</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p>Additional kwargs to be passed to the model</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.head">head</span></code></b>
-            (<code><span title="nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Stack of dense layers on top of the transformer. This will only exists
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.head">head</span></code></b>
+              (<code><span title="nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Stack of dense layers on top of the transformer. This will only exists
 if <code>head_layers_dim</code> is not None</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">HFModel</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_text</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span><span class="mi">1</span><span class="p">]),</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">4</span><span class="p">)),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">long</span><span class="p">()</span>
@@ -12425,9 +12316,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 77</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 77</span>
 <span class="normal"> 78</span>
 <span class="normal"> 79</span>
 <span class="normal"> 80</span>
@@ -12545,9 +12436,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             <span class="n">head_linear_first</span><span class="p">,</span>
         <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -12562,9 +12453,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
 
 
 <h3 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">attention_weight</span>
+            <span class="doc doc-object-name doc-attribute-name">attention_weight</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -12573,15 +12464,15 @@ <h3 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.a
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weight</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Returns the attention weights if the model was created with the
 output_attention_weights=True argument. If not, it will raise an
 AttributeError.</p>
 <p>The shape of the attention weights is <span class="arithmatex">\((N, H, F, F)\)</span>, where <span class="arithmatex">\(N\)</span> is the
 batch size, <span class="arithmatex">\(H\)</span> is the number of attention heads and <span class="arithmatex">\(F\)</span> is the
 sequence length.</p>
-  </div>
+    </div>
 
 </div>
 
@@ -12591,8 +12482,7 @@ <h3 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.a
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -12601,11 +12491,11 @@ <h3 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.a
 
 
 <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Vision</span>
+            <span class="doc doc-object-name doc-class-name">Vision</span>
 
 
 <a href="#pytorch_widedeep.models.image.vision.Vision" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Vision</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Vision</span><span class="p">(</span>
     <span class="n">pretrained_model_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">n_trainable</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">trainable_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -12621,11 +12511,11 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
+
 
-  
       <p>Defines a standard image classifier/regressor using a pretrained
 network or a sequence of convolution layers that can be used as the
 <code>deepimage</code> component of a Wide &amp; Deep model or independently by
@@ -12639,17 +12529,16 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
  through this class</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>pretrained_model_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[str, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[str, <span title="pytorch_widedeep.wdtypes.WeightsEnum">WeightsEnum</span>]]]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>pretrained_model_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[str, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[str, <span title="pytorch_widedeep.wdtypes.WeightsEnum">WeightsEnum</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Name of the pretrained model. Should be a variant of the following
+          –
+          <div class="doc-md-description">
+            <p>Name of the pretrained model. Should be a variant of the following
 architectures: <em>'resnet'</em>, <em>'shufflenet'</em>, <em>'resnext'</em>,
 <em>'wide_resnet'</em>, <em>'regnet'</em>, <em>'densenet'</em>, <em>'mobilenetv3'</em>,
 <em>'mobilenetv2'</em>, <em>'mnasnet'</em>, <em>'efficientnet'</em> and <em>'squeezenet'</em>. if
@@ -12659,154 +12548,152 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
 also be dictionary with the name of the model and the weights (e.g.
 <code>{'resnet50': ResNet50_Weights.DEFAULT}</code> or
 <code>{'resnet50': "IMAGENET1K_V2"}</code>). <br/> Aliased as <code>pretrained_model_name</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_trainable</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Number of trainable layers starting from the layer closer to the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_trainable</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Number of trainable layers starting from the layer closer to the
 output neuron(s). Note that this number DOES NOT take into account
 the so-called <em>'head'</em> which is ALWAYS trainable. If
 <code>trainable_params</code> is not None this parameter will be ignored</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>trainable_params</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of strings containing the names (or substring within the name) of
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>trainable_params</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of strings containing the names (or substring within the name) of
 the parameters that will be trained. For example, if we use a
 <em>'resnet18'</em> pretrained model and we set <code>trainable_params =
 ['layer4']</code> only the parameters of <em>'layer4'</em> of the network
 (and the head, as mentioned before) will be trained. Note that
 setting this or the previous parameter involves some knowledge of
 the architecture used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>channel_sizes</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
-                <code>[64, 128, 256, 512]</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of integers with the channel sizes of a CNN in case we choose not
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>channel_sizes</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+                  <code>[64, 128, 256, 512]</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of integers with the channel sizes of a CNN in case we choose not
 to use a pretrained model</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>kernel_sizes</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>[7, 3, 3, 3]</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of integers with the kernel sizes of a CNN in case we choose not
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>kernel_sizes</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>[7, 3, 3, 3]</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of integers with the kernel sizes of a CNN in case we choose not
 to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>strides</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>[2, 1, 1, 1]</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List of integers with the stride sizes of a CNN in case we choose not
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>strides</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>[2, 1, 1, 1]</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List of integers with the stride sizes of a CNN in case we choose not
 to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per dense layer in the head. e.g: <em>[64,32]</em></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers in the head. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per dense layer in the head. e.g: <em>[64,32]</em></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers in the head. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>float indicating the dropout between the dense layers.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>float indicating the dropout between the dense layers.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm_last</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not batch normalization will be applied
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm_last</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_linear_first</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_linear_first</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.image.vision.Vision.features">features</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>The pretrained model or Standard CNN plus the optional head</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.image.vision.Vision.features">features</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>The pretrained model or Standard CNN plus the optional head</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">Vision</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_img</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">224</span><span class="p">,</span><span class="mi">224</span><span class="p">))</span>
@@ -12814,9 +12701,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_img</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/image/vision.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">125</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/image/vision.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">125</span>
 <span class="normal">126</span>
 <span class="normal">127</span>
 <span class="normal">128</span>
@@ -12916,9 +12803,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span><span class="p">,</span>
         <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -12933,9 +12820,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
 
 
 <h3 id="pytorch_widedeep.models.image.vision.Vision.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -12944,11 +12831,11 @@ <h3 id="pytorch_widedeep.models.image.vision.Vision.output_dim" class="doc doc-h
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>The output dimension of the model. This is a required property
 neccesary to build the <code>WideDeep</code> class</p>
-  </div>
+    </div>
 
 </div>
 
@@ -12958,8 +12845,7 @@ <h3 id="pytorch_widedeep.models.image.vision.Vision.output_dim" class="doc doc-h
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -12968,11 +12854,11 @@ <h3 id="pytorch_widedeep.models.image.vision.Vision.output_dim" class="doc doc-h
 
 
 <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">ModelFuser</span>
+            <span class="doc doc-object-name doc-class-name">ModelFuser</span>
 
 
 <a href="#pytorch_widedeep.models.model_fusion.ModelFuser" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">ModelFuser</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ModelFuser</span><span class="p">(</span>
     <span class="n">models</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">fusion_method</span><span class="p">,</span>
@@ -12987,148 +12873,145 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
+
 
-  
       <p>This class is a wrapper around a list of models that are associated to the
 different text and/or image columns (and datasets) The class is designed
 to 'fuse' the models using a variety of methods.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>models</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>)
-        –
-        <div class="doc-md-description">
-          <p>List of models whose outputs will be fused</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>fusion_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[concatenate, mean, max, sum, mult, <span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[concatenate, mean, max, sum, mult, <span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span>]]]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Method to fuse the output of the models. It can be one of
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>models</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>)
+          –
+          <div class="doc-md-description">
+            <p>List of models whose outputs will be fused</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>fusion_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[concatenate, mean, max, sum, mult, <span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[concatenate, mean, max, sum, mult, <span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span>]]]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Method to fuse the output of the models. It can be one of
 ['concatenate', 'mean', 'max', 'sum', 'mult', 'head'] or a list of
 those. If a list is provided the output of the models will be fused
 using all the methods in the list and the final output will be the
 concatenation of the outputs of each method</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>projection_method</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[min, max, mean]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>If the fusion_method is not 'concatenate', this parameter will
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>projection_method</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[min, max, mean]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>If the fusion_method is not 'concatenate', this parameter will
 determine how to project the output of the models to a common
 dimension. It can be one of ['min', 'max', 'mean']. Default is None</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>custom_head</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="torch.nn.Module">Module</span>]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Custom head to be used to fuse the output of the models. If provided,
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>custom_head</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="torch.nn.Module">Module</span>]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Custom head to be used to fuse the output of the models. If provided,
 this will take precedence over head_hidden_dims. Also, if
 provided, 'projection_method' will be ignored.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the number of neurons per layer in the custom head. If
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the number of neurons per layer in the custom head. If
 custom_head is provided, this parameter will be ignored</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_activation</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function to be used in the custom head. Default is None</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_dropout</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout to be used in the custom head. Default is None</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Whether to use batchnorm in the custom head. Default is None</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm_last</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Whether or not batch normalization will be applied to the last of the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_activation</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function to be used in the custom head. Default is None</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_dropout</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout to be used in the custom head. Default is None</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Whether to use batchnorm in the custom head. Default is None</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm_last</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Whether or not batch normalization will be applied to the last of the
 dense layers</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_linear_first</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_linear_first</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span></code></b>
-            (<code><span title="torch.nn.Module">Module</span> or <span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Custom head to be used to fuse the output of the models. If
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span></code></b>
+              (<code><span title="torch.nn.Module">Module</span> or <span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Custom head to be used to fuse the output of the models. If
 custom_head is provided, this will take precedence over
 head_hidden_dims</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">TextPreprocessor</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">BasicRNN</span><span class="p">,</span> <span class="n">ModelFuser</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
@@ -13170,9 +13053,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">fused_model</span><span class="p">([</span><span class="n">X_text1_tnsr</span><span class="p">,</span> <span class="n">X_text2_tnsr</span><span class="p">])</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/model_fusion.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">102</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/model_fusion.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">102</span>
 <span class="normal">103</span>
 <span class="normal">104</span>
 <span class="normal">105</span>
@@ -13326,9 +13209,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
                 <span class="p">),</span>
             <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
+
 
-  
 
   <div class="doc doc-children">
 
@@ -13343,9 +13226,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
 
 
 <h3 id="pytorch_widedeep.models.model_fusion.ModelFuser.output_dim" class="doc doc-heading">
-          <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
 
-  
   <span class="doc doc-labels">
       <small class="doc doc-label doc-label-property"><code>property</code></small>
   </span>
@@ -13354,35 +13237,33 @@ <h3 id="pytorch_widedeep.models.model_fusion.ModelFuser.output_dim" class="doc d
 <div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Returns the output dimension of the model.</p>
-  </div>
+    </div>
 
 </div>
 
 
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.models.model_fusion.ModelFuser.project" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">project</span>
+            <span class="doc doc-object-name doc-function-name">project</span>
 
 
 <a href="#pytorch_widedeep.models.model_fusion.ModelFuser.project" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">project</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">project</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Projects the output of the models to a common dimension.</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/models/model_fusion.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">223</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/models/model_fusion.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">223</span>
 <span class="normal">224</span>
 <span class="normal">225</span>
 <span class="normal">226</span>
@@ -13442,8 +13323,8 @@ <h3 id="pytorch_widedeep.models.model_fusion.ModelFuser.project" class="doc doc-
 
     <span class="k">return</span> <span class="n">x_proj</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -13451,8 +13332,7 @@ <h3 id="pytorch_widedeep.models.model_fusion.ModelFuser.project" class="doc doc-
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -13461,11 +13341,11 @@ <h3 id="pytorch_widedeep.models.model_fusion.ModelFuser.project" class="doc doc-
 
 
 <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">WideDeep</span>
+            <span class="doc doc-object-name doc-class-name">WideDeep</span>
 
 
 <a href="#pytorch_widedeep.models.wide_deep.WideDeep" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">WideDeep</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">WideDeep</span><span class="p">(</span>
     <span class="n">wide</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">deeptabular</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">deeptext</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -13480,16 +13360,14 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
     <span class="n">enforce_positive</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
     <span class="n">enforce_positive_activation</span><span class="o">=</span><span class="s2">&quot;softplus&quot;</span><span class="p">,</span>
     <span class="n">pred_dim</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">with_fds</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">fds_config</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
 
-  
       <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code>
 <code>deeptext</code> and <code>deepimage</code> models.</p>
 <p>Note that all models described so far in this library must be passed to
@@ -13511,211 +13389,174 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
 </ul>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>wide</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>wide</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p><code>Wide</code> model. This is a linear model where the non-linearities are
+          –
+          <div class="doc-md-description">
+            <p><code>Wide</code> model. This is a linear model where the non-linearities are
 captured via crossed-columns.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>deeptabular</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Currently this library implements a number of possible architectures
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>deeptabular</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Currently this library implements a number of possible architectures
 for the <code>deeptabular</code> component. See the documenation of the
 package.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>deeptext</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Currently this library implements a number of possible architectures
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>deeptext</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Currently this library implements a number of possible architectures
 for the <code>deeptext</code> component. See the documenation of the
 package. Note that <code>deeptext</code> can be a list of models. This is useful
 when using multiple text inputs.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>deepimage</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Currently this library uses <code>torchvision</code> and implements a number of
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>deepimage</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Currently this library uses <code>torchvision</code> and implements a number of
 possible architectures for the <code>deepimage</code> component. See the
 documenation of the package. Note that <code>deepimage</code> can be a list of
 models. This is useful when using multiple image inputs.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>deephead</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Alternatively, the user can pass a custom model that will receive the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>deephead</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Alternatively, the user can pass a custom model that will receive the
 output of the deep component. If <code>deephead</code> is not None all the
 previous fc-head parameters will be ignored</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_hidden_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the dense layers in the head. Currently
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_hidden_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the dense layers in the head. Currently
 <code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_dropout</code></b>
-            (<code>float</code>, default:
-                <code>0.1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Dropout of the dense layers in the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to include batch normalization in
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_dropout</code></b>
+              (<code>float</code>, default:
+                  <code>0.1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Dropout of the dense layers in the head</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to include batch normalization in
 the dense layers that form the <code>'rnn_mlp'</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_batchnorm_last</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether or not to apply batch normalization to the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_batchnorm_last</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether or not to apply batch normalization to the
 last of the dense layers in the head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>head_linear_first</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the order of the operations in the dense
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>head_linear_first</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>enforce_positive</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the output from the final layer must be
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>enforce_positive</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the output from the final layer must be
 positive. This is important if you are using loss functions with
 non-negative input restrictions, e.g. RMSLE, or if you know your
 predictions are bounded in between 0 and inf</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>enforce_positive_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;softplus&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Activation function to enforce that the final layer has a positive
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>enforce_positive_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;softplus&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Activation function to enforce that the final layer has a positive
 output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pred_dim</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Size of the final wide and deep output layer containing the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pred_dim</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>Size of the final wide and deep output layer containing the
 predictions. <code>1</code> for regression and binary classification or number
 of classes for multiclass classification.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_fds</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if Feature Distribution Smoothing (FDS) will be
-applied before the final prediction layer. Only available for
-regression problems.
-See <a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a> for details.</p>
-        </div>
-      </li>
-  </ul>
-
-
-
-  <p>Other Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>**fds_config</code></b>
-        –
-        <div class="doc-md-description">
-          <p>Dictionary with the parameters to be used when using Feature
-Distribution Smoothing. Please, see the docs for the <code>FDSLayer</code>.
-<br/>
-<img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: Feature Distribution Smoothing
- is available when using <strong>ONLY</strong> a <code>deeptabular</code> component
-<br/>
-<img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: We consider Feature Distribution Smoothing
-absolutely experimental and we recommend the user to not use it unless the
-corresponding <a href="https://arxiv.org/abs/2102.09554">publication</a> is
-well understood</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabResnet</span><span class="p">,</span> <span class="n">Vision</span><span class="p">,</span> <span class="n">BasicRNN</span><span class="p">,</span> <span class="n">Wide</span><span class="p">,</span> <span class="n">WideDeep</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">embed_input</span> <span class="o">=</span> <span class="p">[(</span><span class="n">u</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">)</span> <span class="k">for</span> <span class="n">u</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">([</span><span class="s2">&quot;a&quot;</span><span class="p">,</span> <span class="s2">&quot;b&quot;</span><span class="p">,</span> <span class="s2">&quot;c&quot;</span><span class="p">][:</span><span class="mi">4</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">]</span> <span class="o">*</span> <span class="mi">3</span><span class="p">,</span> <span class="p">[</span><span class="mi">8</span><span class="p">]</span> <span class="o">*</span> <span class="mi">3</span><span class="p">)]</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">column_idx</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">([</span><span class="s2">&quot;a&quot;</span><span class="p">,</span> <span class="s2">&quot;b&quot;</span><span class="p">,</span> <span class="s2">&quot;c&quot;</span><span class="p">])}</span>
@@ -13734,9 +13575,29 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
  the size of these last layers of activations. See for example
  <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/wide_deep.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">137</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/models/wide_deep.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
 <span class="normal">138</span>
 <span class="normal">139</span>
 <span class="normal">140</span>
@@ -13791,34 +13652,7 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
 <span class="normal">189</span>
 <span class="normal">190</span>
 <span class="normal">191</span>
-<span class="normal">192</span>
-<span class="normal">193</span>
-<span class="normal">194</span>
-<span class="normal">195</span>
-<span class="normal">196</span>
-<span class="normal">197</span>
-<span class="normal">198</span>
-<span class="normal">199</span>
-<span class="normal">200</span>
-<span class="normal">201</span>
-<span class="normal">202</span>
-<span class="normal">203</span>
-<span class="normal">204</span>
-<span class="normal">205</span>
-<span class="normal">206</span>
-<span class="normal">207</span>
-<span class="normal">208</span>
-<span class="normal">209</span>
-<span class="normal">210</span>
-<span class="normal">211</span>
-<span class="normal">212</span>
-<span class="normal">213</span>
-<span class="normal">214</span>
-<span class="normal">215</span>
-<span class="normal">216</span>
-<span class="normal">217</span>
-<span class="normal">218</span>
-<span class="normal">219</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+<span class="normal">192</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
     <span class="s2">&quot;pred_dim&quot;</span><span class="p">,</span>
     <span class="p">[</span><span class="s2">&quot;num_class&quot;</span><span class="p">,</span> <span class="s2">&quot;pred_size&quot;</span><span class="p">],</span>
 <span class="p">)</span>
@@ -13842,8 +13676,6 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
     <span class="n">enforce_positive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
     <span class="n">enforce_positive_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;softplus&quot;</span><span class="p">,</span>
     <span class="n">pred_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">with_fds</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">fds_config</span><span class="p">,</span>
 <span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">WideDeep</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
@@ -13855,7 +13687,6 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
         <span class="n">deephead</span><span class="p">,</span>
         <span class="n">head_hidden_dims</span><span class="p">,</span>
         <span class="n">pred_dim</span><span class="p">,</span>
-        <span class="n">with_fds</span><span class="p">,</span>
     <span class="p">)</span>
 
     <span class="c1"># this attribute will be eventually over-written by the Trainer&#39;s</span>
@@ -13865,7 +13696,6 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
     <span class="c1"># required as attribute just in case we pass a deephead</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span> <span class="o">=</span> <span class="n">pred_dim</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_fds</span> <span class="o">=</span> <span class="n">with_fds</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">enforce_positive</span> <span class="o">=</span> <span class="n">enforce_positive</span>
 
     <span class="c1"># The main 5 components of the wide and deep assemble: wide,</span>
@@ -13896,420 +13726,13 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
         <span class="n">deeptabular</span><span class="p">,</span> <span class="n">deeptext</span><span class="p">,</span> <span class="n">deepimage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_deephead</span>
     <span class="p">)</span>
 
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_fds</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">fds_layer</span> <span class="o">=</span> <span class="n">FDSLayer</span><span class="p">(</span><span class="n">feature_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">deeptabular</span><span class="o">.</span><span class="n">output_dim</span><span class="p">,</span> <span class="o">**</span><span class="n">fds_config</span><span class="p">)</span>  <span class="c1"># type: ignore[arg-type]</span>
-
     <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">enforce_positive</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">enf_pos</span> <span class="o">=</span> <span class="n">get_activation_fn</span><span class="p">(</span><span class="n">enforce_positive_activation</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
-
-  
-
-  <div class="doc doc-children">
-
-
-
-
-
-
+                  </details>
 
 
 
-
-
-  </div>
-
-  </div>
-
-
-</div>
-
-<div class="doc doc-object doc-class">
-
-
-
-<h2 id="pytorch_widedeep.models.fds_layer.FDSLayer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">FDSLayer</span>
-
-
-<a href="#pytorch_widedeep.models.fds_layer.FDSLayer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">FDSLayer</span><span class="p">(</span>
-    <span class="n">feature_dim</span><span class="p">,</span>
-    <span class="n">granularity</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span>
-    <span class="n">y_max</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">y_min</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">start_update</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
-    <span class="n">start_smooth</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
-    <span class="n">kernel</span><span class="o">=</span><span class="s2">&quot;gaussian&quot;</span><span class="p">,</span>
-    <span class="n">ks</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
-    <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">,</span>
-    <span class="n">clip_min</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">clip_max</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
-
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="torch.Module">Module</span></code></p>
-
-  
-      <p>Feature Distribution Smoothing layer. Please, see
-<a href="https:/arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>
-for details.</p>
-<p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: this is NOT an available model per se,
- but more a utility that can be used as we run a <code>WideDeep</code> model.
- The parameters of this extra layers can be set as the class
- <code>WideDeep</code> is instantiated via the keyword arguments <code>fds_config</code>.</p>
-<p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: Feature Distribution Smoothing is
- available when using ONLY a <code>deeptabular</code> component</p>
-<p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: We consider this feature absolutely
-experimental and we recommend the user to not use it unless the
-corresponding <a href="https://arxiv.org/abs/2102.09554">publication</a> is
-well understood</p>
-<p>The code here is based on the code at the
-<a href="https://github.com/YyzHarry/imbalanced-regression">official repo</a></p>
-
-
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>feature_dim</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>input dimension size, i.e. output size of previous layer. This
-will be the dimension of the output from the <code>deeptabular</code>
-component</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>granularity</code></b>
-            (<code>int</code>, default:
-                <code>100</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>number of bins that the target <span class="arithmatex">\(y\)</span> is divided into and that will
-be used to compute the features' statistics (mean and variance)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>y_max</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p><span class="arithmatex">\(y\)</span> upper limit to be considered when binning</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>y_min</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p><span class="arithmatex">\(y\)</span> lower limit to be considered when binning</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>start_update</code></b>
-            (<code>int</code>, default:
-                <code>0</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>number of _'waiting epochs' after which the FDS layer will start
-to update its statistics</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>start_smooth</code></b>
-            (<code>int</code>, default:
-                <code>2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>number of _'waiting epochs' after which the FDS layer will start
-smoothing the feature distributions</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>kernel</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[gaussian, triang, laplace]</code>, default:
-                <code>&#39;gaussian&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>choice of smoothing kernel</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>ks</code></b>
-            (<code>int</code>, default:
-                <code>5</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>kernel window size</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>sigma</code></b>
-            (<code>float</code>, default:
-                <code>2</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>if a <em>'gaussian'</em> or <em>'laplace'</em> kernels are used, this is the
-corresponding standard deviation</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>momentum</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>0.9</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>to train the layer the authors used a momentum update of the running
-statistics across each epoch. Set to 0.9 in the paper.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>clip_min</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>this parameter is used to clip the ratio between the so called
-running variance and the smoothed variance, and is introduced for
-numerical stability. We leave it as optional as we did not find a
-notable improvement in our experiments. The authors used a value
-of 0.1</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>clip_max</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>same as <code>clip_min</code> but for the upper limit.We leave it as optional
-as we did not find a notable improvement in our experiments. The
-authors used a value of 10.</p>
-        </div>
-      </li>
-  </ul>
-
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/models/fds_layer.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 13</span>
-<span class="normal"> 14</span>
-<span class="normal"> 15</span>
-<span class="normal"> 16</span>
-<span class="normal"> 17</span>
-<span class="normal"> 18</span>
-<span class="normal"> 19</span>
-<span class="normal"> 20</span>
-<span class="normal"> 21</span>
-<span class="normal"> 22</span>
-<span class="normal"> 23</span>
-<span class="normal"> 24</span>
-<span class="normal"> 25</span>
-<span class="normal"> 26</span>
-<span class="normal"> 27</span>
-<span class="normal"> 28</span>
-<span class="normal"> 29</span>
-<span class="normal"> 30</span>
-<span class="normal"> 31</span>
-<span class="normal"> 32</span>
-<span class="normal"> 33</span>
-<span class="normal"> 34</span>
-<span class="normal"> 35</span>
-<span class="normal"> 36</span>
-<span class="normal"> 37</span>
-<span class="normal"> 38</span>
-<span class="normal"> 39</span>
-<span class="normal"> 40</span>
-<span class="normal"> 41</span>
-<span class="normal"> 42</span>
-<span class="normal"> 43</span>
-<span class="normal"> 44</span>
-<span class="normal"> 45</span>
-<span class="normal"> 46</span>
-<span class="normal"> 47</span>
-<span class="normal"> 48</span>
-<span class="normal"> 49</span>
-<span class="normal"> 50</span>
-<span class="normal"> 51</span>
-<span class="normal"> 52</span>
-<span class="normal"> 53</span>
-<span class="normal"> 54</span>
-<span class="normal"> 55</span>
-<span class="normal"> 56</span>
-<span class="normal"> 57</span>
-<span class="normal"> 58</span>
-<span class="normal"> 59</span>
-<span class="normal"> 60</span>
-<span class="normal"> 61</span>
-<span class="normal"> 62</span>
-<span class="normal"> 63</span>
-<span class="normal"> 64</span>
-<span class="normal"> 65</span>
-<span class="normal"> 66</span>
-<span class="normal"> 67</span>
-<span class="normal"> 68</span>
-<span class="normal"> 69</span>
-<span class="normal"> 70</span>
-<span class="normal"> 71</span>
-<span class="normal"> 72</span>
-<span class="normal"> 73</span>
-<span class="normal"> 74</span>
-<span class="normal"> 75</span>
-<span class="normal"> 76</span>
-<span class="normal"> 77</span>
-<span class="normal"> 78</span>
-<span class="normal"> 79</span>
-<span class="normal"> 80</span>
-<span class="normal"> 81</span>
-<span class="normal"> 82</span>
-<span class="normal"> 83</span>
-<span class="normal"> 84</span>
-<span class="normal"> 85</span>
-<span class="normal"> 86</span>
-<span class="normal"> 87</span>
-<span class="normal"> 88</span>
-<span class="normal"> 89</span>
-<span class="normal"> 90</span>
-<span class="normal"> 91</span>
-<span class="normal"> 92</span>
-<span class="normal"> 93</span>
-<span class="normal"> 94</span>
-<span class="normal"> 95</span>
-<span class="normal"> 96</span>
-<span class="normal"> 97</span>
-<span class="normal"> 98</span>
-<span class="normal"> 99</span>
-<span class="normal">100</span>
-<span class="normal">101</span>
-<span class="normal">102</span>
-<span class="normal">103</span>
-<span class="normal">104</span>
-<span class="normal">105</span>
-<span class="normal">106</span>
-<span class="normal">107</span>
-<span class="normal">108</span>
-<span class="normal">109</span>
-<span class="normal">110</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">feature_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">granularity</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">100</span><span class="p">,</span>
-    <span class="n">y_max</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">y_min</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">start_update</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
-    <span class="n">start_smooth</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
-    <span class="n">kernel</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;gaussian&quot;</span><span class="p">,</span> <span class="s2">&quot;triang&quot;</span><span class="p">,</span> <span class="s2">&quot;laplace&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;gaussian&quot;</span><span class="p">,</span>
-    <span class="n">ks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
-    <span class="n">momentum</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="mf">0.9</span><span class="p">,</span>
-    <span class="n">clip_min</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">clip_max</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Feature Distribution Smoothing layer. Please, see</span>
-<span class="sd">    [Delving into Deep Imbalanced Regression](https:/arxiv.org/abs/2102.09554)</span>
-<span class="sd">    for details.</span>
-
-<span class="sd">    :information_source: **NOTE**: this is NOT an available model per se,</span>
-<span class="sd">     but more a utility that can be used as we run a `WideDeep` model.</span>
-<span class="sd">     The parameters of this extra layers can be set as the class</span>
-<span class="sd">     `WideDeep` is instantiated via the keyword arguments `fds_config`.</span>
-
-<span class="sd">    :information_source: **NOTE**: Feature Distribution Smoothing is</span>
-<span class="sd">     available when using ONLY a `deeptabular` component</span>
-
-<span class="sd">    :information_source: **NOTE**: We consider this feature absolutely</span>
-<span class="sd">    experimental and we recommend the user to not use it unless the</span>
-<span class="sd">    corresponding [publication](https://arxiv.org/abs/2102.09554) is</span>
-<span class="sd">    well understood</span>
-
-<span class="sd">    The code here is based on the code at the</span>
-<span class="sd">    [official repo](https://github.com/YyzHarry/imbalanced-regression)</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    feature_dim: int,</span>
-<span class="sd">        input dimension size, i.e. output size of previous layer. This</span>
-<span class="sd">        will be the dimension of the output from the `deeptabular`</span>
-<span class="sd">        component</span>
-<span class="sd">    granularity: int = 100,</span>
-<span class="sd">        number of bins that the target $y$ is divided into and that will</span>
-<span class="sd">        be used to compute the features&#39; statistics (mean and variance)</span>
-<span class="sd">    y_max: Optional[float] = None,</span>
-<span class="sd">        $y$ upper limit to be considered when binning</span>
-<span class="sd">    y_min: Optional[float] = None,</span>
-<span class="sd">        $y$ lower limit to be considered when binning</span>
-<span class="sd">    start_update: int = 0,</span>
-<span class="sd">        number of _&#39;waiting epochs&#39; after which the FDS layer will start</span>
-<span class="sd">        to update its statistics</span>
-<span class="sd">    start_smooth: int = 1,</span>
-<span class="sd">        number of _&#39;waiting epochs&#39; after which the FDS layer will start</span>
-<span class="sd">        smoothing the feature distributions</span>
-<span class="sd">    kernel: Literal[&quot;gaussian&quot;, &quot;triang&quot;, &quot;laplace&quot;, None] = &quot;gaussian&quot;,</span>
-<span class="sd">        choice of smoothing kernel</span>
-<span class="sd">    ks: int = 5,</span>
-<span class="sd">        kernel window size</span>
-<span class="sd">    sigma: Union[int, float] = 2,</span>
-<span class="sd">        if a _&#39;gaussian&#39;_ or _&#39;laplace&#39;_ kernels are used, this is the</span>
-<span class="sd">        corresponding standard deviation</span>
-<span class="sd">    momentum: float = 0.9,</span>
-<span class="sd">        to train the layer the authors used a momentum update of the running</span>
-<span class="sd">        statistics across each epoch. Set to 0.9 in the paper.</span>
-<span class="sd">    clip_min: Optional[float] = None,</span>
-<span class="sd">        this parameter is used to clip the ratio between the so called</span>
-<span class="sd">        running variance and the smoothed variance, and is introduced for</span>
-<span class="sd">        numerical stability. We leave it as optional as we did not find a</span>
-<span class="sd">        notable improvement in our experiments. The authors used a value</span>
-<span class="sd">        of 0.1</span>
-<span class="sd">    clip_max: Optional[float] = None,</span>
-<span class="sd">        same as `clip_min` but for the upper limit.We leave it as optional</span>
-<span class="sd">        as we did not find a notable improvement in our experiments. The</span>
-<span class="sd">        authors used a value of 10.</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">FDSLayer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-    <span class="k">assert</span> <span class="p">(</span>
-        <span class="n">start_update</span> <span class="o">+</span> <span class="mi">1</span> <span class="o">&lt;</span> <span class="n">start_smooth</span>
-    <span class="p">),</span> <span class="s2">&quot;initial update must start at least 2 epoch before smoothing&quot;</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">feature_dim</span> <span class="o">=</span> <span class="n">feature_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">granularity</span> <span class="o">=</span> <span class="n">granularity</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">y_max</span> <span class="o">=</span> <span class="n">y_max</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">y_min</span> <span class="o">=</span> <span class="n">y_min</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">kernel_window</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span>
-        <span class="n">get_kernel_window</span><span class="p">(</span><span class="n">kernel</span><span class="p">,</span> <span class="n">ks</span><span class="p">,</span> <span class="n">sigma</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span>
-    <span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">half_ks</span> <span class="o">=</span> <span class="p">(</span><span class="n">ks</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">//</span> <span class="mi">2</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">momentum</span> <span class="o">=</span> <span class="n">momentum</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">start_update</span> <span class="o">=</span> <span class="n">start_update</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">start_smooth</span> <span class="o">=</span> <span class="n">start_smooth</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">clip_min</span> <span class="o">=</span> <span class="n">clip_min</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">clip_max</span> <span class="o">=</span> <span class="n">clip_max</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">pred_layer</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">feature_dim</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">_register_buffers</span><span class="p">()</span>
-</code></pre></div></td></tr></table></div>
-                </details>
-
-  
-
   <div class="doc doc-children">
 
 
@@ -14324,8 +13747,7 @@ <h2 id="pytorch_widedeep.models.fds_layer.FDSLayer" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -14401,7 +13823,7 @@ <h2 id="pytorch_widedeep.models.fds_layer.FDSLayer" class="doc doc-heading">
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -14419,7 +13841,7 @@ <h2 id="pytorch_widedeep.models.fds_layer.FDSLayer" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/model_components.md b/mkdocs/site/pytorch-widedeep/model_components.md
index 1257735a..10ef9e89 100644
--- a/mkdocs/site/pytorch-widedeep/model_components.md
+++ b/mkdocs/site/pytorch-widedeep/model_components.md
@@ -127,9 +127,3 @@ self-supervised pre-training with tabular data.
         filters:
             - "!^_"  # exclude all members starting with _
             - "!^forward$"
-
-::: pytorch_widedeep.models.fds_layer.FDSLayer
-    selection:
-        filters:
-            - "!^_"  # exclude all members starting with _
-            - "!^forward$"
diff --git a/mkdocs/site/pytorch-widedeep/preprocessing.html b/mkdocs/site/pytorch-widedeep/preprocessing.html
index 4130aadf..01e87641 100644
--- a/mkdocs/site/pytorch-widedeep/preprocessing.html
+++ b/mkdocs/site/pytorch-widedeep/preprocessing.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1117,8 +1117,6 @@
         
       
         
-      
-        
       
         
       
@@ -1456,32 +1454,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1498,11 +1475,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1519,11 +1496,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1540,11 +1517,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1561,11 +1538,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1582,11 +1559,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1603,11 +1580,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1624,11 +1601,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1698,18 +1675,18 @@ <h1 id="the-preprocessing-module">The <code>preprocessing</code> module<a class=
 
 
 <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">WidePreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">WidePreprocessor</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">WidePreprocessor</span><span class="p">(</span><span class="n">wide_cols</span><span class="p">,</span> <span class="n">crossed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">WidePreprocessor</span><span class="p">(</span><span class="n">wide_cols</span><span class="p">,</span> <span class="n">crossed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+
 
-  
       <p>Preprocessor to prepare the wide input dataset</p>
 <p>This Preprocessor prepares the data for the wide, linear component.
 This linear model is implemented via an Embedding layer that is
@@ -1718,75 +1695,72 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class
 crossed_cols</code>. See the Example below.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>wide_cols</code></b>
-            (<code><span title="typing.List">List</span>[str]</code>)
-        –
-        <div class="doc-md-description">
-          <p>List of strings with the name of the columns that will label
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>wide_cols</code></b>
+              (<code><span title="typing.List">List</span>[str]</code>)
+          –
+          <div class="doc-md-description">
+            <p>List of strings with the name of the columns that will label
 encoded and passed through the <code>wide</code> component</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>crossed_cols</code></b>
-            (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>crossed_cols</code></b>
+              (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the name of the columns that will be <code>'crossed'</code>
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code>
 and then label encoded. e.g. <em>[('education', 'occupation'), ...]</em>. For
 binary features, a cross-product transformation is 1 if and only if
 the constituent features are all 1, and 0 otherwise.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.wide_crossed_cols">wide_crossed_cols</span></code></b>
-            (<code><span title="typing.List">List</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>List with the names of all columns that will be label encoded</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.encoding_dict">encoding_dict</span></code></b>
-            (<code>Dict</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dictionary where the keys are the result of pasting <code>colname + '_' +
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.wide_crossed_cols">wide_crossed_cols</span></code></b>
+              (<code><span title="typing.List">List</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>List with the names of all columns that will be label encoded</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.encoding_dict">encoding_dict</span></code></b>
+              (<code>Dict</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dictionary where the keys are the result of pasting <code>colname + '_' +
 column value</code> and the values are the corresponding mapped integer.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_encoding_dict">inverse_encoding_dict</span></code></b>
-            (<code>Dict</code>)
-        –
-        <div class="doc-md-description">
-          <p>the inverse encoding dictionary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.wide_dim">wide_dim</span></code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dimension of the wide model (i.e. dim of the linear layer)</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_encoding_dict">inverse_encoding_dict</span></code></b>
+              (<code>Dict</code>)
+          –
+          <div class="doc-md-description">
+            <p>the inverse encoding dictionary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.wide_dim">wide_dim</span></code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dimension of the wide model (i.e. dim of the linear layer)</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">WidePreprocessor</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;color&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="s1">&#39;g&#39;</span><span class="p">],</span> <span class="s1">&#39;size&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;s&#39;</span><span class="p">,</span> <span class="s1">&#39;n&#39;</span><span class="p">,</span> <span class="s1">&#39;l&#39;</span><span class="p">]})</span>
@@ -1807,9 +1781,9 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class
 <span class="go">2     g        g-l</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">67</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">67</span>
 <span class="normal">68</span>
 <span class="normal">69</span>
 <span class="normal">70</span>
@@ -1827,12 +1801,11 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class
 
     <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1845,49 +1818,46 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit</span>
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Fits the Preprocessor and creates required attributes</p>
-
+    <div class="doc doc-contents ">
 
+      <p>Fits the Preprocessor and creates required attributes</p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor">WidePreprocessor</a></code>
-        –
-        <div class="doc-md-description">
-          <p><code>WidePreprocessor</code> fitted object</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor">WidePreprocessor</a></code>
+          –
+          <div class="doc-md-description">
+            <p><code>WidePreprocessor</code> fitted object</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 77</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 77</span>
 <span class="normal"> 78</span>
 <span class="normal"> 79</span>
 <span class="normal"> 80</span>
@@ -1941,57 +1911,53 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit" c
 
     <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">transform</span>
+            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-
+    <div class="doc doc-contents ">
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>transformed input dataframe</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>transformed input dataframe</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">105</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">105</span>
 <span class="normal">106</span>
 <span class="normal">107</span>
 <span class="normal">108</span>
@@ -2039,60 +2005,56 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transf
         <span class="p">)</span>
     <span class="k">return</span> <span class="n">encoded</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;int64&quot;</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">inverse_transform</span>
+            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">inverse_transform</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Takes as input the output from the <code>transform</code> method and it will
 return the original values.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>encoded</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>numpy array with the encoded values that are the output from the
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>encoded</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>numpy array with the encoded values that are the output from the
 <code>transform</code> method</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="pandas.DataFrame">DataFrame</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Pandas dataframe with the original values</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="pandas.DataFrame">DataFrame</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Pandas dataframe with the original values</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">133</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">133</span>
 <span class="normal">134</span>
 <span class="normal">135</span>
 <span class="normal">136</span>
@@ -2144,58 +2106,54 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.invers
         <span class="n">decoded</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">decoded</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">rm_str</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
     <span class="k">return</span> <span class="n">decoded</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit_transform</span>
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Combines <code>fit</code> and <code>transform</code></p>
-
+    <div class="doc doc-contents ">
 
+      <p>Combines <code>fit</code> and <code>transform</code></p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>transformed input dataframe</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>transformed input dataframe</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">160</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">160</span>
 <span class="normal">161</span>
 <span class="normal">162</span>
 <span class="normal">163</span>
@@ -2223,8 +2181,8 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_tr
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2232,8 +2190,7 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_tr
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2242,11 +2199,11 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_tr
 
 
 <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TabPreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">TabPreprocessor</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TabPreprocessor</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabPreprocessor</span><span class="p">(</span>
     <span class="n">cat_embed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -2265,87 +2222,86 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
 
-  
-      <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p>
 
+      <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>cat_embed_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List containing the name of the categorical columns that will be
+          –
+          <div class="doc-md-description">
+            <p>List containing the name of the categorical columns that will be
 represented by embeddings (e.g. <em>['education', 'relationship', ...]</em>) or
 a Tuple with the name and the embedding dimension (e.g.: <em>[
 ('education',32), ('relationship',16), ...]</em>)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the continuous cols</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the continuous cols</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If
+          –
+          <div class="doc-md-description">
+            <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If
 <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be
 quantized using this value as the number of bins. Alternatively, a
 dictionary where the keys are the column names to quantize and the
 values are the either integers indicating the number of bins or a
 list of scalars indicating the bin edges can also be used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cols_to_scale</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cols_to_scale</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with the names of the columns that will be standarised via
+          –
+          <div class="doc-md-description">
+            <p>List with the names of the columns that will be standarised via
 sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in
 which case all the continuous cols will be scaled.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>auto_embed_dim</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>auto_embed_dim</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the embedding dimensions will be
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the embedding dimensions will be
 automatically defined via rule of thumb. See <code>embedding_rule</code>
 below.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>embedding_rule</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[google, fastai_old, fastai_new]</code>, default:
-                <code>&#39;fastai_new&#39;</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>embedding_rule</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[google, fastai_old, fastai_new]</code>, default:
+                  <code>&#39;fastai_new&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of
+          –
+          <div class="doc-md-description">
+            <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of
 thumb. Choices are:</p>
 <ul>
 <li>
@@ -2358,28 +2314,28 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 <p><em>google</em>: <span class="arithmatex">\(min(600, round(n_{cat}^{0.24}))\)</span></p>
 </li>
 </ul>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>default_embed_dim</code></b>
-            (<code>int</code>, default:
-                <code>16</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>default_embed_dim</code></b>
+              (<code>int</code>, default:
+                  <code>16</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Dimension for the embeddings if the embedding dimension is not
+          –
+          <div class="doc-md-description">
+            <p>Dimension for the embeddings if the embedding dimension is not
 provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is
 set to <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_attention</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>with_attention</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the preprocessed data will be passed to an
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the preprocessed data will be passed to an
 attention-based model (more precisely a model where all embeddings
 must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code>
 must just be a list containing just the categorical column names:
@@ -2388,164 +2344,161 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
  encoded using embeddings of the same dim, which will be specified
  later when the model is defined. <br/> Param alias:
  <code>for_transformer</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_cls_token</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>with_cls_token</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset
 when using attention-based models. The final hidden state
 corresponding to this token is used as the aggregated representation
 for classification and regression tasks. If not, the categorical
 and/or continuous embeddings will be concatenated before being passed
 to the final MLP (if present).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>shared_embed</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>shared_embed</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the embeddings will be "shared" when using
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the embeddings will be "shared" when using
 attention-based models. The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>scale</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          –
+          <div class="doc-md-description">
+            
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>scale</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
+          –
+          <div class="doc-md-description">
+            <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
  releases. Please use <code>cols_to_scale</code> instead. <br/> Bool indicating
  whether or not to scale/standarise continuous cols. It is important
  to emphasize that all the DL models for tabular data in the library
  also include the possibility of normalising the input continuous
  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>. <br/> Param alias:
  <code>scale_cont_cols</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>already_standard</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>already_standard</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
+          –
+          <div class="doc-md-description">
+            <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
  releases. Please use <code>cols_to_scale</code> instead. <br/> List with the
  name of the continuous cols that do not need to be
  scaled/standarised.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Other Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Other Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.embed_dim">embed_dim</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Dictionary where keys are the embed cols and values are the embedding
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.embed_dim">embed_dim</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Dictionary where keys are the embed cols and values are the embedding
 dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute
 is not generated during the <code>fit</code> process</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.label_encoder">label_encoder</span></code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>)
-        –
-        <div class="doc-md-description">
-          <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.cat_embed_input">cat_embed_input</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name, number of individual values for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.label_encoder">label_encoder</span></code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>)
+          –
+          <div class="doc-md-description">
+            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.cat_embed_input">cat_embed_input</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name, number of individual values for
 that column and, If <code>with_attention</code> is set to <code>False</code>, the
 corresponding embeddings dim, e.g. <em>[('education', 16, 10),
 ('relationship', 6, 8), ...]</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.standardize_cols">standardize_cols</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>List of the columns that will be standarized</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.scaler">scaler</span></code></b>
-            (<code><span title="sklearn.preprocessing.StandardScaler">StandardScaler</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>an instance of <code>sklearn.preprocessing.StandardScaler</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.column_idx">column_idx</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Dictionary where keys are column names and values are column indexes.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.standardize_cols">standardize_cols</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>List of the columns that will be standarized</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.scaler">scaler</span></code></b>
+              (<code><span title="sklearn.preprocessing.StandardScaler">StandardScaler</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.column_idx">column_idx</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Dictionary where keys are column names and values are column indexes.
 This is neccesary to slice tensors</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.quantizer">quantizer</span></code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer">Quantizer</a></code>)
-        –
-        <div class="doc-md-description">
-          <p>an instance of <code>Quantizer</code></p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.quantizer">quantizer</span></code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer">Quantizer</a></code>)
+          –
+          <div class="doc-md-description">
+            <p>an instance of <code>Quantizer</code></p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">TabPreprocessor</span>
@@ -2568,9 +2521,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 <span class="gp">&gt;&gt;&gt; </span><span class="n">ft_cont_df2</span> <span class="o">=</span> <span class="n">tab_preprocessor2</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">cont_df</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">249</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">249</span>
 <span class="normal">250</span>
 <span class="normal">251</span>
 <span class="normal">252</span>
@@ -2686,12 +2639,11 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 
     <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2704,49 +2656,46 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit</span>
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Fits the Preprocessor and creates required attributes</p>
-
+    <div class="doc doc-contents ">
 
+      <p>Fits the Preprocessor and creates required attributes</p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
-        –
-        <div class="doc-md-description">
-          <p><code>TabPreprocessor</code> fitted object</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
+          –
+          <div class="doc-md-description">
+            <p><code>TabPreprocessor</code> fitted object</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">308</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">308</span>
 <span class="normal">309</span>
 <span class="normal">310</span>
 <span class="normal">311</span>
@@ -2894,58 +2843,54 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" cla
 
     <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">transform</span>
+            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Returns the processed <code>dataframe</code> as a np.ndarray</p>
-
+    <div class="doc doc-contents ">
 
+      <p>Returns the processed <code>dataframe</code> as a np.ndarray</p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>transformed input dataframe</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>transformed input dataframe</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">383</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">383</span>
 <span class="normal">384</span>
 <span class="normal">385</span>
 <span class="normal">386</span>
@@ -3033,59 +2978,55 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transfor
 
     <span class="k">return</span> <span class="n">df_deep</span><span class="o">.</span><span class="n">values</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">inverse_transform</span>
+            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">inverse_transform</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Takes as input the output from the <code>transform</code> method and it will
 return the original values.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>encoded</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>array with the output of the <code>transform</code> method</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>encoded</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>array with the output of the <code>transform</code> method</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="pandas.DataFrame">DataFrame</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Pandas dataframe with the original values</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="pandas.DataFrame">DataFrame</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Pandas dataframe with the original values</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">431</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">431</span>
 <span class="normal">432</span>
 <span class="normal">433</span>
 <span class="normal">434</span>
@@ -3165,58 +3106,54 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_
 
     <span class="k">return</span> <span class="n">decoded</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit_transform</span>
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Combines <code>fit</code> and <code>transform</code></p>
+    <div class="doc doc-contents ">
 
+      <p>Combines <code>fit</code> and <code>transform</code></p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>transformed input dataframe</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>transformed input dataframe</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">472</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">472</span>
 <span class="normal">473</span>
 <span class="normal">474</span>
 <span class="normal">475</span>
@@ -3244,8 +3181,8 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_tran
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -3253,8 +3190,7 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_tran
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -3263,16 +3199,16 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_tran
 
 
 <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Quantizer</span>
+            <span class="doc doc-object-name doc-class-name">Quantizer</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Quantizer</span><span class="p">(</span><span class="n">quantization_setup</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Quantizer</span><span class="p">(</span><span class="n">quantization_setup</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
+    <div class="doc doc-contents first">
+
 
-  
       <p>Helper class to perform the quantization of continuous columns. It is
 included in this docs for completion, since depending on the value of the
 parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that
@@ -3280,24 +3216,23 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc do
 designed to always run internally within the <code>TabPreprocessor</code> class.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>quantization_setup</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dictionary where the keys are the column names to quantize and the
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>quantization_setup</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dictionary where the keys are the column names to quantize and the
 values are the either integers indicating the number of bins or a
 list of scalars indicating the bin edges.</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">61</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">61</span>
 <span class="normal">62</span>
 <span class="normal">63</span>
 <span class="normal">64</span>
@@ -3315,12 +3250,11 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc do
 
     <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -3331,10 +3265,10 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc do
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -3343,11 +3277,11 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc do
 
 
 <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">TextPreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">TextPreprocessor</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">TextPreprocessor</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TextPreprocessor</span><span class="p">(</span>
     <span class="n">text_col</span><span class="p">,</span>
     <span class="n">max_vocab</span><span class="o">=</span><span class="mi">30000</span><span class="p">,</span>
     <span class="n">min_freq</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
@@ -3361,149 +3295,146 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
 
-  
-      <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p>
 
+      <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>text_col</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>column in the input dataframe containing the texts</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>max_vocab</code></b>
-            (<code>int</code>, default:
-                <code>30000</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>text_col</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>column in the input dataframe containing the texts</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>max_vocab</code></b>
+              (<code>int</code>, default:
+                  <code>30000</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Maximum number of tokens in the vocabulary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>min_freq</code></b>
-            (<code>int</code>, default:
-                <code>5</code>
+          –
+          <div class="doc-md-description">
+            <p>Maximum number of tokens in the vocabulary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>min_freq</code></b>
+              (<code>int</code>, default:
+                  <code>5</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Minimum frequency for a token to be part of the vocabulary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>maxlen</code></b>
-            (<code>int</code>, default:
-                <code>80</code>
+          –
+          <div class="doc-md-description">
+            <p>Minimum frequency for a token to be part of the vocabulary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>maxlen</code></b>
+              (<code>int</code>, default:
+                  <code>80</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Maximum length of the tokenized sequences</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pad_first</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
+          –
+          <div class="doc-md-description">
+            <p>Maximum length of the tokenized sequences</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pad_first</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Indicates whether the padding index will be added at the beginning or the
+          –
+          <div class="doc-md-description">
+            <p>Indicates whether the padding index will be added at the beginning or the
 end of the sequences</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pad_idx</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pad_idx</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>already_processed</code></b>
-            (<code><span title="typing.Optional">Optional</span>[bool]</code>, default:
-                <code>False</code>
+          –
+          <div class="doc-md-description">
+            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>already_processed</code></b>
+              (<code><span title="typing.Optional">Optional</span>[bool]</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the sequence of elements is already processed or
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the sequence of elements is already processed or
 prepared. If this is the case, this Preprocessor will simply tokenize
 and pad the sequence. <br/></p>
 <div class="highlight"><pre><span></span><code>Param aliases: `not_text`. &lt;br/&gt;
 </code></pre></div>
 <p>This parameter is thought for those cases where the input sequences
 are already fully processed or are directly not text (e.g. IDs)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>word_vectors_path</code></b>
-            (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>word_vectors_path</code></b>
+              (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Path to the pretrained word vectors</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_cpus</code></b>
-            (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>Path to the pretrained word vectors</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_cpus</code></b>
+              (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of CPUs to used during the tokenization process</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>number of CPUs to used during the tokenization process</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Enable verbose output.</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>Enable verbose output.</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.vocab">vocab</span></code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>)
-        –
-        <div class="doc-md-description">
-          <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.embedding_matrix">embedding_matrix</span></code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Array with the pretrained embeddings</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.vocab">vocab</span></code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>)
+          –
+          <div class="doc-md-description">
+            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.embedding_matrix">embedding_matrix</span></code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Array with the pretrained embeddings</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">TextPreprocessor</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">df_train</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;text_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;life is like a box of chocolates&quot;</span><span class="p">,</span>
@@ -3518,9 +3449,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
 <span class="go">array([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 79</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 79</span>
 <span class="normal"> 80</span>
 <span class="normal"> 81</span>
 <span class="normal"> 82</span>
@@ -3576,12 +3507,11 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
 
     <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -3594,49 +3524,46 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit</span>
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Builds the vocabulary</p>
-
+    <div class="doc doc-contents ">
 
+      <p>Builds the vocabulary</p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor">TextPreprocessor</a></code>
-        –
-        <div class="doc-md-description">
-          <p><code>TextPreprocessor</code> fitted object</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor">TextPreprocessor</a></code>
+          –
+          <div class="doc-md-description">
+            <p><code>TextPreprocessor</code> fitted object</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">108</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">108</span>
 <span class="normal">109</span>
 <span class="normal">110</span>
 <span class="normal">111</span>
@@ -3706,58 +3633,54 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" c
 
     <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">transform</span>
+            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Returns the padded, <em>'numericalised'</em> sequences</p>
+    <div class="doc doc-contents ">
 
+      <p>Returns the padded, <em>'numericalised'</em> sequences</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Padded, <em>'numericalised'</em> sequences</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Padded, <em>'numericalised'</em> sequences</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
 <span class="normal">145</span>
 <span class="normal">146</span>
 <span class="normal">147</span>
@@ -3791,58 +3714,54 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transf
     <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">transform_sample</span>
+            <span class="doc doc-object-name doc-function-name">transform_sample</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">transform_sample</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform_sample</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Returns the padded, <em>'numericalised'</em> sequence</p>
+    <div class="doc doc-contents ">
 
+      <p>Returns the padded, <em>'numericalised'</em> sequence</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>text</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>text to be tokenized and padded</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>text</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>text to be tokenized and padded</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Padded, <em>'numericalised'</em> sequence</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Padded, <em>'numericalised'</em> sequence</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">162</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">162</span>
 <span class="normal">163</span>
 <span class="normal">164</span>
 <span class="normal">165</span>
@@ -3874,58 +3793,54 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transf
     <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">([</span><span class="n">text</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit_transform</span>
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Combines <code>fit</code> and <code>transform</code></p>
-
+    <div class="doc doc-contents ">
 
+      <p>Combines <code>fit</code> and <code>transform</code></p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Padded, <em>'numericalised'</em> sequences</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Padded, <em>'numericalised'</em> sequences</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">179</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">179</span>
 <span class="normal">180</span>
 <span class="normal">181</span>
 <span class="normal">182</span>
@@ -3953,58 +3868,54 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_tr
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">inverse_transform</span>
+            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">inverse_transform</span><span class="p">(</span><span class="n">padded_seq</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">padded_seq</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Returns the original text plus the added 'special' tokens</p>
-
+    <div class="doc doc-contents ">
 
+      <p>Returns the original text plus the added 'special' tokens</p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>padded_seq</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>array with the output of the <code>transform</code> method</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>padded_seq</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>array with the output of the <code>transform</code> method</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="pandas.DataFrame">DataFrame</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Pandas dataframe with the original text plus the added 'special' tokens</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="pandas.DataFrame">DataFrame</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Pandas dataframe with the original text plus the added 'special' tokens</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">194</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">194</span>
 <span class="normal">195</span>
 <span class="normal">196</span>
 <span class="normal">197</span>
@@ -4034,8 +3945,8 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.invers
     <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">num</span><span class="p">)</span> <span class="k">for</span> <span class="n">num</span> <span class="ow">in</span> <span class="n">padded_seq</span><span class="p">]</span>
     <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">:</span> <span class="n">texts</span><span class="p">})</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -4043,8 +3954,7 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.invers
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -4053,11 +3963,11 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.invers
 
 
 <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">HFPreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">HFPreprocessor</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">HFPreprocessor</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">HFPreprocessor</span><span class="p">(</span>
     <span class="n">model_name</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">use_fast_tokenizer</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
@@ -4071,11 +3981,11 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+
 
-  
       <p>Text processor to prepare the <code>deeptext</code> input dataset that is a
 wrapper around HuggingFace's tokenizers.</p>
 <p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this
@@ -4084,125 +3994,122 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
 or following the API call 'protocol' of the rest of the library.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>model_name</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>The model name from the transformers library e.g. <em>'bert-base-uncased'</em>.
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>model_name</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>The model name from the transformers library e.g. <em>'bert-base-uncased'</em>.
 Currently supported models are those from the families: BERT, RoBERTa,
 DistilBERT, ALBERT and ELECTRA.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_fast_tokenizer</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_fast_tokenizer</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Whether to use the fast tokenizer from HuggingFace or not</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>text_col</code></b>
-            (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>Whether to use the fast tokenizer from HuggingFace or not</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>text_col</code></b>
+              (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The column in the input dataframe containing the text data. If this
+          –
+          <div class="doc-md-description">
+            <p>The column in the input dataframe containing the text data. If this
 tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this
 argument is mandatory. If the tokenizer is used via the <code>encode</code>
 method, this argument is not needed since the input text is passed
 directly to the <code>encode</code> method.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>num_workers</code></b>
-            (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>num_workers</code></b>
+              (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Number of workers to use when preprocessing the text data. If not
+          –
+          <div class="doc-md-description">
+            <p>Number of workers to use when preprocessing the text data. If not
 None, and <code>use_fast_tokenizer</code> is False, the text data will be
 preprocessed in parallel using the number of workers specified. If
 <code>use_fast_tokenizer</code> is True, this argument is ignored.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>preprocessing_rules</code></b>
-            (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>preprocessing_rules</code></b>
+              (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>A list of functions to be applied to the text data before encoding.
+          –
+          <div class="doc-md-description">
+            <p>A list of functions to be applied to the text data before encoding.
 This can be useful to clean the text data before encoding. For
 example, removing html tags, special characters, etc.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>tokenizer_params</code></b>
-            (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>tokenizer_params</code></b>
+              (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Additional parameters to be passed to the HuggingFace's
+          –
+          <div class="doc-md-description">
+            <p>Additional parameters to be passed to the HuggingFace's
 <code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code>
 can also be passed via the <code>**kwargs</code> argument</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>encode_params</code></b>
-            (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>encode_params</code></b>
+              (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
+          –
+          <div class="doc-md-description">
+            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
 of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code>
 methods are used, the <code>encode_params</code> dict parameter is mandatory. If
 the <code>encode</code> method is used, this parameter is not needed since the
 input text is passed directly to the <code>encode</code> method.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p>Additional kwargs to be passed to the model, in particular to the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p>Additional kwargs to be passed to the model, in particular to the
 <code>PreTrainedTokenizer</code> class.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.is_fitted">is_fitted</span></code></b>
-            (<code>bool</code>)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the preprocessor has been fitted. This is a
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.is_fitted">is_fitted</span></code></b>
+              (<code>bool</code>)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the preprocessor has been fitted. This is a
 HuggingFacea tokenizer, so it is always considered fitted and this
 attribute is manually set to True internally. This parameter exists
 for consistency with the rest of the library and because is needed
 for some functionality in the library.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">HFPreprocessor</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s2">&quot;text&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;this is the first text&quot;</span><span class="p">,</span> <span class="s2">&quot;this is the second text&quot;</span><span class="p">]})</span>
@@ -4213,9 +4120,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_text_2</span> <span class="o">=</span> <span class="n">hf_processor_2</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="n">max_length</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">padding</span><span class="o">=</span><span class="s2">&quot;max_length&quot;</span><span class="p">,</span> <span class="n">truncation</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 84</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 84</span>
 <span class="normal"> 85</span>
 <span class="normal"> 86</span>
 <span class="normal"> 87</span>
@@ -4291,12 +4198,11 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
     <span class="c1"># attribute elsewhere in the library, we simply set it to True</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -4309,61 +4215,58 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">encode</span>
+            <span class="doc doc-object-name doc-function-name">encode</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Encodes a list of texts. The method is a wrapper around the
 <code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p>
 <p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>texts</code></b>
-            (<code><span title="typing.List">List</span>[str]</code>)
-        –
-        <div class="doc-md-description">
-          <p>List of texts to be encoded</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>texts</code></b>
+              (<code><span title="typing.List">List</span>[str]</code>)
+          –
+          <div class="doc-md-description">
+            <p>List of texts to be encoded</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
 of the HuggingFace's tokenizer. If the 'encode_params' dict was passed
 when instantiating the class, that dictionaly will be updated with
 the kwargs passed here.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.array">array</span></code>
-        –
-        <div class="doc-md-description">
-          <p>The encoded texts</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.array">array</span></code>
+          –
+          <div class="doc-md-description">
+            <p>The encoded texts</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">123</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">123</span>
 <span class="normal">124</span>
 <span class="normal">125</span>
 <span class="normal">126</span>
@@ -4483,68 +4386,64 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" cl
 
     <span class="k">return</span> <span class="n">output</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">decode</span>
+            <span class="doc doc-object-name doc-function-name">decode</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Decodes a list of input_ids. The method is a wrapper around the
 <code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the
 HuggingFace's tokenizer.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input_ids</code></b>
-            (<code><span title="numpy.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>)
-        –
-        <div class="doc-md-description">
-          <p>The input_ids to be decoded</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>skip_special_tokens</code></b>
-            (<code>bool</code>)
-        –
-        <div class="doc-md-description">
-          <p>Whether to skip the special tokens or not</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input_ids</code></b>
+              (<code><span title="numpy.typing.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>)
+          –
+          <div class="doc-md-description">
+            <p>The input_ids to be decoded</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>skip_special_tokens</code></b>
+              (<code>bool</code>)
+          –
+          <div class="doc-md-description">
+            <p>Whether to skip the special tokens or not</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="typing.List">List</span>[str]</code>
-        –
-        <div class="doc-md-description">
-          <p>The decoded texts</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="typing.List">List</span>[str]</code>
+          –
+          <div class="doc-md-description">
+            <p>The decoded texts</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">184</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">184</span>
 <span class="normal">185</span>
 <span class="normal">186</span>
 <span class="normal">187</span>
@@ -4598,26 +4497,24 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" cl
     <span class="p">]</span>
     <span class="k">return</span> <span class="n">texts</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit</span>
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>This method is included for consistency with the rest of the library
 in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's
 tokenizers and models are already trained. Therefore, the 'fit' method
@@ -4625,23 +4522,22 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class
 not <code>None</code>.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>The dataframe containing the text data in the column specified by
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>The dataframe containing the text data in the column specified by
 the 'text_col' parameter</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">212</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">212</span>
 <span class="normal">213</span>
 <span class="normal">214</span>
 <span class="normal">215</span>
@@ -4681,62 +4577,58 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class
         <span class="p">)</span>
     <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">transform</span>
+            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Encodes the text data in the input dataframe. This method simply
 calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method,
 this method is included for consistency with the rest of the library
 in general and with the <code>BasePreprocessor</code> in particular.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>The dataframe containing the text data in the column specified by
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>The dataframe containing the text data in the column specified by
 the 'text_col' parameter</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.array">array</span></code>
-        –
-        <div class="doc-md-description">
-          <p>The encoded texts</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.array">array</span></code>
+          –
+          <div class="doc-md-description">
+            <p>The encoded texts</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">233</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">233</span>
 <span class="normal">234</span>
 <span class="normal">235</span>
 <span class="normal">236</span>
@@ -4790,58 +4682,54 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform"
 
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">transform_sample</span>
+            <span class="doc doc-object-name doc-function-name">transform_sample</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">transform_sample</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform_sample</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Encodes a single text sample.</p>
-
+    <div class="doc doc-contents ">
 
+      <p>Encodes a single text sample.</p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>text</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>The text sample to be encoded</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>text</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>The text sample to be encoded</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.array">array</span></code>
-        –
-        <div class="doc-md-description">
-          <p>The encoded text</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.array">array</span></code>
+          –
+          <div class="doc-md-description">
+            <p>The encoded text</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">261</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">261</span>
 <span class="normal">262</span>
 <span class="normal">263</span>
 <span class="normal">264</span>
@@ -4881,59 +4769,55 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_
         <span class="p">)</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">([</span><span class="n">text</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit_transform</span>
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Encodes the text data in the input dataframe.</p>
+    <div class="doc doc-contents ">
 
+      <p>Encodes the text data in the input dataframe.</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>The dataframe containing the text data in the column specified by
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>The dataframe containing the text data in the column specified by
 the 'text_col' parameter</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.array">array</span></code>
-        –
-        <div class="doc-md-description">
-          <p>The encoded texts</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.array">array</span></code>
+          –
+          <div class="doc-md-description">
+            <p>The encoded texts</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">282</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">282</span>
 <span class="normal">283</span>
 <span class="normal">284</span>
 <span class="normal">285</span>
@@ -4965,67 +4849,63 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transf
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">inverse_transform</span>
+            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">inverse_transform</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method
 under the hood.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>input_ids</code></b>
-            (<code><span title="numpy.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>)
-        –
-        <div class="doc-md-description">
-          <p>The input_ids to be decoded</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>skip_special_tokens</code></b>
-            (<code>bool</code>)
-        –
-        <div class="doc-md-description">
-          <p>Whether to skip the special tokens or not</p>
-        </div>
-      </li>
-  </ul>
-
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>input_ids</code></b>
+              (<code><span title="numpy.typing.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>)
+          –
+          <div class="doc-md-description">
+            <p>The input_ids to be decoded</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>skip_special_tokens</code></b>
+              (<code>bool</code>)
+          –
+          <div class="doc-md-description">
+            <p>Whether to skip the special tokens or not</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="typing.List">List</span>[str]</code>
-        –
-        <div class="doc-md-description">
-          <p>The decoded texts</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="typing.List">List</span>[str]</code>
+          –
+          <div class="doc-md-description">
+            <p>The decoded texts</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">299</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">299</span>
 <span class="normal">300</span>
 <span class="normal">301</span>
 <span class="normal">302</span>
@@ -5065,8 +4945,8 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_tr
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -5074,8 +4954,7 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_tr
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -5084,110 +4963,107 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_tr
 
 
 <h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">ImagePreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">ImagePreprocessor</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">ImagePreprocessor</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ImagePreprocessor</span><span class="p">(</span>
     <span class="n">img_col</span><span class="p">,</span> <span class="n">img_path</span><span class="p">,</span> <span class="n">width</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+
 
-  
       <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p>
 <p>The Preprocessing consists simply on resizing according to their
 aspect ratio</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>img_col</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>name of the column with the images filenames</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>img_path</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>path to the dicrectory where the images are stored</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>width</code></b>
-            (<code>int</code>, default:
-                <code>224</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>img_col</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>name of the column with the images filenames</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>img_path</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>path to the dicrectory where the images are stored</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>width</code></b>
+              (<code>int</code>, default:
+                  <code>224</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>width of the resulting processed image.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>height</code></b>
-            (<code>int</code>, default:
-                <code>224</code>
+          –
+          <div class="doc-md-description">
+            <p>width of the resulting processed image.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>height</code></b>
+              (<code>int</code>, default:
+                  <code>224</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>width of the resulting processed image.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>width of the resulting processed image.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Enable verbose output.</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>Enable verbose output.</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.aap">aap</span></code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" href="utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor">AspectAwarePreprocessor</a></code>)
-        –
-        <div class="doc-md-description">
-          <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.spp">spp</span></code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.image_utils.SimplePreprocessor" href="utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor">SimplePreprocessor</a></code>)
-        –
-        <div class="doc-md-description">
-          <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.normalise_metrics">normalise_metrics</span></code></b>
-            (<code>Dict</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dict containing the normalisation metrics of the image dataset, i.e.
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.aap">aap</span></code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" href="utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor">AspectAwarePreprocessor</a></code>)
+          –
+          <div class="doc-md-description">
+            <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.spp">spp</span></code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.image_utils.SimplePreprocessor" href="utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor">SimplePreprocessor</a></code>)
+          –
+          <div class="doc-md-description">
+            <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.normalise_metrics">normalise_metrics</span></code></b>
+              (<code>Dict</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dict containing the normalisation metrics of the image dataset, i.e.
 mean and std for the R, G and B channels</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">ImagePreprocessor</span>
@@ -5207,9 +5083,9 @@ <h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" cla
 metrics and running <code>fit</code> only simply instantiates the resizing
 functions.</p>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">66</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">66</span>
 <span class="normal">67</span>
 <span class="normal">68</span>
 <span class="normal">69</span>
@@ -5249,12 +5125,11 @@ <h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" cla
 
     <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span> <span class="o">=</span> <span class="kc">False</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -5267,49 +5142,46 @@ <h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" cla
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">transform</span>
+            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Resizes the images to the input height and width.</p>
-
+    <div class="doc doc-contents ">
 
+      <p>Resizes the images to the input height and width.</p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe with the <code>img_col</code></p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe with the <code>img_col</code></p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Resized images to the input height and width</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Resized images to the input height and width</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">100</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">100</span>
 <span class="normal">101</span>
 <span class="normal">102</span>
 <span class="normal">103</span>
@@ -5439,58 +5311,54 @@ <h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.tran
         <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span> <span class="o">=</span> <span class="kc">True</span>
     <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">resized_imgs</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit_transform</span>
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Combines <code>fit</code> and <code>transform</code></p>
-
+    <div class="doc doc-contents ">
 
+      <p>Combines <code>fit</code> and <code>transform</code></p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Resized images to the input height and width</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Resized images to the input height and width</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">166</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">166</span>
 <span class="normal">167</span>
 <span class="normal">168</span>
 <span class="normal">169</span>
@@ -5518,8 +5386,8 @@ <h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -5527,8 +5395,7 @@ <h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_
 
   </div>
 
-  </div>
-
+    </div>
 
 </div><h2 id="chunked-versions">Chunked versions<a class="headerlink" href="#chunked-versions" title="Permanent link">&para;</a></h2>
 <p>Chunked versions of the preprocessors are also available. These are useful
@@ -5544,20 +5411,20 @@ <h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_
 
 
 <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">ChunkWidePreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">ChunkWidePreprocessor</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">ChunkWidePreprocessor</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ChunkWidePreprocessor</span><span class="p">(</span>
     <span class="n">wide_cols</span><span class="p">,</span> <span class="n">n_chunks</span><span class="p">,</span> <span class="n">crossed_cols</span><span class="o">=</span><span class="kc">None</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor">WidePreprocessor</a></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor">WidePreprocessor</a></code></p>
+
 
-  
       <p>Preprocessor to prepare the wide input dataset</p>
 <p>This Preprocessor prepares the data for the wide, linear component.
 This linear model is implemented via an Embedding layer that is
@@ -5566,75 +5433,72 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
 crossed_cols</code>. See the Example below.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>wide_cols</code></b>
-            (<code><span title="typing.List">List</span>[str]</code>)
-        –
-        <div class="doc-md-description">
-          <p>List of strings with the name of the columns that will label
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>wide_cols</code></b>
+              (<code><span title="typing.List">List</span>[str]</code>)
+          –
+          <div class="doc-md-description">
+            <p>List of strings with the name of the columns that will label
 encoded and passed through the <code>wide</code> component</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>crossed_cols</code></b>
-            (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>crossed_cols</code></b>
+              (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the name of the columns that will be <code>'crossed'</code>
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code>
 and then label encoded. e.g. <em>[('education', 'occupation'), ...]</em>. For
 binary features, a cross-product transformation is 1 if and only if
 the constituent features are all 1, and 0 otherwise.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.wide_crossed_cols">wide_crossed_cols</span></code></b>
-            (<code><span title="typing.List">List</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>List with the names of all columns that will be label encoded</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.encoding_dict">encoding_dict</span></code></b>
-            (<code>Dict</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dictionary where the keys are the result of pasting <code>colname + '_' +
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.wide_crossed_cols">wide_crossed_cols</span></code></b>
+              (<code><span title="typing.List">List</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>List with the names of all columns that will be label encoded</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.encoding_dict">encoding_dict</span></code></b>
+              (<code>Dict</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dictionary where the keys are the result of pasting <code>colname + '_' +
 column value</code> and the values are the corresponding mapped integer.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.inverse_encoding_dict">inverse_encoding_dict</span></code></b>
-            (<code>Dict</code>)
-        –
-        <div class="doc-md-description">
-          <p>the inverse encoding dictionary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.wide_dim">wide_dim</span></code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Dimension of the wide model (i.e. dim of the linear layer)</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.inverse_encoding_dict">inverse_encoding_dict</span></code></b>
+              (<code>Dict</code>)
+          –
+          <div class="doc-md-description">
+            <p>the inverse encoding dictionary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.wide_dim">wide_dim</span></code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Dimension of the wide model (i.e. dim of the linear layer)</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">ChunkWidePreprocessor</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">chunk</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;color&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="s1">&#39;g&#39;</span><span class="p">],</span> <span class="s1">&#39;size&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;s&#39;</span><span class="p">,</span> <span class="s1">&#39;n&#39;</span><span class="p">,</span> <span class="s1">&#39;l&#39;</span><span class="p">]})</span>
@@ -5645,9 +5509,9 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_wide</span> <span class="o">=</span> <span class="n">chunk_wide_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">254</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">254</span>
 <span class="normal">255</span>
 <span class="normal">256</span>
 <span class="normal">257</span>
@@ -5673,12 +5537,11 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
 
     <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -5691,49 +5554,46 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">partial_fit</span>
+            <span class="doc doc-object-name doc-function-name">partial_fit</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">partial_fit</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">partial_fit</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Fits the Preprocessor and creates required attributes</p>
-
+    <div class="doc doc-contents ">
 
+      <p>Fits the Preprocessor and creates required attributes</p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>chunk</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input pandas dataframe</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>chunk</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input pandas dataframe</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor">ChunkWidePreprocessor</a></code>
-        –
-        <div class="doc-md-description">
-          <p><code>ChunkWidePreprocessor</code> fitted object</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor">ChunkWidePreprocessor</a></code>
+          –
+          <div class="doc-md-description">
+            <p><code>ChunkWidePreprocessor</code> fitted object</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">268</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">268</span>
 <span class="normal">269</span>
 <span class="normal">270</span>
 <span class="normal">271</span>
@@ -5805,32 +5665,30 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.p
 
     <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit</span>
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Runs <code>partial_fit</code>. This is just to override the fit method in the base
 class. This class is not designed or thought to run fit</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">305</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">305</span>
 <span class="normal">306</span>
 <span class="normal">307</span>
 <span class="normal">308</span>
@@ -5842,8 +5700,8 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.f
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -5851,8 +5709,7 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.f
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -5861,11 +5718,11 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.f
 
 
 <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">ChunkTabPreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">ChunkTabPreprocessor</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">ChunkTabPreprocessor</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ChunkTabPreprocessor</span><span class="p">(</span>
     <span class="n">n_chunks</span><span class="p">,</span>
     <span class="n">cat_embed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -5883,92 +5740,91 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code></p>
 
-  
-      <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p>
 
+      <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>n_chunks</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Number of chunks that the tabular dataset is divided by.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_embed_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>n_chunks</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Number of chunks that the tabular dataset is divided by.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_embed_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List containing the name of the categorical columns that will be
+          –
+          <div class="doc-md-description">
+            <p>List containing the name of the categorical columns that will be
 represented by embeddings (e.g. <em>['education', 'relationship', ...]</em>) or
 a Tuple with the name and the embedding dimension (e.g.: <em>[
 ('education',32), ('relationship',16), ...]</em>)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>continuous_cols</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>continuous_cols</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with the name of the continuous cols</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cols_and_bins</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>List with the name of the continuous cols</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cols_and_bins</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Continuous columns can be turned into categorical via
+          –
+          <div class="doc-md-description">
+            <p>Continuous columns can be turned into categorical via
 <code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column
 names to quantize and the values are a list of scalars indicating the
 bin edges.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cols_to_scale</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cols_to_scale</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with the names of the columns that will be standarised via
+          –
+          <div class="doc-md-description">
+            <p>List with the names of the columns that will be standarised via
 sklearn's <code>StandardScaler</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>default_embed_dim</code></b>
-            (<code>int</code>, default:
-                <code>16</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>default_embed_dim</code></b>
+              (<code>int</code>, default:
+                  <code>16</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Dimension for the embeddings if the embed_dim is not provided in the
+          –
+          <div class="doc-md-description">
+            <p>Dimension for the embeddings if the embed_dim is not provided in the
 <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to
 <code>False</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_attention</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>with_attention</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the preprocessed data will be passed to an
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the preprocessed data will be passed to an
 attention-based model (more precisely a model where all embeddings
 must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code>
 must just be a list containing just the categorical column names:
@@ -5977,165 +5833,162 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
  encoded using embeddings of the same dim, which will be specified
  later when the model is defined. <br/> Param alias:
  <code>for_transformer</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_cls_token</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>with_cls_token</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset
 when using attention-based models. The final hidden state
 corresponding to this token is used as the aggregated representation
 for classification and regression tasks. If not, the categorical
 (and continuous embeddings if present) will be concatenated before
 being passed to the final MLP (if present).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>shared_embed</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>shared_embed</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the embeddings will be "shared" when using
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the embeddings will be "shared" when using
 attention-based models. The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>scale</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          –
+          <div class="doc-md-description">
+            
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>scale</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
+          –
+          <div class="doc-md-description">
+            <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
  releases. Please use <code>cols_to_scale</code> instead. <br/> Bool indicating
  whether or not to scale/standarise continuous cols. It is important
  to emphasize that all the DL models for tabular data in the library
  also include the possibility of normalising the input continuous
  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>. <br/> Param alias:
  <code>scale_cont_cols</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>already_standard</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>already_standard</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
+          –
+          <div class="doc-md-description">
+            <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
  releases. Please use <code>cols_to_scale</code> instead. <br/> List with the
  name of the continuous cols that do not need to be
  scaled/standarised.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Other Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Other Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.embed_dim">embed_dim</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Dictionary where keys are the embed cols and values are the embedding
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.embed_dim">embed_dim</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Dictionary where keys are the embed cols and values are the embedding
 dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute
 is not generated during the <code>fit</code> process</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.label_encoder">label_encoder</span></code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>)
-        –
-        <div class="doc-md-description">
-          <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.cat_embed_input">cat_embed_input</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>List of Tuples with the column name, number of individual values for
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.label_encoder">label_encoder</span></code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>)
+          –
+          <div class="doc-md-description">
+            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.cat_embed_input">cat_embed_input</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>List of Tuples with the column name, number of individual values for
 that column and, If <code>with_attention</code> is set to <code>False</code>, the
 corresponding embeddings dim, e.g. <em>[('education', 16, 10),
 ('relationship', 6, 8), ...]</em>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.standardize_cols">standardize_cols</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>List of the columns that will be standarized</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.scaler">scaler</span></code></b>
-            (<code><span title="sklearn.preprocessing.StandardScaler">StandardScaler</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>an instance of <code>sklearn.preprocessing.StandardScaler</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.standardize_cols">standardize_cols</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>List of the columns that will be standarized</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.scaler">scaler</span></code></b>
+              (<code><span title="sklearn.preprocessing.StandardScaler">StandardScaler</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code>
 if 'cols_to_scale' is not None or 'scale' is 'True'</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.column_idx">column_idx</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Dictionary where keys are column names and values are column indexes.
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.column_idx">column_idx</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Dictionary where keys are column names and values are column indexes.
 This is neccesary to slice tensors</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.quantizer">quantizer</span></code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer">Quantizer</a></code>)
-        –
-        <div class="doc-md-description">
-          <p>an instance of <code>Quantizer</code></p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.quantizer">quantizer</span></code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer">Quantizer</a></code>)
+          –
+          <div class="doc-md-description">
+            <p>an instance of <code>Quantizer</code></p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">ChunkTabPreprocessor</span>
@@ -6154,9 +6007,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
 <span class="go">{&#39;cat_col&#39;: 0, &#39;cont_col&#39;: 1}</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">825</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">825</span>
 <span class="normal">826</span>
 <span class="normal">827</span>
 <span class="normal">828</span>
@@ -6250,12 +6103,11 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
     <span class="bp">self</span><span class="o">.</span><span class="n">embed_prepared</span> <span class="o">=</span> <span class="kc">False</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">continuous_prepared</span> <span class="o">=</span> <span class="kc">False</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -6266,10 +6118,10 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -6278,11 +6130,11 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
 
 
 <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">ChunkTextPreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">ChunkTextPreprocessor</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">ChunkTextPreprocessor</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ChunkTextPreprocessor</span><span class="p">(</span>
     <span class="n">text_col</span><span class="p">,</span>
     <span class="n">n_chunks</span><span class="p">,</span>
     <span class="n">root_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -6298,153 +6150,150 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor">TextPreprocessor</a></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor">TextPreprocessor</a></code></p>
 
-  
-      <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p>
 
+      <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>text_col</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>column in the input dataframe containing either the texts or the
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>text_col</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>column in the input dataframe containing either the texts or the
 filenames where the text documents are stored</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_chunks</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Number of chunks that the text dataset is divided by.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>root_dir</code></b>
-            (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_chunks</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Number of chunks that the text dataset is divided by.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>root_dir</code></b>
+              (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>If 'text_col' contains the filenames with the text documents, this is
+          –
+          <div class="doc-md-description">
+            <p>If 'text_col' contains the filenames with the text documents, this is
 the path to the directory where those documents are stored.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>max_vocab</code></b>
-            (<code>int</code>, default:
-                <code>30000</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>max_vocab</code></b>
+              (<code>int</code>, default:
+                  <code>30000</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Maximum number of tokens in the vocabulary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>min_freq</code></b>
-            (<code>int</code>, default:
-                <code>5</code>
+          –
+          <div class="doc-md-description">
+            <p>Maximum number of tokens in the vocabulary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>min_freq</code></b>
+              (<code>int</code>, default:
+                  <code>5</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Minimum frequency for a token to be part of the vocabulary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>maxlen</code></b>
-            (<code>int</code>, default:
-                <code>80</code>
+          –
+          <div class="doc-md-description">
+            <p>Minimum frequency for a token to be part of the vocabulary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>maxlen</code></b>
+              (<code>int</code>, default:
+                  <code>80</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Maximum length of the tokenized sequences</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pad_first</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
+          –
+          <div class="doc-md-description">
+            <p>Maximum length of the tokenized sequences</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pad_first</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Indicates whether the padding index will be added at the beginning or the
+          –
+          <div class="doc-md-description">
+            <p>Indicates whether the padding index will be added at the beginning or the
 end of the sequences</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pad_idx</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pad_idx</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>word_vectors_path</code></b>
-            (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>word_vectors_path</code></b>
+              (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Path to the pretrained word vectors</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_cpus</code></b>
-            (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>Path to the pretrained word vectors</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_cpus</code></b>
+              (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of CPUs to used during the tokenization process</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>number of CPUs to used during the tokenization process</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Enable verbose output.</p>
-        </div>
-      </li>
-  </ul>
-
-
+          –
+          <div class="doc-md-description">
+            <p>Enable verbose output.</p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor.vocab">vocab</span></code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>)
-        –
-        <div class="doc-md-description">
-          <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor.embedding_matrix">embedding_matrix</span></code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor.vocab">vocab</span></code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>)
+          –
+          <div class="doc-md-description">
+            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor.embedding_matrix">embedding_matrix</span></code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">ChunkTextPreprocessor</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">chunk_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;text_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;life is like a box of chocolates&quot;</span><span class="p">,</span>
@@ -6454,9 +6303,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">processed_chunk</span> <span class="o">=</span> <span class="n">chunk_text_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">chunk_df</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">313</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">313</span>
 <span class="normal">314</span>
 <span class="normal">315</span>
 <span class="normal">316</span>
@@ -6524,12 +6373,11 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
 
     <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -6540,10 +6388,10 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -6552,11 +6400,11 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
 
 
 <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">ChunkHFPreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">ChunkHFPreprocessor</span>
 
 
 <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">ChunkHFPreprocessor</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ChunkHFPreprocessor</span><span class="p">(</span>
     <span class="n">model_name</span><span class="p">,</span>
     <span class="o">*</span><span class="p">,</span>
     <span class="n">text_col</span><span class="p">,</span>
@@ -6569,11 +6417,11 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor">HFPreprocessor</a></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor">HFPreprocessor</a></code></p>
+
 
-  
       <p>Text processor to prepare the <code>deeptext</code> input dataset that is a
 wrapper around HuggingFace's tokenizers.</p>
 <p>Hugginface Tokenizer's are already 'trained'. Therefore, unlike the
@@ -6585,124 +6433,122 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
  all sequences have the same length when encoding in chunks.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>model_name</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>The model name from the transformers library e.g. <em>'bert-base-uncased'</em>.
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>model_name</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>The model name from the transformers library e.g. <em>'bert-base-uncased'</em>.
 Currently supported models are those from the families: BERT, RoBERTa,
 DistilBERT, ALBERT and ELECTRA.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>text_col</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>The column in the input dataframe containing the text data. When using
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>text_col</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>The column in the input dataframe containing the text data. When using
 the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>root_dir</code></b>
-            (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>root_dir</code></b>
+              (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The root directory where the text files are located. This is only
+          –
+          <div class="doc-md-description">
+            <p>The root directory where the text files are located. This is only
 needed if the text data is stored in text files. If the text data is
 stored in a column in the input dataframe, this parameter is not
 needed.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>use_fast_tokenizer</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>use_fast_tokenizer</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Whether to use the fast tokenizer from HuggingFace or not</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>num_workers</code></b>
-            (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>Whether to use the fast tokenizer from HuggingFace or not</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>num_workers</code></b>
+              (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Number of workers to use when preprocessing the text data. If not
+          –
+          <div class="doc-md-description">
+            <p>Number of workers to use when preprocessing the text data. If not
 None, and <code>use_fast_tokenizer</code> is False, the text data will be
 preprocessed in parallel using the number of workers specified. If
 <code>use_fast_tokenizer</code> is True, this argument is ignored.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>preprocessing_rules</code></b>
-            (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>preprocessing_rules</code></b>
+              (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>A list of functions to be applied to the text data before encoding.
+          –
+          <div class="doc-md-description">
+            <p>A list of functions to be applied to the text data before encoding.
 This can be useful to clean the text data before encoding. For
 example, removing html tags, special characters, etc.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>tokenizer_params</code></b>
-            (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>tokenizer_params</code></b>
+              (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Additional parameters to be passed to the HuggingFace's
+          –
+          <div class="doc-md-description">
+            <p>Additional parameters to be passed to the HuggingFace's
 <code>PreTrainedTokenizer</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>encode_params</code></b>
-            (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>encode_params</code></b>
+              (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
+          –
+          <div class="doc-md-description">
+            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
 of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the
 <code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It
 must be passed containing at least the 'max_length' encoding
 parameter</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor.is_fitted">is_fitted</span></code></b>
-            (<code>bool</code>)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the preprocessor has been fitted. This is a
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor.is_fitted">is_fitted</span></code></b>
+              (<code>bool</code>)
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the preprocessor has been fitted. This is a
 HuggingFacea tokenizer, so it is always considered fitted and this
 attribute is manually set to True internally. This parameter exists
 for consistency with the rest of the library and because is needed
 for some functionality in the library.</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">427</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">427</span>
 <span class="normal">428</span>
 <span class="normal">429</span>
 <span class="normal">430</span>
@@ -6782,12 +6628,11 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
     <span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;truncation&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -6798,10 +6643,10 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
 
 
 
-  </div>
 
   </div>
 
+    </div>
 
 </div>
 
@@ -6877,7 +6722,7 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -6895,7 +6740,7 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html b/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html
index 30e02fd0..91279c47 100644
--- a/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html
+++ b/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1036,8 +1036,6 @@
         
       
         
-      
-        
       
         
       
@@ -1375,11 +1373,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1396,11 +1394,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1417,11 +1415,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1438,11 +1436,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1459,11 +1457,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1480,11 +1478,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1501,11 +1499,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1522,32 +1520,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1649,11 +1626,11 @@ <h1 id="self-supervised-pre-training-for-tabular-data">Self Supervised Pre-train
 
 
 <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">EncoderDecoderTrainer</span>
+            <span class="doc doc-object-name doc-class-name">EncoderDecoderTrainer</span>
 
 
 <a href="#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">EncoderDecoderTrainer</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">EncoderDecoderTrainer</span><span class="p">(</span>
     <span class="n">encoder</span><span class="p">,</span>
     <span class="n">decoder</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">masked_prob</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span>
@@ -1666,118 +1643,116 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.self_supervised_training._base_encoder_decoder_trainer.BaseEncoderDecoderTrainer">BaseEncoderDecoderTrainer</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.self_supervised_training._base_encoder_decoder_trainer.BaseEncoderDecoderTrainer">BaseEncoderDecoderTrainer</span></code></p>
+
 
-  
       <p>This class implements an Encoder-Decoder self-supervised 'routine'
 inspired by
 <a href="https://arxiv.org/abs/1908.07442">TabNet: Attentive Interpretable Tabular Learning</a>.
 See Figure 1 above.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>encoder</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.ModelWithoutAttention">ModelWithoutAttention</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>decoder</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.DecoderWithoutAttention">DecoderWithoutAttention</span>]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>encoder</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.ModelWithoutAttention">ModelWithoutAttention</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>decoder</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.DecoderWithoutAttention">DecoderWithoutAttention</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or
+          –
+          <div class="doc-md-description">
+            <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or
 <code>TabNetDecoder</code> model. if <code>None</code> the decoder will be automatically
 built as a '<em>simetric</em>' model to the Encoder</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>masked_prob</code></b>
-            (<code>float</code>, default:
-                <code>0.2</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>masked_prob</code></b>
+              (<code>float</code>, default:
+                  <code>0.2</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Indicates the fraction of elements in the embedding tensor that will
+          –
+          <div class="doc-md-description">
+            <p>Indicates the fraction of elements in the embedding tensor that will
 be masked and hence used for reconstruction</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>optimizer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>optimizer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam
+          –
+          <div class="doc-md-description">
+            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam
 ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lr_scheduler</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>lr_scheduler</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>An instance of Pytorch's <code>LRScheduler</code> object
+          –
+          <div class="doc-md-description">
+            <p>An instance of Pytorch's <code>LRScheduler</code> object
 (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>callbacks</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>callbacks</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with <code>Callback</code> objects. The three callbacks available in
+          –
+          <div class="doc-md-description">
+            <p>List with <code>Callback</code> objects. The three callbacks available in
 <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and
 <code>EarlyStopping</code>. This can also be a custom callback. See
 <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the
 repo.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Setting it to 0 will print nothing during training.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>seed</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>Setting it to 0 will print nothing during training.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>seed</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Random seed to be used internally for train_test_split</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>Random seed to be used internally for train_test_split</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Other Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
+<p><span class="doc-section-title">Other Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
 <ul>
 <li>
 <p><strong>device</strong>: <code>str</code><br/>
@@ -1794,13 +1769,16 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
     learning rate is a bit particular.</p>
 </li>
 </ul>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 83</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
 <span class="normal"> 84</span>
 <span class="normal"> 85</span>
 <span class="normal"> 86</span>
@@ -1819,10 +1797,7 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
 <span class="normal"> 99</span>
 <span class="normal">100</span>
 <span class="normal">101</span>
-<span class="normal">102</span>
-<span class="normal">103</span>
-<span class="normal">104</span>
-<span class="normal">105</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+<span class="normal">102</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">encoder</span><span class="p">:</span> <span class="n">ModelWithoutAttention</span><span class="p">,</span>
     <span class="n">decoder</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DecoderWithoutAttention</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -1846,12 +1821,11 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
         <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1864,13 +1838,12 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">pretrain</span>
+            <span class="doc doc-object-name doc-function-name">pretrain</span>
 
 
 <a href="#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">pretrain</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">pretrain</span><span class="p">(</span>
     <span class="n">X_tab</span><span class="p">,</span>
     <span class="n">X_tab_val</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">val_split</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -1880,78 +1853,80 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p>
+    <div class="doc doc-contents ">
 
+      <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>X_tab</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>tabular dataset</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_tab_val</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>tabular dataset</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab_val</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>validation data</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>val_split</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>validation data</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>val_split</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>An alterative to passing the validation set is to use a train/val
+          –
+          <div class="doc-md-description">
+            <p>An alterative to passing the validation set is to use a train/val
 split fraction via <code>val_split</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>validation_freq</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>validation_freq</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>epochs validation frequency</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_epochs</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>epochs validation frequency</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_epochs</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of epochs</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>batch_size</code></b>
-            (<code>int</code>, default:
-                <code>32</code>
+          –
+          <div class="doc-md-description">
+            <p>number of epochs</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>batch_size</code></b>
+              (<code>int</code>, default:
+                  <code>32</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>batch size</p>
-        </div>
-      </li>
-  </ul>
+          –
+          <div class="doc-md-description">
+            <p>batch size</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">107</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
 <span class="normal">108</span>
 <span class="normal">109</span>
 <span class="normal">110</span>
@@ -2040,10 +2015,7 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
 <span class="normal">193</span>
 <span class="normal">194</span>
 <span class="normal">195</span>
-<span class="normal">196</span>
-<span class="normal">197</span>
-<span class="normal">198</span>
-<span class="normal">199</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pretrain</span><span class="p">(</span>
+<span class="normal">196</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pretrain</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
     <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -2137,150 +2109,8 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
     <span class="bp">self</span><span class="o">.</span><span class="n">_restore_best_weights</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">ed_model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
-
-</div>
-
-
-<div class="doc doc-object doc-function">
-
-
-
-<h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.save" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">save</span>
-
-
-<a href="#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.save" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">save</span><span class="p">(</span>
-    <span class="n">path</span><span class="p">,</span>
-    <span class="n">save_state_dict</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">model_filename</span><span class="o">=</span><span class="s2">&quot;ed_model.pt&quot;</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
-
-  <div class="doc doc-contents ">
-  
-      <p>Saves the model, training and evaluation history (if any) to disk</p>
-
-
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>path</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>path to the directory where the model and the feature importance
-attribute will be saved.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>save_state_dict</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether to save directly the model or the
-model's state dictionary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>model_filename</code></b>
-            (<code>str</code>, default:
-                <code>&#39;ed_model.pt&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>filename where the model weights will be store</p>
-        </div>
-      </li>
-  </ul>
-
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">214</span>
-<span class="normal">215</span>
-<span class="normal">216</span>
-<span class="normal">217</span>
-<span class="normal">218</span>
-<span class="normal">219</span>
-<span class="normal">220</span>
-<span class="normal">221</span>
-<span class="normal">222</span>
-<span class="normal">223</span>
-<span class="normal">224</span>
-<span class="normal">225</span>
-<span class="normal">226</span>
-<span class="normal">227</span>
-<span class="normal">228</span>
-<span class="normal">229</span>
-<span class="normal">230</span>
-<span class="normal">231</span>
-<span class="normal">232</span>
-<span class="normal">233</span>
-<span class="normal">234</span>
-<span class="normal">235</span>
-<span class="normal">236</span>
-<span class="normal">237</span>
-<span class="normal">238</span>
-<span class="normal">239</span>
-<span class="normal">240</span>
-<span class="normal">241</span>
-<span class="normal">242</span>
-<span class="normal">243</span>
-<span class="normal">244</span>
-<span class="normal">245</span>
-<span class="normal">246</span>
-<span class="normal">247</span>
-<span class="normal">248</span>
-<span class="normal">249</span>
-<span class="normal">250</span>
-<span class="normal">251</span>
-<span class="normal">252</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">save</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">save_state_dict</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">model_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;ed_model.pt&quot;</span><span class="p">,</span>
-<span class="p">):</span>
-<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Saves the model, training and evaluation history (if any) to disk</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    path: str</span>
-<span class="sd">        path to the directory where the model and the feature importance</span>
-<span class="sd">        attribute will be saved.</span>
-<span class="sd">    save_state_dict: bool, default = False</span>
-<span class="sd">        Boolean indicating whether to save directly the model or the</span>
-<span class="sd">        model&#39;s state dictionary</span>
-<span class="sd">    model_filename: str, Optional, default = &quot;ed_model.pt&quot;</span>
-<span class="sd">        filename where the model weights will be store</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">save_dir</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
-    <span class="n">history_dir</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="s2">&quot;history&quot;</span>
-    <span class="n">history_dir</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-
-    <span class="c1"># the trainer is run with the History Callback by default</span>
-    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">history_dir</span> <span class="o">/</span> <span class="s2">&quot;train_eval_history.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">teh</span><span class="p">:</span>
-        <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">history</span><span class="p">,</span> <span class="n">teh</span><span class="p">)</span>  <span class="c1"># type: ignore[attr-defined]</span>
-
-    <span class="n">has_lr_history</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span>
-        <span class="p">[</span><span class="n">clbk</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;LRHistory&quot;</span> <span class="k">for</span> <span class="n">clbk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">callbacks</span><span class="p">]</span>
-    <span class="p">)</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">lr_scheduler</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">has_lr_history</span><span class="p">:</span>
-        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">history_dir</span> <span class="o">/</span> <span class="s2">&quot;lr_history.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">lrh</span><span class="p">:</span>
-            <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_history</span><span class="p">,</span> <span class="n">lrh</span><span class="p">)</span>  <span class="c1"># type: ignore[attr-defined]</span>
-
-    <span class="n">model_path</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="n">model_filename</span>
-    <span class="k">if</span> <span class="n">save_state_dict</span><span class="p">:</span>
-        <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ed_model</span><span class="o">.</span><span class="n">state_dict</span><span class="p">(),</span> <span class="n">model_path</span><span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ed_model</span><span class="p">,</span> <span class="n">model_path</span><span class="p">)</span>
-</code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2288,8 +2118,7 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.save" cl
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2298,11 +2127,11 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.save" cl
 
 
 <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">ContrastiveDenoisingTrainer</span>
+            <span class="doc doc-object-name doc-class-name">ContrastiveDenoisingTrainer</span>
 
 
 <a href="#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">ContrastiveDenoisingTrainer</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ContrastiveDenoisingTrainer</span><span class="p">(</span>
     <span class="n">model</span><span class="p">,</span>
     <span class="n">preprocessor</span><span class="p">,</span>
     <span class="n">optimizer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -2321,96 +2150,95 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.self_supervised_training._base_contrastive_denoising_trainer.BaseContrastiveDenoisingTrainer">BaseContrastiveDenoisingTrainer</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.self_supervised_training._base_contrastive_denoising_trainer.BaseContrastiveDenoisingTrainer">BaseContrastiveDenoisingTrainer</span></code></p>
+
 
-  
       <p>This class trains a Contrastive, Denoising Self Supervised 'routine' that
 is based on the one described in
 <a href="https://arxiv.org/abs/2106.01342">SAINT: Improved Neural Networks for Tabular Data via Row Attention and
 Contrastive Pre-Training</a>, their Figure 1.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>model</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.ModelWithAttention">ModelWithAttention</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>,
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>model</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.ModelWithAttention">ModelWithAttention</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>,
 <code>TabFastFormer</code>, <code>TabPerceiver</code>, <code>ContextAttentionMLP</code> and
 <code>SelfAttentionMLP</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>preprocessor</code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.TabPreprocessor" href="preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>)
-        –
-        <div class="doc-md-description">
-          <p>A fitted <code>TabPreprocessor</code> object. See
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>preprocessor</code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.TabPreprocessor" href="preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>)
+          –
+          <div class="doc-md-description">
+            <p>A fitted <code>TabPreprocessor</code> object. See
 <code>pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>optimizer</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>optimizer</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam
+          –
+          <div class="doc-md-description">
+            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam
 ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lr_scheduler</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>lr_scheduler</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>An instance of Pytorch's <code>LRScheduler</code> object
+          –
+          <div class="doc-md-description">
+            <p>An instance of Pytorch's <code>LRScheduler</code> object
 (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>callbacks</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>callbacks</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with <code>Callback</code> objects. The three callbacks available in
+          –
+          <div class="doc-md-description">
+            <p>List with <code>Callback</code> objects. The three callbacks available in
 <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and
 <code>EarlyStopping</code>. This can also be a custom callback. See
 <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the
 repo.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>loss_type</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[contrastive, denoising, both]</code>, default:
-                <code>&#39;both&#39;</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>loss_type</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[contrastive, denoising, both]</code>, default:
+                  <code>&#39;both&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>One of '<em>contrastive</em>', '<em>denoising</em>' or '<em>both</em>'. See <a href="https://arxiv.org/abs/2203.05556">SAINT: Improved
+          –
+          <div class="doc-md-description">
+            <p>One of '<em>contrastive</em>', '<em>denoising</em>' or '<em>both</em>'. See <a href="https://arxiv.org/abs/2203.05556">SAINT: Improved
 Neural Networks for Tabular Data via Row Attention and Contrastive
 Pre-Training</a>, their figure (1)
 and their equation (5).</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>projection_head1_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>projection_head1_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The projection heads are simply MLPs. This parameter is a list
+          –
+          <div class="doc-md-description">
+            <p>The projection heads are simply MLPs. This parameter is a list
 of integers with the dimensions of the MLP hidden layers. See the
 <a href="https://arxiv.org/abs/2203.05556">paper</a> for details. Note that
 setting up this parameter requires some knowledge of the architecture
@@ -2418,94 +2246,93 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
 embeddings of dim 32 (i.e. the so called dimension of the model is
 32), then the first dimension of the projection head must be 32 (e.g.
 [32, 16])</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>projection_head2_dims</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>projection_head2_dims</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Same as '<em>projection_head1_dims</em>' for the second head</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>projection_heads_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>Same as '<em>projection_head1_dims</em>' for the second head</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>projection_heads_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Activation function for the projection heads</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cat_mlp_type</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[single, multiple]</code>, default:
-                <code>&#39;multiple&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>Activation function for the projection heads</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cat_mlp_type</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[single, multiple]</code>, default:
+                  <code>&#39;multiple&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>If '<em>denoising</em>' loss is used, one can choose two types of 'stacked'
+          –
+          <div class="doc-md-description">
+            <p>If '<em>denoising</em>' loss is used, one can choose two types of 'stacked'
 MLPs to process the output from the transformer-based encoder that
 receives 'corrupted' (cut-mixed and mixed-up) features. These
 are '<em>single</em>' or '<em>multiple</em>'. The former approach will apply a single
 MLP to all the categorical features while the latter will use one MLP
 per categorical feature</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>cont_mlp_type</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[single, multiple]</code>, default:
-                <code>&#39;multiple&#39;</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>cont_mlp_type</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[single, multiple]</code>, default:
+                  <code>&#39;multiple&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Same as 'cat_mlp_type' but for the continuous features</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>denoise_mlps_activation</code></b>
-            (<code>str</code>, default:
-                <code>&#39;relu&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>Same as 'cat_mlp_type' but for the continuous features</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>denoise_mlps_activation</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;relu&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>activation function for the so called 'denoising mlps'.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>activation function for the so called 'denoising mlps'.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Setting it to 0 will print nothing during training.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>seed</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>Setting it to 0 will print nothing during training.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>seed</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Random seed to be used internally for train_test_split</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>Random seed to be used internally for train_test_split</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Other Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
+<p><span class="doc-section-title">Other Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
 <ul>
 <li>
 <p><strong>device</strong>: <code>str</code><br/>
@@ -2522,13 +2349,16 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
     learning rate is a bit particular.</p>
 </li>
 </ul>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">111</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
 <span class="normal">112</span>
 <span class="normal">113</span>
 <span class="normal">114</span>
@@ -2559,10 +2389,7 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
 <span class="normal">139</span>
 <span class="normal">140</span>
 <span class="normal">141</span>
-<span class="normal">142</span>
-<span class="normal">143</span>
-<span class="normal">144</span>
-<span class="normal">145</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+<span class="normal">142</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">model</span><span class="p">:</span> <span class="n">ModelWithAttention</span><span class="p">,</span>
     <span class="n">preprocessor</span><span class="p">:</span> <span class="n">TabPreprocessor</span><span class="p">,</span>
@@ -2598,12 +2425,11 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
         <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2616,13 +2442,12 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pretrain" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">pretrain</span>
+            <span class="doc doc-object-name doc-function-name">pretrain</span>
 
 
 <a href="#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pretrain" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">pretrain</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">pretrain</span><span class="p">(</span>
     <span class="n">X_tab</span><span class="p">,</span>
     <span class="n">X_tab_val</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">val_split</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -2632,30 +2457,29 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p>
+    <div class="doc doc-contents ">
 
+      <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>X_tab</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>tabular dataset</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_tab_val</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>tabular dataset</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab_val</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>validation data. Note that, although it is possible to use
+          –
+          <div class="doc-md-description">
+            <p>validation data. Note that, although it is possible to use
 contrastive-denoising training with a validation set, such set
 must include feature values that are <em>all</em> seen in the training
 set in the case of the categorical columns. This is because the
@@ -2664,54 +2488,57 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
 the validation set that was not seen in training this will
 effectively be like trying to predict a new, never seen category
 (and Pytorch will throw an error)</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>val_split</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>val_split</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>An alterative to passing the validation set is to use a train/val
+          –
+          <div class="doc-md-description">
+            <p>An alterative to passing the validation set is to use a train/val
 split fraction via <code>val_split</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>validation_freq</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>validation_freq</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>epochs validation frequency</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_epochs</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>epochs validation frequency</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_epochs</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of epochs</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>batch_size</code></b>
-            (<code>int</code>, default:
-                <code>32</code>
+          –
+          <div class="doc-md-description">
+            <p>number of epochs</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>batch_size</code></b>
+              (<code>int</code>, default:
+                  <code>32</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>batch size</p>
-        </div>
-      </li>
-  </ul>
+          –
+          <div class="doc-md-description">
+            <p>batch size</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">147</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
 <span class="normal">148</span>
 <span class="normal">149</span>
 <span class="normal">150</span>
@@ -2808,10 +2635,7 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
 <span class="normal">241</span>
 <span class="normal">242</span>
 <span class="normal">243</span>
-<span class="normal">244</span>
-<span class="normal">245</span>
-<span class="normal">246</span>
-<span class="normal">247</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pretrain</span><span class="p">(</span>
+<span class="normal">244</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pretrain</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
     <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -2913,150 +2737,8 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
     <span class="bp">self</span><span class="o">.</span><span class="n">_restore_best_weights</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">cd_model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
-
-</div>
-
-
-<div class="doc doc-object doc-function">
-
-
-
-<h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.save" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">save</span>
-
-
-<a href="#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.save" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">save</span><span class="p">(</span>
-    <span class="n">path</span><span class="p">,</span>
-    <span class="n">save_state_dict</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">model_filename</span><span class="o">=</span><span class="s2">&quot;cd_model.pt&quot;</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
-
-  <div class="doc doc-contents ">
-  
-      <p>Saves the model, training and evaluation history (if any) to disk</p>
-
-
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>path</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>path to the directory where the model and the feature importance
-attribute will be saved.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>save_state_dict</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether to save directly the model or the
-model's state dictionary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>model_filename</code></b>
-            (<code>str</code>, default:
-                <code>&#39;cd_model.pt&#39;</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>filename where the model weights will be store</p>
-        </div>
-      </li>
-  </ul>
-
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">262</span>
-<span class="normal">263</span>
-<span class="normal">264</span>
-<span class="normal">265</span>
-<span class="normal">266</span>
-<span class="normal">267</span>
-<span class="normal">268</span>
-<span class="normal">269</span>
-<span class="normal">270</span>
-<span class="normal">271</span>
-<span class="normal">272</span>
-<span class="normal">273</span>
-<span class="normal">274</span>
-<span class="normal">275</span>
-<span class="normal">276</span>
-<span class="normal">277</span>
-<span class="normal">278</span>
-<span class="normal">279</span>
-<span class="normal">280</span>
-<span class="normal">281</span>
-<span class="normal">282</span>
-<span class="normal">283</span>
-<span class="normal">284</span>
-<span class="normal">285</span>
-<span class="normal">286</span>
-<span class="normal">287</span>
-<span class="normal">288</span>
-<span class="normal">289</span>
-<span class="normal">290</span>
-<span class="normal">291</span>
-<span class="normal">292</span>
-<span class="normal">293</span>
-<span class="normal">294</span>
-<span class="normal">295</span>
-<span class="normal">296</span>
-<span class="normal">297</span>
-<span class="normal">298</span>
-<span class="normal">299</span>
-<span class="normal">300</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">save</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">save_state_dict</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">model_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;cd_model.pt&quot;</span><span class="p">,</span>
-<span class="p">):</span>
-<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Saves the model, training and evaluation history (if any) to disk</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    path: str</span>
-<span class="sd">        path to the directory where the model and the feature importance</span>
-<span class="sd">        attribute will be saved.</span>
-<span class="sd">    save_state_dict: bool, default = False</span>
-<span class="sd">        Boolean indicating whether to save directly the model or the</span>
-<span class="sd">        model&#39;s state dictionary</span>
-<span class="sd">    model_filename: str, Optional, default = &quot;cd_model.pt&quot;</span>
-<span class="sd">        filename where the model weights will be store</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">save_dir</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
-    <span class="n">history_dir</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="s2">&quot;history&quot;</span>
-    <span class="n">history_dir</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-
-    <span class="c1"># the trainer is run with the History Callback by default</span>
-    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">history_dir</span> <span class="o">/</span> <span class="s2">&quot;train_eval_history.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">teh</span><span class="p">:</span>
-        <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">history</span><span class="p">,</span> <span class="n">teh</span><span class="p">)</span>  <span class="c1"># type: ignore[attr-defined]</span>
-
-    <span class="n">has_lr_history</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span>
-        <span class="p">[</span><span class="n">clbk</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;LRHistory&quot;</span> <span class="k">for</span> <span class="n">clbk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">callbacks</span><span class="p">]</span>
-    <span class="p">)</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">lr_scheduler</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">has_lr_history</span><span class="p">:</span>
-        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">history_dir</span> <span class="o">/</span> <span class="s2">&quot;lr_history.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">lrh</span><span class="p">:</span>
-            <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_history</span><span class="p">,</span> <span class="n">lrh</span><span class="p">)</span>  <span class="c1"># type: ignore[attr-defined]</span>
-
-    <span class="n">model_path</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="n">model_filename</span>
-    <span class="k">if</span> <span class="n">save_state_dict</span><span class="p">:</span>
-        <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cd_model</span><span class="o">.</span><span class="n">state_dict</span><span class="p">(),</span> <span class="n">model_path</span><span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cd_model</span><span class="p">,</span> <span class="n">model_path</span><span class="p">)</span>
-</code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -3064,8 +2746,7 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.sa
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -3141,7 +2822,7 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.sa
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -3159,7 +2840,7 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.sa
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/tab2vec.html b/mkdocs/site/pytorch-widedeep/tab2vec.html
index ca5599ce..34106d94 100644
--- a/mkdocs/site/pytorch-widedeep/tab2vec.html
+++ b/mkdocs/site/pytorch-widedeep/tab2vec.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1027,8 +1027,6 @@
         
       
         
-      
-        
       
         
       
@@ -1366,11 +1364,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1387,11 +1385,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1408,11 +1406,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1429,11 +1427,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1450,11 +1448,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1471,11 +1469,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1492,11 +1490,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1513,32 +1511,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1601,11 +1578,11 @@ <h1 id="tab2vec">Tab2Vec<a class="headerlink" href="#tab2vec" title="Permanent l
 
 
 <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Tab2Vec</span>
+            <span class="doc doc-object-name doc-class-name">Tab2Vec</span>
 
 
 <a href="#pytorch_widedeep.tab2vec.Tab2Vec" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Tab2Vec</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Tab2Vec</span><span class="p">(</span>
     <span class="n">tab_preprocessor</span><span class="p">,</span>
     <span class="n">model</span><span class="p">,</span>
     <span class="n">return_dataframe</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
@@ -1613,9 +1590,9 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
+    <div class="doc doc-contents first">
+
 
-  
       <p>Class to transform an input dataframe into vectorized form.</p>
 <p>This class will take an input dataframe in the form of the dataframe used
 for training, and it will turn it into a vectorised form based on the
@@ -1627,55 +1604,52 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
  We will be adding these functionalities in future versions</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>model</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.wdtypes.WideDeep" href="model_components.html#pytorch_widedeep.models.wide_deep.WideDeep">WideDeep</a>, <a class="autorefs autorefs-internal" title="pytorch_widedeep.bayesian_models.BayesianWide" href="bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide">BayesianWide</a>, <a class="autorefs autorefs-internal" title="pytorch_widedeep.bayesian_models.BayesianTabMlp" href="bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp">BayesianTabMlp</a>]</code>)
-        –
-        <div class="doc-md-description">
-          <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>tab_preprocessor</code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.TabPreprocessor" href="preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>)
-        –
-        <div class="doc-md-description">
-          <p><code>TabPreprocessor</code> object. Must be fitted.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>return_dataframe</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>model</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.wdtypes.WideDeep" href="model_components.html#pytorch_widedeep.models.wide_deep.WideDeep">WideDeep</a>, <a class="autorefs autorefs-internal" title="pytorch_widedeep.bayesian_models.BayesianWide" href="bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide">BayesianWide</a>, <a class="autorefs autorefs-internal" title="pytorch_widedeep.bayesian_models.BayesianTabMlp" href="bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp">BayesianTabMlp</a>]</code>)
+          –
+          <div class="doc-md-description">
+            <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>tab_preprocessor</code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.TabPreprocessor" href="preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>)
+          –
+          <div class="doc-md-description">
+            <p><code>TabPreprocessor</code> object. Must be fitted.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>return_dataframe</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating of the returned object(s) will be array(s) or
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating of the returned object(s) will be array(s) or
 pandas dataframe(s)</p>
-        </div>
-      </li>
-  </ul>
-
-
+          </div>
+        </li>
+    </ul>
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.tab2vec.Tab2Vec.vectorizer">vectorizer</span></code></b>
-            (<code><span title="nn.Module">Module</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Torch module with the categorical and continuous encoding process</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.tab2vec.Tab2Vec.vectorizer">vectorizer</span></code></b>
+              (<code><span title="nn.Module">Module</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Torch module with the categorical and continuous encoding process</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">string</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">choices</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
@@ -1716,9 +1690,9 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_vec</span> <span class="o">=</span> <span class="n">t2v</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_t2v</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 95</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 95</span>
 <span class="normal"> 96</span>
 <span class="normal"> 97</span>
 <span class="normal"> 98</span>
@@ -1754,12 +1728,11 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 
     <span class="bp">self</span><span class="o">.</span><span class="n">_set_dim_attributes</span><span class="p">(</span><span class="n">tab_preprocessor</span><span class="p">,</span> <span class="n">model</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1772,64 +1745,61 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit</span>
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
 <a href="#pytorch_widedeep.tab2vec.Tab2Vec.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">target_col</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">target_col</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>This is an empty method i.e. Returns the unchanged object itself. Is
 only included for consistency in case <code>Tab2Vec</code> is used as part of a
 Pipeline</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>DataFrame to be vectorised, i.e. the categorical and continuous
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>DataFrame to be vectorised, i.e. the categorical and continuous
 columns will be encoded based on the processing applied within
 the model</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target_col</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target_col</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Column name of the target_col variable. If <code>None</code> only the array of
+          –
+          <div class="doc-md-description">
+            <p>Column name of the target_col variable. If <code>None</code> only the array of
 predictors will be returned</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.tab2vec.Tab2Vec" href="#pytorch_widedeep.tab2vec.Tab2Vec">Tab2Vec</a></code>
-        –
-        <div class="doc-md-description">
-          
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.tab2vec.Tab2Vec" href="#pytorch_widedeep.tab2vec.Tab2Vec">Tab2Vec</a></code>
+          –
+          <div class="doc-md-description">
+            
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">114</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">114</span>
 <span class="normal">115</span>
 <span class="normal">116</span>
 <span class="normal">117</span>
@@ -1871,76 +1841,72 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit" class="doc doc-heading">
 
     <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">transform</span>
+            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
 <a href="#pytorch_widedeep.tab2vec.Tab2Vec.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">target_col</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">target_col</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Transforms the input dataframe into vectorized form. If a target
 column name is passed the target values will be returned separately
 in their corresponding type (np.ndarray or pd.DataFrame)</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>df</code></b>
-            (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>DataFrame to be vectorised, i.e. the categorical and continuous
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>df</code></b>
+              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>DataFrame to be vectorised, i.e. the categorical and continuous
 columns will be encoded based on the processing applied within
 the model</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target_col</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target_col</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Column name of the target_col variable. If <code>None</code> only the array of
+          –
+          <div class="doc-md-description">
+            <p>Column name of the target_col variable. If <code>None</code> only the array of
 predictors will be returned</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code>
-        –
-        <div class="doc-md-description">
-          <p>Returns eiter a numpy array with the vectorised values, or a Tuple
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code>
+          –
+          <div class="doc-md-description">
+            <p>Returns eiter a numpy array with the vectorised values, or a Tuple
 of numpy arrays with the vectorised values and the target. The
 same applies to dataframes in case we choose to set
 <code>return_dataframe = True</code></p>
-      </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">136</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">136</span>
 <span class="normal">137</span>
 <span class="normal">138</span>
 <span class="normal">139</span>
@@ -2070,31 +2036,29 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.transform" class="doc doc-heading">
         <span class="k">else</span><span class="p">:</span>
             <span class="k">return</span> <span class="n">x_vec</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit_transform</span>
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
 
 
 <a href="#pytorch_widedeep.tab2vec.Tab2Vec.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">target_col</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">target_col</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Combines <code>fit</code> and <code>transform</code></p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">202</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">202</span>
 <span class="normal">203</span>
 <span class="normal">204</span>
 <span class="normal">205</span>
@@ -2114,8 +2078,8 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit_transform" class="doc doc-heading">
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`&quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">target_col</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">target_col</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2123,8 +2087,7 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit_transform" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2199,7 +2162,7 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit_transform" class="doc doc-heading">
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -2217,7 +2180,7 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit_transform" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/trainer.html b/mkdocs/site/pytorch-widedeep/trainer.html
index 4876f1bb..65c3bcd0 100644
--- a/mkdocs/site/pytorch-widedeep/trainer.html
+++ b/mkdocs/site/pytorch-widedeep/trainer.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1027,8 +1027,6 @@
         
       
         
-      
-        
       
         
       
@@ -1366,32 +1364,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1408,11 +1385,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1429,11 +1406,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1450,11 +1427,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1471,11 +1448,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1492,11 +1469,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1513,11 +1490,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1534,11 +1511,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1608,11 +1585,11 @@ <h1 id="training-multimodal-deep-learning-models">Training multimodal Deep Learn
 
 
 <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Trainer</span>
+            <span class="doc doc-object-name doc-class-name">Trainer</span>
 
 
 <a href="#pytorch_widedeep.training.Trainer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Trainer</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Trainer</span><span class="p">(</span>
     <span class="n">model</span><span class="p">,</span>
     <span class="n">objective</span><span class="p">,</span>
     <span class="n">custom_loss_function</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -1628,32 +1605,31 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-          <p class="doc doc-class-bases">
-            Bases: <code><span title="pytorch_widedeep.training._base_trainer.BaseTrainer">BaseTrainer</span></code></p>
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.training._base_trainer.BaseTrainer">BaseTrainer</span></code></p>
+
 
-  
       <p>Class to set the of attributes that will be used during the
 training process.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>model</code></b>
-            (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.wdtypes.WideDeep" href="model_components.html#pytorch_widedeep.models.wide_deep.WideDeep">WideDeep</a></code>)
-        –
-        <div class="doc-md-description">
-          <p>An object of class <code>WideDeep</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>objective</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>Defines the objective, loss or cost function. <br/></p>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>model</code></b>
+              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.wdtypes.WideDeep" href="model_components.html#pytorch_widedeep.models.wide_deep.WideDeep">WideDeep</a></code>)
+          –
+          <div class="doc-md-description">
+            <p>An object of class <code>WideDeep</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>objective</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>Defines the objective, loss or cost function. <br/></p>
 <p>Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>,
 <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code>. <br/></p>
 <p>Possible values are:</p>
@@ -1699,31 +1675,31 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 </li>
 </ul>
 <p><strong>NOTE</strong>: For <code>multitarget</code> a custom loss function must be passed</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>custom_loss_function</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>custom_loss_function</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>It is possible to pass a custom loss function. See for example
+          –
+          <div class="doc-md-description">
+            <p>It is possible to pass a custom loss function. See for example
 <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the
 object or the Examples section in this documentation or in the repo.
 Note that if <code>custom_loss_function</code> is not <code>None</code>, <code>objective</code> must
 be <em>'binary'</em>, <em>'multiclass'</em> or <em>'regression'</em>, consistent with the
 loss function</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>optimizers</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]]]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>optimizers</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <ul>
+          –
+          <div class="doc-md-description">
+            <ul>
 <li>An instance of Pytorch's <code>Optimizer</code> object
   (e.g. <code>torch.optim.Adam()</code>) or</li>
 <li>a dictionary where there keys are the model components (i.e.
@@ -1736,16 +1712,16 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 </ul>
 <p>if no optimizers are passed it will default to <code>Adam</code> for all
 model components</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lr_schedulers</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]]]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>lr_schedulers</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <ul>
+          –
+          <div class="doc-md-description">
+            <ul>
 <li>An instance of Pytorch's <code>LRScheduler</code> object (e.g
   <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>) or</li>
 <li>a dictionary where there keys are the model componenst (i.e. <em>'wide'</em>,
@@ -1754,16 +1730,16 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
     learning rate schedulers if multiple models are used for the given
     data mode (e.g. two text columns/models for the deeptext component).</li>
 </ul>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>initializers</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>]]]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>initializers</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>]]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <ul>
+          –
+          <div class="doc-md-description">
+            <ul>
 <li>An instance of an <code>Initializer</code> object see <code>pytorch-widedeep.initializers</code> or</li>
 <li>a dictionary where there keys are the model components (i.e. <em>'wide'</em>,
   <em>'deeptabular'</em>, <em>'deeptext'</em>, <em>'deepimage'</em> and/or <em>'deephead'</em>)
@@ -1771,44 +1747,44 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
     initializers if multiple models are used for the given data mode (e.g.
     two text columns/models for the deeptext component).</li>
 </ul>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>transforms</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Transforms">Transforms</span>]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>transforms</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Transforms">Transforms</span>]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with <code>torchvision.transforms</code> to be applied to the image
+          –
+          <div class="doc-md-description">
+            <p>List with <code>torchvision.transforms</code> to be applied to the image
 component of the model (i.e. <code>deepimage</code>) See
 <a href="https://pytorch.org/docs/stable/torchvision/transforms.html">torchvision transforms</a>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>callbacks</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>callbacks</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List with <code>Callback</code> objects. The three callbacks available in
+          –
+          <div class="doc-md-description">
+            <p>List with <code>Callback</code> objects. The three callbacks available in
 <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and
 <code>EarlyStopping</code>. The <code>History</code> and the <code>LRShedulerCallback</code> callbacks
 are used by default. This can also be a custom callback as long as
 the object of type <code>Callback</code>. See
 <code>pytorch_widedeep.callbacks.Callback</code> or the examples folder in the
 repo.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>metrics</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.metrics.Metric">Metric</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="torchmetrics.Metric">Metric</span>]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>metrics</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.metrics.Metric">Metric</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="torchmetrics.Metric">Metric</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <ul>
+          –
+          <div class="doc-md-description">
+            <ul>
 <li>List of objects of type <code>Metric</code>. Metrics available are:
   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,
   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as long
@@ -1822,39 +1798,38 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
   it is an object of type <code>Metric</code>. See
   <a href="https://torchmetrics.readthedocs.io/en/latest/">the instructions</a>.</li>
 </ul>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Verbosity level. If set to 0 nothing will be printed during training</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>seed</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>Verbosity level. If set to 0 nothing will be printed during training</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>seed</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Random seed to be used internally for train/test split</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>Random seed to be used internally for train/test split</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Other Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
+<p><span class="doc-section-title">Other Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
 <ul>
 <li>
 <p><strong>device</strong>: <code>str</code><br/>
@@ -1878,39 +1853,37 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
     take a step: One of <em>'loss'</em> or <em>'metric'</em>. The ReduceLROnPlateau
     learning rate is a bit particular.</li>
 </ul>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.training.Trainer.cyclic_lr">cyclic_lr</span></code></b>
-            (<code>bool</code>)
-        –
-        <div class="doc-md-description">
-          <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.training.Trainer.cyclic_lr">cyclic_lr</span></code></b>
+              (<code>bool</code>)
+          –
+          <div class="doc-md-description">
+            <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr
 (i.e. <code>CyclicLR</code> or
 <code>OneCycleLR</code>). See <a href="https://pytorch.org/docs/stable/optim.html">Pytorch schedulers</a>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.training.Trainer.feature_importance">feature_importance</span></code></b>
-            (<code>dict</code>)
-        –
-        <div class="doc-md-description">
-          <p>dict where the keys are the column names and the values are the
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.training.Trainer.feature_importance">feature_importance</span></code></b>
+              (<code>dict</code>)
+          –
+          <div class="doc-md-description">
+            <p>dict where the keys are the column names and the values are the
 corresponding feature importances. This attribute will only exist
 if the <code>deeptabular</code> component is a Tabnet model.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">torchvision.transforms</span> <span class="kn">import</span> <span class="n">ToTensor</span>
 <span class="gp">&gt;&gt;&gt;</span>
@@ -1954,9 +1927,13 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 <span class="gp">... </span><span class="n">lr_schedulers</span><span class="o">=</span><span class="n">schedulers</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span> <span class="n">transforms</span><span class="o">=</span><span class="n">transforms</span><span class="p">)</span>
 </code></pre></div>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">233</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
 <span class="normal">234</span>
 <span class="normal">235</span>
 <span class="normal">236</span>
@@ -1990,11 +1967,7 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 <span class="normal">264</span>
 <span class="normal">265</span>
 <span class="normal">266</span>
-<span class="normal">267</span>
-<span class="normal">268</span>
-<span class="normal">269</span>
-<span class="normal">270</span>
-<span class="normal">271</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+<span class="normal">267</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
     <span class="s2">&quot;objective&quot;</span><span class="p">,</span>
     <span class="p">[</span><span class="s2">&quot;loss_function&quot;</span><span class="p">,</span> <span class="s2">&quot;loss_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;loss&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_function&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;cost&quot;</span><span class="p">],</span>
 <span class="p">)</span>
@@ -2034,12 +2007,11 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
         <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2052,13 +2024,12 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit</span>
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
 <a href="#pytorch_widedeep.training.Trainer.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span>
     <span class="n">X_wide</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">X_text</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -2073,168 +2044,166 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
     <span class="n">custom_dataloader</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">feature_importance_sample_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">finetune</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">with_lds</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
     <span class="o">**</span><span class="n">kwargs</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Fit method.</p>
 <p>The input datasets can be passed either directly via numpy arrays
 (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in
 dictionaries (<code>X_train</code> or <code>X_val</code>).</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>X_wide</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>X_wide</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>wide</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>wide</code> model component.
 See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_tab</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deeptabular</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deeptabular</code> model component.
 See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_text</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_text</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deeptext</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deeptext</code> model component.
 See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code>.
 If multiple text columns/models are used, this should be a list of
 numpy arrays</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_img</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_img</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deepimage</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deepimage</code> model component.
 See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code>.
 If multiple image columns/models are used, this should be a list of
 numpy arrays</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_train</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_train</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The training dataset can also be passed in a dictionary. Keys are
+          –
+          <div class="doc-md-description">
+            <p>The training dataset can also be passed in a dictionary. Keys are
 <em>'X_wide'</em>, <em>'X_tab'</em>, <em>'X_text'</em>, <em>'X_img'</em> and <em>'target'</em>. Values
 are the corresponding matrices. Note that of multiple text or image
 columns/models are used, the corresponding values should be lists
 of numpy arrays</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_val</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_val</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The validation dataset can also be passed in a dictionary. Keys
+          –
+          <div class="doc-md-description">
+            <p>The validation dataset can also be passed in a dictionary. Keys
 are <em>'X_wide'</em>, <em>'X_tab'</em>, <em>'X_text'</em>, <em>'X_img'</em> and <em>'target'</em>.
 Values are the corresponding matrices. Note that of multiple text
 or image columns/models are used, the corresponding values should
 be lists of numpy arrays</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>val_split</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>val_split</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>train/val split fraction</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>target</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>train/val split fraction</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>target</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>target values</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_epochs</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>target values</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_epochs</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of epochs</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>validation_freq</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          –
+          <div class="doc-md-description">
+            <p>number of epochs</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>validation_freq</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>epochs validation frequency</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>batch_size</code></b>
-            (<code>int</code>, default:
-                <code>32</code>
+          –
+          <div class="doc-md-description">
+            <p>epochs validation frequency</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>batch_size</code></b>
+              (<code>int</code>, default:
+                  <code>32</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>batch size</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>custom_dataloader</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.utils.data.DataLoader">DataLoader</span>]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>batch size</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>custom_dataloader</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.utils.data.DataLoader">DataLoader</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>object of class <code>torch.utils.data.DataLoader</code>. Available
+          –
+          <div class="doc-md-description">
+            <p>object of class <code>torch.utils.data.DataLoader</code>. Available
 predefined dataloaders are in <code>pytorch-widedeep.dataloaders</code>.If
 <code>None</code>, a standard torch <code>DataLoader</code> is used.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>finetune</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>finetune</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>fine-tune individual model components. This functionality can also
+          –
+          <div class="doc-md-description">
+            <p>fine-tune individual model components. This functionality can also
 be used to 'warm-up' (and hence the alias <code>warmup</code>) individual
 components before the joined training starts, and hence its
 alias. See the Examples folder in the repo for more details</p>
@@ -2262,33 +2231,18 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 <p>For details on how these routines work, please see the Examples
 section in this documentation and the Examples folder in the repo. <br/>
 Param Alias: <code>warmup</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_lds</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
-)
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if Label Distribution Smoothing will be used. <br/>
-information_source: <strong>NOTE</strong>: We consider this feature absolutely
-experimental and we recommend the user to not use it unless the
-corresponding <a href="https://arxiv.org/abs/2102.09554">publication</a> is
-well understood</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Other Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>**kwargs</code></b>
-        –
-        <div class="doc-md-description">
-          <p>Other keyword arguments are:</p>
+<p><span class="doc-section-title">Other Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>**kwargs</code></b>
+          –
+          <div class="doc-md-description">
+            <p>Other keyword arguments are:</p>
 <ul>
 <li>
 <p><strong>DataLoader related parameters</strong>:<br/>
@@ -2298,27 +2252,6 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
     for details.</p>
 </li>
 <li>
-<p><strong>Label Distribution Smoothing related parameters</strong>:<br/></p>
-<ul>
-<li>lds_kernel (<code>Literal['gaussian', 'triang', 'laplace']</code>):
-    choice of kernel for Label Distribution Smoothing</li>
-<li>lds_ks (<code>int</code>):
-    LDS kernel window size</li>
-<li>lds_sigma (<code>float</code>):
-    standard deviation of ['gaussian','laplace'] kernel for LDS</li>
-<li>lds_granularity (<code>int</code>):
-    number of bins in histogram used in LDS to count occurence of sample values</li>
-<li>lds_reweight (<code>bool</code>):
-    option to reweight bin frequency counts in LDS</li>
-<li>lds_y_max (<code>Optional[float]</code>):
-    option to restrict LDS bins by upper label limit</li>
-<li>lds_y_min (<code>Optional[float]</code>):
-    option to restrict LDS bins by lower label limit</li>
-</ul>
-<p>See <code>pytorch_widedeep.trainer._wd_dataset</code> for more details on
-the implications of these parameters</p>
-</li>
-<li>
 <p><strong>Finetune related parameters</strong>:<br/>
     see the source code at <code>pytorch_widedeep._finetune</code>. Namely, these are:</p>
 <ul>
@@ -2352,20 +2285,23 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 </ul>
 </li>
 </ul>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <p>For a series of comprehensive examples on how to use the <code>fit</code> method, please see the
 <a href="https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples">Examples</a>
 folder in the repo</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">273</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
 <span class="normal">274</span>
 <span class="normal">275</span>
 <span class="normal">276</span>
@@ -2619,52 +2555,7 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 <span class="normal">524</span>
 <span class="normal">525</span>
 <span class="normal">526</span>
-<span class="normal">527</span>
-<span class="normal">528</span>
-<span class="normal">529</span>
-<span class="normal">530</span>
-<span class="normal">531</span>
-<span class="normal">532</span>
-<span class="normal">533</span>
-<span class="normal">534</span>
-<span class="normal">535</span>
-<span class="normal">536</span>
-<span class="normal">537</span>
-<span class="normal">538</span>
-<span class="normal">539</span>
-<span class="normal">540</span>
-<span class="normal">541</span>
-<span class="normal">542</span>
-<span class="normal">543</span>
-<span class="normal">544</span>
-<span class="normal">545</span>
-<span class="normal">546</span>
-<span class="normal">547</span>
-<span class="normal">548</span>
-<span class="normal">549</span>
-<span class="normal">550</span>
-<span class="normal">551</span>
-<span class="normal">552</span>
-<span class="normal">553</span>
-<span class="normal">554</span>
-<span class="normal">555</span>
-<span class="normal">556</span>
-<span class="normal">557</span>
-<span class="normal">558</span>
-<span class="normal">559</span>
-<span class="normal">560</span>
-<span class="normal">561</span>
-<span class="normal">562</span>
-<span class="normal">563</span>
-<span class="normal">564</span>
-<span class="normal">565</span>
-<span class="normal">566</span>
-<span class="normal">567</span>
-<span class="normal">568</span>
-<span class="normal">569</span>
-<span class="normal">570</span>
-<span class="normal">571</span>
-<span class="normal">572</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;finetune&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;warmup&quot;</span><span class="p">])</span>
+<span class="normal">527</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;finetune&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;warmup&quot;</span><span class="p">])</span>
 <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -2681,7 +2572,6 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
     <span class="n">custom_dataloader</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DataLoader</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">feature_importance_sample_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">finetune</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">with_lds</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
     <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
 <span class="p">):</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fit method.</span>
@@ -2764,12 +2654,6 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 <span class="sd">        For details on how these routines work, please see the Examples</span>
 <span class="sd">        section in this documentation and the Examples folder in the repo. &lt;br/&gt;</span>
 <span class="sd">        Param Alias: `warmup`</span>
-<span class="sd">    with_lds: bool, default=False</span>
-<span class="sd">        Boolean indicating if Label Distribution Smoothing will be used. &lt;br/&gt;</span>
-<span class="sd">        information_source: **NOTE**: We consider this feature absolutely</span>
-<span class="sd">        experimental and we recommend the user to not use it unless the</span>
-<span class="sd">        corresponding [publication](https://arxiv.org/abs/2102.09554) is</span>
-<span class="sd">        well understood</span>
 
 <span class="sd">    Other Parameters</span>
 <span class="sd">    ----------------</span>
@@ -2782,26 +2666,6 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 <span class="sd">            [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)</span>
 <span class="sd">            for details.</span>
 
-<span class="sd">        - **Label Distribution Smoothing related parameters**:&lt;br/&gt;</span>
-
-<span class="sd">            - lds_kernel (`Literal[&#39;gaussian&#39;, &#39;triang&#39;, &#39;laplace&#39;]`):</span>
-<span class="sd">                choice of kernel for Label Distribution Smoothing</span>
-<span class="sd">            - lds_ks (`int`):</span>
-<span class="sd">                LDS kernel window size</span>
-<span class="sd">            - lds_sigma (`float`):</span>
-<span class="sd">                standard deviation of [&#39;gaussian&#39;,&#39;laplace&#39;] kernel for LDS</span>
-<span class="sd">            - lds_granularity (`int`):</span>
-<span class="sd">                number of bins in histogram used in LDS to count occurence of sample values</span>
-<span class="sd">            - lds_reweight (`bool`):</span>
-<span class="sd">                option to reweight bin frequency counts in LDS</span>
-<span class="sd">            - lds_y_max (`Optional[float]`):</span>
-<span class="sd">                option to restrict LDS bins by upper label limit</span>
-<span class="sd">            - lds_y_min (`Optional[float]`):</span>
-<span class="sd">                option to restrict LDS bins by lower label limit</span>
-
-<span class="sd">            See `pytorch_widedeep.trainer._wd_dataset` for more details on</span>
-<span class="sd">            the implications of these parameters</span>
-
 <span class="sd">        - **Finetune related parameters**:&lt;br/&gt;</span>
 <span class="sd">            see the source code at `pytorch_widedeep._finetune`. Namely, these are:</span>
 
@@ -2841,9 +2705,7 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 <span class="sd">    folder in the repo</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
-    <span class="n">lds_args</span><span class="p">,</span> <span class="n">dataloader_args</span><span class="p">,</span> <span class="n">finetune_args</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_kwargs</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
-    <span class="n">lds_args</span><span class="p">[</span><span class="s2">&quot;with_lds&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">with_lds</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_lds</span> <span class="o">=</span> <span class="n">with_lds</span>
+    <span class="n">dataloader_args</span><span class="p">,</span> <span class="n">finetune_args</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_kwargs</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
 
     <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="n">batch_size</span>
 
@@ -2859,25 +2721,18 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
         <span class="n">val_split</span><span class="p">,</span>
         <span class="n">target</span><span class="p">,</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">transforms</span><span class="p">,</span>
-        <span class="o">**</span><span class="n">lds_args</span><span class="p">,</span>
     <span class="p">)</span>
-    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">custom_dataloader</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
-        <span class="k">if</span> <span class="nb">issubclass</span><span class="p">(</span><span class="n">custom_dataloader</span><span class="p">,</span> <span class="n">DataLoader</span><span class="p">):</span>
-            <span class="n">train_loader</span> <span class="o">=</span> <span class="n">custom_dataloader</span><span class="p">(</span>  <span class="c1"># type: ignore[misc]</span>
-                <span class="n">dataset</span><span class="o">=</span><span class="n">train_set</span><span class="p">,</span>
-                <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
-                <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
-                <span class="o">**</span><span class="n">dataloader_args</span><span class="p">,</span>
-            <span class="p">)</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="ne">NotImplementedError</span><span class="p">(</span>
-                <span class="s2">&quot;Custom DataLoader must be a subclass of &quot;</span>
-                <span class="s2">&quot;torch.utils.data.DataLoader, please see the &quot;</span>
-                <span class="s2">&quot;pytorch documentation or examples in &quot;</span>
-                <span class="s2">&quot;pytorch_widedeep.dataloaders&quot;</span>
-            <span class="p">)</span>
+    <span class="k">if</span> <span class="n">custom_dataloader</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="c1"># make sure is callable (and HAS to be an subclass of DataLoader)</span>
+        <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">custom_dataloader</span><span class="p">,</span> <span class="nb">type</span><span class="p">)</span>
+        <span class="n">train_loader</span> <span class="o">=</span> <span class="n">custom_dataloader</span><span class="p">(</span>  <span class="c1"># type: ignore[misc]</span>
+            <span class="n">dataset</span><span class="o">=</span><span class="n">train_set</span><span class="p">,</span>
+            <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
+            <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">dataloader_args</span><span class="p">,</span>
+        <span class="p">)</span>
     <span class="k">else</span><span class="p">:</span>
-        <span class="n">train_loader</span> <span class="o">=</span> <span class="n">DataLoaderDefault</span><span class="p">(</span>
+        <span class="n">train_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
             <span class="n">dataset</span><span class="o">=</span><span class="n">train_set</span><span class="p">,</span>
             <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
             <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
@@ -2913,11 +2768,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 
         <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
         <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">train_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">t</span><span class="p">:</span>
-            <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">,</span> <span class="n">lds_weightt</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loader</span><span class="p">):</span>
+            <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loader</span><span class="p">):</span>
                 <span class="n">t</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;epoch </span><span class="si">%i</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
-                <span class="n">train_score</span><span class="p">,</span> <span class="n">train_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_step</span><span class="p">(</span>
-                    <span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="n">epoch</span><span class="p">,</span> <span class="n">lds_weightt</span>
-                <span class="p">)</span>
+                <span class="n">train_score</span><span class="p">,</span> <span class="n">train_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_step</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">)</span>
                 <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="n">train_score</span><span class="p">)</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_batch_end</span><span class="p">(</span><span class="n">batch</span><span class="o">=</span><span class="n">batch_idx</span><span class="p">)</span>
         <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="n">train_score</span><span class="p">,</span> <span class="s2">&quot;train&quot;</span><span class="p">)</span>
@@ -2953,9 +2806,6 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             <span class="c1"># self.callback_container.on_train_end(epoch_logs)</span>
             <span class="k">break</span>
 
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">with_fds</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">_update_fds_stats</span><span class="p">(</span><span class="n">train_loader</span><span class="p">,</span> <span class="n">epoch</span><span class="p">)</span>
-
     <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_end</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">)</span>
 
     <span class="k">if</span> <span class="n">feature_importance_sample_size</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
@@ -2965,22 +2815,20 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">_restore_best_weights</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">predict</span>
+            <span class="doc doc-object-name doc-function-name">predict</span>
 
 
 <a href="#pytorch_widedeep.training.Trainer.predict" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">predict</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">predict</span><span class="p">(</span>
     <span class="n">X_wide</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">X_text</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -2990,157 +2838,155 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Returns the predictions</p>
 <p>The input datasets can be passed either directly via numpy arrays
 (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in
 a dictionary (<code>X_test</code>)</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>X_wide</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>X_wide</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>wide</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>wide</code> model component.
 See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_tab</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deeptabular</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deeptabular</code> model component.
 See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_text</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_text</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deeptext</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deeptext</code> model component.
 See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_img</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_img</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deepimage</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deepimage</code> model component.
 See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_test</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_test</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The test dataset can also be passed in a dictionary. Keys are
+          –
+          <div class="doc-md-description">
+            <p>The test dataset can also be passed in a dictionary. Keys are
 <code>X_wide</code>, <em>'X_tab'</em>, <em>'X_text'</em>, <em>'X_img'</em> and <em>'target'</em>. Values
 are the corresponding matrices.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>batch_size</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>batch_size</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>If a trainer is used to predict after having trained a model, the
+          –
+          <div class="doc-md-description">
+            <p>If a trainer is used to predict after having trained a model, the
 <code>batch_size</code> needs to be defined as it will not be defined as
 the <code>Trainer</code> is instantiated</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code>np.ndarray:</code>
-        –
-        <div class="doc-md-description">
-          <p>array with the predictions</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code>np.ndarray:</code>
+          –
+          <div class="doc-md-description">
+            <p>array with the predictions</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">574</span>
-<span class="normal">575</span>
-<span class="normal">576</span>
-<span class="normal">577</span>
-<span class="normal">578</span>
-<span class="normal">579</span>
-<span class="normal">580</span>
-<span class="normal">581</span>
-<span class="normal">582</span>
-<span class="normal">583</span>
-<span class="normal">584</span>
-<span class="normal">585</span>
-<span class="normal">586</span>
-<span class="normal">587</span>
-<span class="normal">588</span>
-<span class="normal">589</span>
-<span class="normal">590</span>
-<span class="normal">591</span>
-<span class="normal">592</span>
-<span class="normal">593</span>
-<span class="normal">594</span>
-<span class="normal">595</span>
-<span class="normal">596</span>
-<span class="normal">597</span>
-<span class="normal">598</span>
-<span class="normal">599</span>
-<span class="normal">600</span>
-<span class="normal">601</span>
-<span class="normal">602</span>
-<span class="normal">603</span>
-<span class="normal">604</span>
-<span class="normal">605</span>
-<span class="normal">606</span>
-<span class="normal">607</span>
-<span class="normal">608</span>
-<span class="normal">609</span>
-<span class="normal">610</span>
-<span class="normal">611</span>
-<span class="normal">612</span>
-<span class="normal">613</span>
-<span class="normal">614</span>
-<span class="normal">615</span>
-<span class="normal">616</span>
-<span class="normal">617</span>
-<span class="normal">618</span>
-<span class="normal">619</span>
-<span class="normal">620</span>
-<span class="normal">621</span>
-<span class="normal">622</span>
-<span class="normal">623</span>
-<span class="normal">624</span>
-<span class="normal">625</span>
-<span class="normal">626</span>
-<span class="normal">627</span>
-<span class="normal">628</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">529</span>
+<span class="normal">530</span>
+<span class="normal">531</span>
+<span class="normal">532</span>
+<span class="normal">533</span>
+<span class="normal">534</span>
+<span class="normal">535</span>
+<span class="normal">536</span>
+<span class="normal">537</span>
+<span class="normal">538</span>
+<span class="normal">539</span>
+<span class="normal">540</span>
+<span class="normal">541</span>
+<span class="normal">542</span>
+<span class="normal">543</span>
+<span class="normal">544</span>
+<span class="normal">545</span>
+<span class="normal">546</span>
+<span class="normal">547</span>
+<span class="normal">548</span>
+<span class="normal">549</span>
+<span class="normal">550</span>
+<span class="normal">551</span>
+<span class="normal">552</span>
+<span class="normal">553</span>
+<span class="normal">554</span>
+<span class="normal">555</span>
+<span class="normal">556</span>
+<span class="normal">557</span>
+<span class="normal">558</span>
+<span class="normal">559</span>
+<span class="normal">560</span>
+<span class="normal">561</span>
+<span class="normal">562</span>
+<span class="normal">563</span>
+<span class="normal">564</span>
+<span class="normal">565</span>
+<span class="normal">566</span>
+<span class="normal">567</span>
+<span class="normal">568</span>
+<span class="normal">569</span>
+<span class="normal">570</span>
+<span class="normal">571</span>
+<span class="normal">572</span>
+<span class="normal">573</span>
+<span class="normal">574</span>
+<span class="normal">575</span>
+<span class="normal">576</span>
+<span class="normal">577</span>
+<span class="normal">578</span>
+<span class="normal">579</span>
+<span class="normal">580</span>
+<span class="normal">581</span>
+<span class="normal">582</span>
+<span class="normal">583</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3196,22 +3042,20 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
         <span class="n">preds</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>  <span class="c1"># type: ignore[return-value]</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">predict_uncertainty</span>
+            <span class="doc doc-object-name doc-function-name">predict_uncertainty</span>
 
 
 <a href="#pytorch_widedeep.training.Trainer.predict_uncertainty" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">predict_uncertainty</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">predict_uncertainty</span><span class="p">(</span>
     <span class="n">X_wide</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">X_text</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -3222,8 +3066,8 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Returns the predicted ucnertainty of the model for the test dataset
 using a Monte Carlo method during which dropout layers are activated
 in the evaluation/prediction phase and each sample is predicted N
@@ -3233,95 +3077,93 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
 Model Uncertainty in Deep Learning</a>.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>X_wide</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>X_wide</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>wide</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>wide</code> model component.
 See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_tab</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deeptabular</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deeptabular</code> model component.
 See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_text</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_text</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deeptext</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deeptext</code> model component.
 See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_img</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_img</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deepimage</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deepimage</code> model component.
 See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_test</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_test</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The test dataset can also be passed in a dictionary. Keys are
+          –
+          <div class="doc-md-description">
+            <p>The test dataset can also be passed in a dictionary. Keys are
 <em>'X_wide'</em>, <em>'X_tab'</em>, <em>'X_text'</em>, <em>'X_img'</em> and <em>'target'</em>. Values
 are the corresponding matrices.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>batch_size</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>batch_size</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>If a trainer is used to predict after having trained a model, the
+          –
+          <div class="doc-md-description">
+            <p>If a trainer is used to predict after having trained a model, the
 <code>batch_size</code> needs to be defined as it will not be defined as
 the <code>Trainer</code> is instantiated</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>uncertainty_granularity</code></b>
-        –
-        <div class="doc-md-description">
-          <p>number of times the model does prediction for each sample</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>uncertainty_granularity</code></b>
+          –
+          <div class="doc-md-description">
+            <p>number of times the model does prediction for each sample</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code>np.ndarray:</code>
-        –
-        <div class="doc-md-description">
-          <ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code>np.ndarray:</code>
+          –
+          <div class="doc-md-description">
+            <ul>
 <li>if <code>method = regression</code>, it will return an array with <code>(max, min, mean, stdev)</code>
   values for each sample.</li>
 <li>if <code>method = binary</code> it will return an array with
@@ -3330,13 +3172,58 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
   <code>(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)</code>
   values for each sample.</li>
 </ul>
-      </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">630</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">585</span>
+<span class="normal">586</span>
+<span class="normal">587</span>
+<span class="normal">588</span>
+<span class="normal">589</span>
+<span class="normal">590</span>
+<span class="normal">591</span>
+<span class="normal">592</span>
+<span class="normal">593</span>
+<span class="normal">594</span>
+<span class="normal">595</span>
+<span class="normal">596</span>
+<span class="normal">597</span>
+<span class="normal">598</span>
+<span class="normal">599</span>
+<span class="normal">600</span>
+<span class="normal">601</span>
+<span class="normal">602</span>
+<span class="normal">603</span>
+<span class="normal">604</span>
+<span class="normal">605</span>
+<span class="normal">606</span>
+<span class="normal">607</span>
+<span class="normal">608</span>
+<span class="normal">609</span>
+<span class="normal">610</span>
+<span class="normal">611</span>
+<span class="normal">612</span>
+<span class="normal">613</span>
+<span class="normal">614</span>
+<span class="normal">615</span>
+<span class="normal">616</span>
+<span class="normal">617</span>
+<span class="normal">618</span>
+<span class="normal">619</span>
+<span class="normal">620</span>
+<span class="normal">621</span>
+<span class="normal">622</span>
+<span class="normal">623</span>
+<span class="normal">624</span>
+<span class="normal">625</span>
+<span class="normal">626</span>
+<span class="normal">627</span>
+<span class="normal">628</span>
+<span class="normal">629</span>
+<span class="normal">630</span>
 <span class="normal">631</span>
 <span class="normal">632</span>
 <span class="normal">633</span>
@@ -3386,52 +3273,7 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
 <span class="normal">677</span>
 <span class="normal">678</span>
 <span class="normal">679</span>
-<span class="normal">680</span>
-<span class="normal">681</span>
-<span class="normal">682</span>
-<span class="normal">683</span>
-<span class="normal">684</span>
-<span class="normal">685</span>
-<span class="normal">686</span>
-<span class="normal">687</span>
-<span class="normal">688</span>
-<span class="normal">689</span>
-<span class="normal">690</span>
-<span class="normal">691</span>
-<span class="normal">692</span>
-<span class="normal">693</span>
-<span class="normal">694</span>
-<span class="normal">695</span>
-<span class="normal">696</span>
-<span class="normal">697</span>
-<span class="normal">698</span>
-<span class="normal">699</span>
-<span class="normal">700</span>
-<span class="normal">701</span>
-<span class="normal">702</span>
-<span class="normal">703</span>
-<span class="normal">704</span>
-<span class="normal">705</span>
-<span class="normal">706</span>
-<span class="normal">707</span>
-<span class="normal">708</span>
-<span class="normal">709</span>
-<span class="normal">710</span>
-<span class="normal">711</span>
-<span class="normal">712</span>
-<span class="normal">713</span>
-<span class="normal">714</span>
-<span class="normal">715</span>
-<span class="normal">716</span>
-<span class="normal">717</span>
-<span class="normal">718</span>
-<span class="normal">719</span>
-<span class="normal">720</span>
-<span class="normal">721</span>
-<span class="normal">722</span>
-<span class="normal">723</span>
-<span class="normal">724</span>
-<span class="normal">725</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict_uncertainty</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
+<span class="normal">680</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict_uncertainty</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3528,22 +3370,20 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
         <span class="n">preds</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">hstack</span><span class="p">((</span><span class="n">preds</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="mi">1</span><span class="p">))))</span>
         <span class="k">return</span> <span class="n">preds</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">predict_proba</span>
+            <span class="doc doc-object-name doc-function-name">predict_proba</span>
 
 
 <a href="#pytorch_widedeep.training.Trainer.predict_proba" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">predict_proba</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">predict_proba</span><span class="p">(</span>
     <span class="n">X_wide</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">X_text</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -3553,8 +3393,8 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Returns the predicted probabilities for the test dataset for  binary
 and multiclass methods</p>
 <p>The input datasets can be passed either directly via numpy arrays
@@ -3562,95 +3402,138 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
 a dictionary (<code>X_test</code>)</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>X_wide</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>X_wide</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>wide</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>wide</code> model component.
 See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_tab</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_tab</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deeptabular</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deeptabular</code> model component.
 See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_text</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_text</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deeptext</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deeptext</code> model component.
 See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_img</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_img</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Input for the <code>deepimage</code> model component.
+          –
+          <div class="doc-md-description">
+            <p>Input for the <code>deepimage</code> model component.
 See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>X_test</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>X_test</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>The test dataset can also be passed in a dictionary. Keys are
+          –
+          <div class="doc-md-description">
+            <p>The test dataset can also be passed in a dictionary. Keys are
 <code>X_wide</code>, <em>'X_tab'</em>, <em>'X_text'</em>, <em>'X_img'</em> and <em>'target'</em>. Values
 are the corresponding matrices.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>batch_size</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>batch_size</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>If a trainer is used to predict after having trained a model, the
+          –
+          <div class="doc-md-description">
+            <p>If a trainer is used to predict after having trained a model, the
 <code>batch_size</code> needs to be defined as it will not be defined as
 the <code>Trainer</code> is instantiated</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>array with the probabilities per class</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>array with the probabilities per class</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">727</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">682</span>
+<span class="normal">683</span>
+<span class="normal">684</span>
+<span class="normal">685</span>
+<span class="normal">686</span>
+<span class="normal">687</span>
+<span class="normal">688</span>
+<span class="normal">689</span>
+<span class="normal">690</span>
+<span class="normal">691</span>
+<span class="normal">692</span>
+<span class="normal">693</span>
+<span class="normal">694</span>
+<span class="normal">695</span>
+<span class="normal">696</span>
+<span class="normal">697</span>
+<span class="normal">698</span>
+<span class="normal">699</span>
+<span class="normal">700</span>
+<span class="normal">701</span>
+<span class="normal">702</span>
+<span class="normal">703</span>
+<span class="normal">704</span>
+<span class="normal">705</span>
+<span class="normal">706</span>
+<span class="normal">707</span>
+<span class="normal">708</span>
+<span class="normal">709</span>
+<span class="normal">710</span>
+<span class="normal">711</span>
+<span class="normal">712</span>
+<span class="normal">713</span>
+<span class="normal">714</span>
+<span class="normal">715</span>
+<span class="normal">716</span>
+<span class="normal">717</span>
+<span class="normal">718</span>
+<span class="normal">719</span>
+<span class="normal">720</span>
+<span class="normal">721</span>
+<span class="normal">722</span>
+<span class="normal">723</span>
+<span class="normal">724</span>
+<span class="normal">725</span>
+<span class="normal">726</span>
+<span class="normal">727</span>
 <span class="normal">728</span>
 <span class="normal">729</span>
 <span class="normal">730</span>
@@ -3658,52 +3541,7 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
 <span class="normal">732</span>
 <span class="normal">733</span>
 <span class="normal">734</span>
-<span class="normal">735</span>
-<span class="normal">736</span>
-<span class="normal">737</span>
-<span class="normal">738</span>
-<span class="normal">739</span>
-<span class="normal">740</span>
-<span class="normal">741</span>
-<span class="normal">742</span>
-<span class="normal">743</span>
-<span class="normal">744</span>
-<span class="normal">745</span>
-<span class="normal">746</span>
-<span class="normal">747</span>
-<span class="normal">748</span>
-<span class="normal">749</span>
-<span class="normal">750</span>
-<span class="normal">751</span>
-<span class="normal">752</span>
-<span class="normal">753</span>
-<span class="normal">754</span>
-<span class="normal">755</span>
-<span class="normal">756</span>
-<span class="normal">757</span>
-<span class="normal">758</span>
-<span class="normal">759</span>
-<span class="normal">760</span>
-<span class="normal">761</span>
-<span class="normal">762</span>
-<span class="normal">763</span>
-<span class="normal">764</span>
-<span class="normal">765</span>
-<span class="normal">766</span>
-<span class="normal">767</span>
-<span class="normal">768</span>
-<span class="normal">769</span>
-<span class="normal">770</span>
-<span class="normal">771</span>
-<span class="normal">772</span>
-<span class="normal">773</span>
-<span class="normal">774</span>
-<span class="normal">775</span>
-<span class="normal">776</span>
-<span class="normal">777</span>
-<span class="normal">778</span>
-<span class="normal">779</span>
-<span class="normal">780</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict_proba</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]  # noqa: C901</span>
+<span class="normal">735</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict_proba</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]  # noqa: C901</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3758,30 +3596,29 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
     <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
         <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">save</span>
+            <span class="doc doc-object-name doc-function-name">save</span>
 
 
 <a href="#pytorch_widedeep.training.Trainer.save" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">save</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">save</span><span class="p">(</span>
     <span class="n">path</span><span class="p">,</span>
     <span class="n">save_state_dict</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+    <span class="n">save_optimizer</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
     <span class="n">model_filename</span><span class="o">=</span><span class="s2">&quot;wd_model.pt&quot;</span><span class="p">,</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Saves the model, training and evaluation history, and the
 <code>feature_importance</code> attribute (if the <code>deeptabular</code> component is a
 Tabnet model) to disk</p>
@@ -3801,108 +3638,108 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
 history and learning rate history.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>path</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>path to the directory where the model and the feature importance
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>path</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>path to the directory where the model and the feature importance
 attribute will be saved.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>save_state_dict</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>save_state_dict</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether to save directly the model or the
-model's state dictionary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>model_filename</code></b>
-            (<code>str</code>, default:
-                <code>&#39;wd_model.pt&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether to save directly the model
+(and optimizer) or the model's (and optimizer's) state
+dictionary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>save_optimizer</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>filename where the model weights will be store</p>
-        </div>
-      </li>
-  </ul>
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether to save the optimizer</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>model_filename</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;wd_model.pt&#39;</code>
+)
+          –
+          <div class="doc-md-description">
+            <p>filename where the model weights will be store</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">793</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">748</span>
+<span class="normal">749</span>
+<span class="normal">750</span>
+<span class="normal">751</span>
+<span class="normal">752</span>
+<span class="normal">753</span>
+<span class="normal">754</span>
+<span class="normal">755</span>
+<span class="normal">756</span>
+<span class="normal">757</span>
+<span class="normal">758</span>
+<span class="normal">759</span>
+<span class="normal">760</span>
+<span class="normal">761</span>
+<span class="normal">762</span>
+<span class="normal">763</span>
+<span class="normal">764</span>
+<span class="normal">765</span>
+<span class="normal">766</span>
+<span class="normal">767</span>
+<span class="normal">768</span>
+<span class="normal">769</span>
+<span class="normal">770</span>
+<span class="normal">771</span>
+<span class="normal">772</span>
+<span class="normal">773</span>
+<span class="normal">774</span>
+<span class="normal">775</span>
+<span class="normal">776</span>
+<span class="normal">777</span>
+<span class="normal">778</span>
+<span class="normal">779</span>
+<span class="normal">780</span>
+<span class="normal">781</span>
+<span class="normal">782</span>
+<span class="normal">783</span>
+<span class="normal">784</span>
+<span class="normal">785</span>
+<span class="normal">786</span>
+<span class="normal">787</span>
+<span class="normal">788</span>
+<span class="normal">789</span>
+<span class="normal">790</span>
+<span class="normal">791</span>
+<span class="normal">792</span>
+<span class="normal">793</span>
 <span class="normal">794</span>
 <span class="normal">795</span>
 <span class="normal">796</span>
 <span class="normal">797</span>
-<span class="normal">798</span>
-<span class="normal">799</span>
-<span class="normal">800</span>
-<span class="normal">801</span>
-<span class="normal">802</span>
-<span class="normal">803</span>
-<span class="normal">804</span>
-<span class="normal">805</span>
-<span class="normal">806</span>
-<span class="normal">807</span>
-<span class="normal">808</span>
-<span class="normal">809</span>
-<span class="normal">810</span>
-<span class="normal">811</span>
-<span class="normal">812</span>
-<span class="normal">813</span>
-<span class="normal">814</span>
-<span class="normal">815</span>
-<span class="normal">816</span>
-<span class="normal">817</span>
-<span class="normal">818</span>
-<span class="normal">819</span>
-<span class="normal">820</span>
-<span class="normal">821</span>
-<span class="normal">822</span>
-<span class="normal">823</span>
-<span class="normal">824</span>
-<span class="normal">825</span>
-<span class="normal">826</span>
-<span class="normal">827</span>
-<span class="normal">828</span>
-<span class="normal">829</span>
-<span class="normal">830</span>
-<span class="normal">831</span>
-<span class="normal">832</span>
-<span class="normal">833</span>
-<span class="normal">834</span>
-<span class="normal">835</span>
-<span class="normal">836</span>
-<span class="normal">837</span>
-<span class="normal">838</span>
-<span class="normal">839</span>
-<span class="normal">840</span>
-<span class="normal">841</span>
-<span class="normal">842</span>
-<span class="normal">843</span>
-<span class="normal">844</span>
-<span class="normal">845</span>
-<span class="normal">846</span>
-<span class="normal">847</span>
-<span class="normal">848</span>
-<span class="normal">849</span>
-<span class="normal">850</span>
-<span class="normal">851</span>
-<span class="normal">852</span>
-<span class="normal">853</span>
-<span class="normal">854</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">save</span><span class="p">(</span>
+<span class="normal">798</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">save</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="n">save_state_dict</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">save_optimizer</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
     <span class="n">model_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;wd_model.pt&quot;</span><span class="p">,</span>
 <span class="p">):</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Saves the model, training and evaluation history, and the</span>
@@ -3931,39 +3768,27 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
 <span class="sd">        path to the directory where the model and the feature importance</span>
 <span class="sd">        attribute will be saved.</span>
 <span class="sd">    save_state_dict: bool, default = False</span>
-<span class="sd">        Boolean indicating whether to save directly the model or the</span>
-<span class="sd">        model&#39;s state dictionary</span>
+<span class="sd">        Boolean indicating whether to save directly the model</span>
+<span class="sd">        (and optimizer) or the model&#39;s (and optimizer&#39;s) state</span>
+<span class="sd">        dictionary</span>
+<span class="sd">    save_optimizer: bool, default = False</span>
+<span class="sd">        Boolean indicating whether to save the optimizer</span>
 <span class="sd">    model_filename: str, Optional, default = &quot;wd_model.pt&quot;</span>
 <span class="sd">        filename where the model weights will be store</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
-    <span class="n">save_dir</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
-    <span class="n">history_dir</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="s2">&quot;history&quot;</span>
-    <span class="n">history_dir</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_save_history</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
 
-    <span class="c1"># the trainer is run with the History Callback by default</span>
-    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">history_dir</span> <span class="o">/</span> <span class="s2">&quot;train_eval_history.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">teh</span><span class="p">:</span>
-        <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">history</span><span class="p">,</span> <span class="n">teh</span><span class="p">)</span>  <span class="c1"># type: ignore[attr-defined]</span>
-
-    <span class="n">has_lr_history</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span>
-        <span class="p">[</span><span class="n">clbk</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;LRHistory&quot;</span> <span class="k">for</span> <span class="n">clbk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">callbacks</span><span class="p">]</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_save_model_and_optimizer</span><span class="p">(</span>
+        <span class="n">path</span><span class="p">,</span> <span class="n">save_state_dict</span><span class="p">,</span> <span class="n">save_optimizer</span><span class="p">,</span> <span class="n">model_filename</span>
     <span class="p">)</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">lr_scheduler</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">has_lr_history</span><span class="p">:</span>
-        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">history_dir</span> <span class="o">/</span> <span class="s2">&quot;lr_history.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">lrh</span><span class="p">:</span>
-            <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_history</span><span class="p">,</span> <span class="n">lrh</span><span class="p">)</span>  <span class="c1"># type: ignore[attr-defined]</span>
-
-    <span class="n">model_path</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="n">model_filename</span>
-    <span class="k">if</span> <span class="n">save_state_dict</span><span class="p">:</span>
-        <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">state_dict</span><span class="p">(),</span> <span class="n">model_path</span><span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">,</span> <span class="n">model_path</span><span class="p">)</span>
 
     <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">is_tabnet</span><span class="p">:</span>
-        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">save_dir</span> <span class="o">/</span> <span class="s2">&quot;feature_importance.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fi</span><span class="p">:</span>
+        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> <span class="o">/</span> <span class="s2">&quot;feature_importance.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fi</span><span class="p">:</span>
             <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">feature_importance</span><span class="p">,</span> <span class="n">fi</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -3971,8 +3796,7 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -4048,7 +3872,7 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -4066,7 +3890,7 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html b/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html
index 09f03821..90762c5e 100644
--- a/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html
+++ b/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1027,8 +1027,6 @@
         
       
         
-      
-        
       
         
       
@@ -1366,32 +1364,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1408,11 +1385,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1429,11 +1406,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1450,11 +1427,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1471,11 +1448,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1492,11 +1469,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1513,11 +1490,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1534,11 +1511,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="../../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1601,109 +1578,107 @@ <h1 id="deeptabular-utils">deeptabular utils<a class="headerlink" href="#deeptab
 
 
 <h2 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">LabelEncoder</span>
+            <span class="doc doc-object-name doc-class-name">LabelEncoder</span>
 
 
 <a href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">LabelEncoder</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">LabelEncoder</span><span class="p">(</span>
     <span class="n">columns_to_encode</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">with_attention</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
     <span class="n">shared_embed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
+    <div class="doc doc-contents first">
+
 
-  
       <p>Label Encode categorical values for multiple columns at once</p>
 <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>:
 LabelEncoder reserves 0 for <code>unseen</code> new categories. This is convenient
 when defining the embedding layers, since we can just set padding idx to 0.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>columns_to_encode</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>columns_to_encode</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>List of strings containing the names of the columns to encode. If
+          –
+          <div class="doc-md-description">
+            <p>List of strings containing the names of the columns to encode. If
 <code>None</code> all columns of type <code>object</code> in the dataframe will be label
 encoded.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>with_attention</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>with_attention</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating whether the preprocessed data will be passed to an
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating whether the preprocessed data will be passed to an
 attention-based model. Aliased as <code>for_transformer</code>.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>shared_embed</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>shared_embed</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the embeddings will be "<em>shared</em>" when using
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the embeddings will be "<em>shared</em>" when using
 attention-based models. The idea behind <code>shared_embed</code> is described
 in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 '<em>The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the
 other columns</em>'. In other words, the idea is to let the model learn
 which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.encoding_dict">encoding_dict</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Dictionary containing the encoding mappings in the format, e.g. : <br/>
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.encoding_dict">encoding_dict</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Dictionary containing the encoding mappings in the format, e.g. : <br/>
 <code>{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_encoding_dict">inverse_encoding_dict</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Dictionary containing the inverse encoding mappings in the format, e.g. : <br/>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_encoding_dict">inverse_encoding_dict</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Dictionary containing the inverse encoding mappings in the format, e.g. : <br/>
 <code>{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}</code></p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">62</span>
-<span class="normal">63</span>
-<span class="normal">64</span>
-<span class="normal">65</span>
-<span class="normal">66</span>
-<span class="normal">67</span>
-<span class="normal">68</span>
-<span class="normal">69</span>
-<span class="normal">70</span>
-<span class="normal">71</span>
-<span class="normal">72</span>
-<span class="normal">73</span>
-<span class="normal">74</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;with_attention&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;for_transformer&quot;</span><span class="p">])</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">50</span>
+<span class="normal">51</span>
+<span class="normal">52</span>
+<span class="normal">53</span>
+<span class="normal">54</span>
+<span class="normal">55</span>
+<span class="normal">56</span>
+<span class="normal">57</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span>
+<span class="normal">62</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;with_attention&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;for_transformer&quot;</span><span class="p">])</span>
 <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">columns_to_encode</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -1717,12 +1692,11 @@ <h2 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="doc doc-he
 
     <span class="bp">self</span><span class="o">.</span><span class="n">reset_embed_idx</span> <span class="o">=</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1735,35 +1709,45 @@ <h2 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="doc doc-he
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">partial_fit</span>
+            <span class="doc doc-object-name doc-function-name">partial_fit</span>
 
 
 <a href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Main method. Creates encoding attributes.</p>
+    <div class="doc doc-contents ">
 
+      <p>Main method. Creates encoding attributes.</p>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
-        –
-        <div class="doc-md-description">
-          <p><code>LabelEncoder</code> fitted object</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
+          –
+          <div class="doc-md-description">
+            <p><code>LabelEncoder</code> fitted object</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 76</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
 <span class="normal"> 77</span>
 <span class="normal"> 78</span>
 <span class="normal"> 79</span>
@@ -1825,19 +1809,7 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit" class
 <span class="normal">135</span>
 <span class="normal">136</span>
 <span class="normal">137</span>
-<span class="normal">138</span>
-<span class="normal">139</span>
-<span class="normal">140</span>
-<span class="normal">141</span>
-<span class="normal">142</span>
-<span class="normal">143</span>
-<span class="normal">144</span>
-<span class="normal">145</span>
-<span class="normal">146</span>
-<span class="normal">147</span>
-<span class="normal">148</span>
-<span class="normal">149</span>
-<span class="normal">150</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">partial_fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;LabelEncoder&quot;</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+<span class="normal">138</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">partial_fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;LabelEncoder&quot;</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Main method. Creates encoding attributes.</span>
 
 <span class="sd">    Returns</span>
@@ -1913,58 +1885,55 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit" class
 
     <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit</span>
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
 <a href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p>
+    <div class="doc doc-contents ">
 
+      <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
-        –
-        <div class="doc-md-description">
-          <p><code>LabelEncoder</code> fitted object</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
+          –
+          <div class="doc-md-description">
+            <p><code>LabelEncoder</code> fitted object</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">152</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
 <span class="normal">153</span>
-<span class="normal">154</span>
-<span class="normal">155</span>
-<span class="normal">156</span>
-<span class="normal">157</span>
-<span class="normal">158</span>
-<span class="normal">159</span>
-<span class="normal">160</span>
-<span class="normal">161</span>
-<span class="normal">162</span>
-<span class="normal">163</span>
-<span class="normal">164</span>
-<span class="normal">165</span>
-<span class="normal">166</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;LabelEncoder&quot;</span><span class="p">:</span>
+<span class="normal">154</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;LabelEncoder&quot;</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Simply runs the `partial_fit` method when the data fits in memory</span>
 
 <span class="sd">    Returns</span>
@@ -1980,44 +1949,53 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit" class="doc do
 
     <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">transform</span>
+            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
 <a href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Label Encoded the categories in <code>columns_to_encode</code></p>
+    <div class="doc doc-contents ">
 
+      <p>Label Encoded the categories in <code>columns_to_encode</code></p>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="pandas.DataFrame">DataFrame</span></code>
-        –
-        <div class="doc-md-description">
-          <p>label-encoded dataframe</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="pandas.DataFrame">DataFrame</span></code>
+          –
+          <div class="doc-md-description">
+            <p>label-encoded dataframe</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">168</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
 <span class="normal">169</span>
 <span class="normal">170</span>
 <span class="normal">171</span>
@@ -2030,19 +2008,7 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform" class="
 <span class="normal">178</span>
 <span class="normal">179</span>
 <span class="normal">180</span>
-<span class="normal">181</span>
-<span class="normal">182</span>
-<span class="normal">183</span>
-<span class="normal">184</span>
-<span class="normal">185</span>
-<span class="normal">186</span>
-<span class="normal">187</span>
-<span class="normal">188</span>
-<span class="normal">189</span>
-<span class="normal">190</span>
-<span class="normal">191</span>
-<span class="normal">192</span>
-<span class="normal">193</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+<span class="normal">181</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Label Encoded the categories in `columns_to_encode`</span>
 
 <span class="sd">    Returns</span>
@@ -2069,31 +2035,28 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform" class="
 
     <span class="k">return</span> <span class="n">df_inp</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit_transform</span>
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
 
 
 <a href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Combines <code>fit</code> and <code>transform</code></p>
+    <div class="doc doc-contents ">
 
+      <p>Combines <code>fit</code> and <code>transform</code></p>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">LabelEncoder</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;col1&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">],</span> <span class="s1">&#39;col2&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;me&#39;</span><span class="p">,</span> <span class="s1">&#39;you&#39;</span><span class="p">,</span> <span class="s1">&#39;him&#39;</span><span class="p">]})</span>
@@ -2109,21 +2072,32 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform" cla
 </code></pre></div>
 
 
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="pandas.DataFrame">DataFrame</span></code>
+          –
+          <div class="doc-md-description">
+            <p>label-encoded dataframe</p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="pandas.DataFrame">DataFrame</span></code>
-        –
-        <div class="doc-md-description">
-          <p>label-encoded dataframe</p>
-      </div>
-      </li>
-  </ul>
-
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">195</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
 <span class="normal">196</span>
 <span class="normal">197</span>
 <span class="normal">198</span>
@@ -2135,19 +2109,7 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform" cla
 <span class="normal">204</span>
 <span class="normal">205</span>
 <span class="normal">206</span>
-<span class="normal">207</span>
-<span class="normal">208</span>
-<span class="normal">209</span>
-<span class="normal">210</span>
-<span class="normal">211</span>
-<span class="normal">212</span>
-<span class="normal">213</span>
-<span class="normal">214</span>
-<span class="normal">215</span>
-<span class="normal">216</span>
-<span class="normal">217</span>
-<span class="normal">218</span>
-<span class="normal">219</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+<span class="normal">207</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
 
 <span class="sd">    Examples</span>
@@ -2173,31 +2135,28 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform" cla
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">inverse_transform</span>
+            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
 
 
 <a href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">inverse_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Returns the original categories</p>
+    <div class="doc doc-contents ">
 
+      <p>Returns the original categories</p>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">LabelEncoder</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;col1&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">],</span> <span class="s1">&#39;col2&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;me&#39;</span><span class="p">,</span> <span class="s1">&#39;you&#39;</span><span class="p">,</span> <span class="s1">&#39;him&#39;</span><span class="p">]})</span>
@@ -2212,21 +2171,32 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform"
 </code></pre></div>
 
 
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="pandas.DataFrame">DataFrame</span></code>
+          –
+          <div class="doc-md-description">
+            <p>DataFrame with original categories</p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="pandas.DataFrame">DataFrame</span></code>
-        –
-        <div class="doc-md-description">
-          <p>DataFrame with original categories</p>
-      </div>
-      </li>
-  </ul>
-
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">228</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
 <span class="normal">229</span>
 <span class="normal">230</span>
 <span class="normal">231</span>
@@ -2244,19 +2214,7 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform"
 <span class="normal">243</span>
 <span class="normal">244</span>
 <span class="normal">245</span>
-<span class="normal">246</span>
-<span class="normal">247</span>
-<span class="normal">248</span>
-<span class="normal">249</span>
-<span class="normal">250</span>
-<span class="normal">251</span>
-<span class="normal">252</span>
-<span class="normal">253</span>
-<span class="normal">254</span>
-<span class="normal">255</span>
-<span class="normal">256</span>
-<span class="normal">257</span>
-<span class="normal">258</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+<span class="normal">246</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the original categories</span>
 
 <span class="sd">    Examples</span>
@@ -2288,8 +2246,8 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform"
 
     <span class="k">return</span> <span class="n">df</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2297,8 +2255,7 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform"
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2373,7 +2330,7 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform"
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -2391,7 +2348,7 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform"
     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html b/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html
index 0b94247c..9faf3791 100644
--- a/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html
+++ b/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1036,8 +1036,6 @@
         
       
         
-      
-        
       
         
       
@@ -1375,32 +1373,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1417,11 +1394,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1438,11 +1415,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1459,11 +1436,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1480,11 +1457,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1501,11 +1478,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1522,11 +1499,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1543,11 +1520,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="../../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1617,11 +1594,11 @@ <h1 id="fastai-transforms">Fastai transforms<a class="headerlink" href="#fastai-
 
 
 <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Tokenizer</span>
+            <span class="doc doc-object-name doc-class-name">Tokenizer</span>
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Tokenizer</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Tokenizer</span><span class="p">(</span>
     <span class="n">tok_func</span><span class="o">=</span><span class="n">SpacyTokenizer</span><span class="p">,</span>
     <span class="n">lang</span><span class="o">=</span><span class="s2">&quot;en&quot;</span><span class="p">,</span>
     <span class="n">pre_rules</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -1631,88 +1608,87 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
+    <div class="doc doc-contents first">
+
 
-  
       <p>Class to combine a series of rules and a tokenizer function to tokenize
 text with multiprocessing.</p>
 <p>Setting some of the parameters of this class require perhaps some
 familiarity with the source code.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>tok_func</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Callable">Callable</span></code>, default:
-                <code><span title="pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer">SpacyTokenizer</span></code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>tok_func</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Callable">Callable</span></code>, default:
+                  <code><span title="pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer">SpacyTokenizer</span></code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lang</code></b>
-            (<code>str</code>, default:
-                <code>&#39;en&#39;</code>
+          –
+          <div class="doc-md-description">
+            <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>lang</code></b>
+              (<code>str</code>, default:
+                  <code>&#39;en&#39;</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Text's Language</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pre_rules</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.ListRules">ListRules</span>]</code>, default:
-                <code>None</code>
+          –
+          <div class="doc-md-description">
+            <p>Text's Language</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pre_rules</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.ListRules">ListRules</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are
+          –
+          <div class="doc-md-description">
+            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are
 <code>Callable</code> objects that will be applied to the text (str) directly as
 <code>rule(tok)</code> before being tokenized.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>post_rules</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.ListRules">ListRules</span>]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>post_rules</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.ListRules">ListRules</span>]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are
+          –
+          <div class="doc-md-description">
+            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are
 <code>Callable</code> objects that will be applied to the tokens as
 <code>rule(tokens)</code> after the text has been tokenized.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>special_cases</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Collection">Collection</span>[str]]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>special_cases</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Collection">Collection</span>[str]]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>special cases to be added to the tokenizer via <code>Spacy</code>'s
+          –
+          <div class="doc-md-description">
+            <p>special cases to be added to the tokenizer via <code>Spacy</code>'s
 <code>add_special_case</code> method</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_cpus</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_cpus</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of CPUs to used during the tokenization process</p>
-        </div>
-      </li>
-  </ul>
+          –
+          <div class="doc-md-description">
+            <p>number of CPUs to used during the tokenization process</p>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">255</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">255</span>
 <span class="normal">256</span>
 <span class="normal">257</span>
 <span class="normal">258</span>
@@ -1744,12 +1720,11 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
     <span class="p">)</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="o">=</span> <span class="n">ifnone</span><span class="p">(</span><span class="n">n_cpus</span><span class="p">,</span> <span class="n">defaults</span><span class="o">.</span><span class="n">cpus</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1762,58 +1737,55 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">process_text</span>
+            <span class="doc doc-object-name doc-function-name">process_text</span>
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">process_text</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">tok</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">process_text</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">tok</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Process and tokenize one text <code>t</code> with tokenizer <code>tok</code>.</p>
+    <div class="doc doc-contents ">
 
+      <p>Process and tokenize one text <code>t</code> with tokenizer <code>tok</code>.</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>t</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>text to be processed and tokenized</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>tok</code></b>
-            (<code><span title="pytorch_widedeep.utils.fastai_transforms.BaseTokenizer">BaseTokenizer</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Instance of <code>BaseTokenizer</code>. See
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>t</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>text to be processed and tokenized</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>tok</code></b>
+              (<code><span title="pytorch_widedeep.utils.fastai_transforms.BaseTokenizer">BaseTokenizer</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Instance of <code>BaseTokenizer</code>. See
 <code>pytorch_widedeep.utils.fastai_transforms.BaseTokenizer</code></p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
-        –
-        <div class="doc-md-description">
-          <p>List of tokens</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
+          –
+          <div class="doc-md-description">
+            <p>List of tokens</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">280</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">280</span>
 <span class="normal">281</span>
 <span class="normal">282</span>
 <span class="normal">283</span>
@@ -1857,31 +1829,28 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text" class="
         <span class="n">toks</span> <span class="o">=</span> <span class="n">rule</span><span class="p">(</span><span class="n">toks</span><span class="p">)</span>
     <span class="k">return</span> <span class="n">toks</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">process_all</span>
+            <span class="doc doc-object-name doc-function-name">process_all</span>
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">process_all</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">process_all</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Process a list of texts. Parallel execution of <code>process_text</code>.</p>
+    <div class="doc doc-contents ">
 
+      <p>Process a list of texts. Parallel execution of <code>process_text</code>.</p>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">Tokenizer</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;Machine learning is great&#39;</span><span class="p">,</span> <span class="s1">&#39;but building stuff is even better&#39;</span><span class="p">]</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">tok</span> <span class="o">=</span> <span class="n">Tokenizer</span><span class="p">()</span>
@@ -1894,21 +1863,20 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all" class="d
 details of special tokens please see the <a href="https://docs.fast.ai/text.core.html#Tokenizing"><code>fastai</code> docs</a>.</p>
 
 
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+          –
+          <div class="doc-md-description">
+            <p>List containing lists of tokens. One list per "<em>document</em>"</p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
-        –
-        <div class="doc-md-description">
-          <p>List containing lists of tokens. One list per "<em>document</em>"</p>
-      </div>
-      </li>
-  </ul>
-
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">311</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">311</span>
 <span class="normal">312</span>
 <span class="normal">313</span>
 <span class="normal">314</span>
@@ -1966,8 +1934,8 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all" class="d
             <span class="n">e</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_all_1</span><span class="p">,</span> <span class="n">partition_by_cores</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)),</span> <span class="p">[]</span>
         <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -1975,8 +1943,7 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all" class="d
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -1985,79 +1952,77 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all" class="d
 
 
 <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">Vocab</span>
+            <span class="doc doc-object-name doc-class-name">Vocab</span>
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Vocab" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">Vocab</span><span class="p">(</span><span class="n">max_vocab</span><span class="p">,</span> <span class="n">min_freq</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Vocab</span><span class="p">(</span><span class="n">max_vocab</span><span class="p">,</span> <span class="n">min_freq</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
+    <div class="doc doc-contents first">
 
-  
-      <p>Contains the correspondence between numbers and tokens.</p>
 
+      <p>Contains the correspondence between numbers and tokens.</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>max_vocab</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>maximum vocabulary size</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>min_freq</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>minimum frequency for a token to be considereds</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pad_idx</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>max_vocab</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>maximum vocabulary size</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>min_freq</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>minimum frequency for a token to be considereds</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pad_idx</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index
+          –
+          <div class="doc-md-description">
+            <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index
 for the unknown token (<em>'xxunk'</em>) and defaults to 1 for the padding
 token (<em>'xxpad'</em>).</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Attributes:</p>
-  <ul>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.utils.fastai_transforms.Vocab.itos">itos</span></code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Collection">Collection</span></code>)
-        –
-        <div class="doc-md-description">
-          <p><code>index to str</code>. Collection of strings that are the tokens of the
+<p><span class="doc-section-title">Attributes:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.utils.fastai_transforms.Vocab.itos">itos</span></code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Collection">Collection</span></code>)
+          –
+          <div class="doc-md-description">
+            <p><code>index to str</code>. Collection of strings that are the tokens of the
 vocabulary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code><span title="pytorch_widedeep.utils.fastai_transforms.Vocab.stoi">stoi</span></code></b>
-            (<code><span title="collections.defaultdict">defaultdict</span></code>)
-        –
-        <div class="doc-md-description">
-          <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code><span title="pytorch_widedeep.utils.fastai_transforms.Vocab.stoi">stoi</span></code></b>
+              (<code><span title="collections.defaultdict">defaultdict</span></code>)
+          –
+          <div class="doc-md-description">
+            <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and
 their corresponding index</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">366</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">366</span>
 <span class="normal">367</span>
 <span class="normal">368</span>
 <span class="normal">369</span>
@@ -2075,12 +2040,11 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
     <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span> <span class="o">=</span> <span class="n">min_freq</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span> <span class="o">=</span> <span class="n">pad_idx</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -2093,38 +2057,35 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.create" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">create</span>
+            <span class="doc doc-object-name doc-function-name">create</span>
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Vocab.create" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">create</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">create</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Create a vocabulary object from a set of tokens.</p>
+    <div class="doc doc-contents ">
 
+      <p>Create a vocabulary object from a set of tokens.</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>tokens</code></b>
-            (<code><span title="pytorch_widedeep.wdtypes.Tokens">Tokens</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Custom type: <code>Collection[Collection[str]]</code>  see
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>tokens</code></b>
+              (<code><span title="pytorch_widedeep.wdtypes.Tokens">Tokens</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Custom type: <code>Collection[Collection[str]]</code>  see
 <code>pytorch_widedeep.wdtypes</code>. Collection of collection of
 strings (e.g. list of tokenized sentences)</p>
-        </div>
-      </li>
-  </ul>
-
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">Tokenizer</span><span class="p">,</span> <span class="n">Vocab</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;Machine learning is great&#39;</span><span class="p">,</span> <span class="s1">&#39;but building stuff is even better&#39;</span><span class="p">]</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">tokens</span> <span class="o">=</span> <span class="n">Tokenizer</span><span class="p">()</span><span class="o">.</span><span class="n">process_all</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
@@ -2140,21 +2101,20 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.create" class="doc doc-he
 classification/Regression tasks. Please see the <a href="https://docs.fast.ai/text.core.html#Tokenizing"><code>fastai</code> docs</a>.</p>
 
 
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>
+          –
+          <div class="doc-md-description">
+            <p>An instance of a <code>Vocab</code> object</p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>
-        –
-        <div class="doc-md-description">
-          <p>An instance of a <code>Vocab</code> object</p>
-      </div>
-      </li>
-  </ul>
-
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">376</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">376</span>
 <span class="normal">377</span>
 <span class="normal">378</span>
 <span class="normal">379</span>
@@ -2280,32 +2240,30 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.create" class="doc doc-he
 
     <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.fit" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">fit</span>
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Vocab.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">fit</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Calls the <code>create</code> method. I simply want to honor fast ai naming, but
 for consistency with the rest of the library I am including a fit method</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">440</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">440</span>
 <span class="normal">441</span>
 <span class="normal">442</span>
 <span class="normal">443</span>
@@ -2323,44 +2281,41 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.fit" class="doc doc-headi
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">numericalize</span>
+            <span class="doc doc-object-name doc-function-name">numericalize</span>
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">numericalize</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">numericalize</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Convert a list of tokens <code>t</code> to their ids.</p>
+    <div class="doc doc-contents ">
 
+      <p>Convert a list of tokens <code>t</code> to their ids.</p>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
-        –
-        <div class="doc-md-description">
-          <p>List of '<em>numericalsed</em>' tokens</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
+          –
+          <div class="doc-md-description">
+            <p>List of '<em>numericalsed</em>' tokens</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">450</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">450</span>
 <span class="normal">451</span>
 <span class="normal">452</span>
 <span class="normal">453</span>
@@ -2378,33 +2333,31 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize" class="doc
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">stoi</span><span class="p">[</span><span class="n">w</span><span class="p">]</span> <span class="k">for</span> <span class="n">w</span> <span class="ow">in</span> <span class="n">t</span><span class="p">]</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">transform</span>
+            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Vocab.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">transform</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Calls the <code>numericalize</code> method. I simply want to honor fast ai naming,
 but for consistency with the rest of the library I am including a
 transform method</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">460</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">460</span>
 <span class="normal">461</span>
 <span class="normal">462</span>
 <span class="normal">463</span>
@@ -2418,44 +2371,41 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.transform" class="doc doc
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">numericalize</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.textify" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">textify</span>
+            <span class="doc doc-object-name doc-function-name">textify</span>
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Vocab.textify" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">textify</span><span class="p">(</span><span class="n">nums</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39; &#39;</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">textify</span><span class="p">(</span><span class="n">nums</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39; &#39;</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p>
+    <div class="doc doc-contents ">
 
+      <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
-        –
-        <div class="doc-md-description">
-          <p>List of tokens</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
+          –
+          <div class="doc-md-description">
+            <p>List of tokens</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">468</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">468</span>
 <span class="normal">469</span>
 <span class="normal">470</span>
 <span class="normal">471</span>
@@ -2481,33 +2431,31 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.textify" class="doc doc-h
         <span class="k">else</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">itos</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">nums</span><span class="p">]</span>
     <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">inverse_transform</span>
+            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">inverse_transform</span><span class="p">(</span><span class="n">nums</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39; &#39;</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">nums</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39; &#39;</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
+    <div class="doc doc-contents ">
+
       <p>Calls the <code>textify</code> method. I simply want to honor fast ai naming, but
 for consistency with the rest of the library I am including an
 inverse_transform method</p>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">482</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">482</span>
 <span class="normal">483</span>
 <span class="normal">484</span>
 <span class="normal">485</span>
@@ -2529,8 +2477,8 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class=
     <span class="c1"># rest of the library I am including an inverse_transform method</span>
     <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">textify</span><span class="p">(</span><span class="n">nums</span><span class="p">,</span> <span class="n">sep</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2538,8 +2486,7 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class=
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2615,7 +2562,7 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class=
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -2633,7 +2580,7 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class=
     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/utils/image_utils.html b/mkdocs/site/pytorch-widedeep/utils/image_utils.html
index 3b91e3e6..d7e3aa08 100644
--- a/mkdocs/site/pytorch-widedeep/utils/image_utils.html
+++ b/mkdocs/site/pytorch-widedeep/utils/image_utils.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1036,8 +1036,6 @@
         
       
         
-      
-        
       
         
       
@@ -1375,32 +1373,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1417,11 +1394,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1438,11 +1415,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1459,11 +1436,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1480,11 +1457,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1501,11 +1478,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1522,11 +1499,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1543,11 +1520,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="../../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1615,54 +1592,53 @@ <h1 id="image-utils">Image utils<a class="headerlink" href="#image-utils" title=
 
 
 <h2 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">AspectAwarePreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">AspectAwarePreprocessor</span>
 
 
 <a href="#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">AspectAwarePreprocessor</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">AspectAwarePreprocessor</span><span class="p">(</span>
     <span class="n">width</span><span class="p">,</span> <span class="n">height</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
+    <div class="doc doc-contents first">
+
 
-  
       <p>Class to resize an image to a certain width and height taking into account
 the image aspect ratio</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>width</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>output width</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>height</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>output height</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>inter</code></b>
-        –
-        <div class="doc-md-description">
-          <p><code>opencv</code> interpolation method. See <code>opencv</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>width</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>output width</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>height</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>output height</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>inter</code></b>
+          –
+          <div class="doc-md-description">
+            <p><code>opencv</code> interpolation method. See <code>opencv</code>
 <code>InterpolationFlags</code>.</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">32</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">32</span>
 <span class="normal">33</span>
 <span class="normal">34</span>
 <span class="normal">35</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">width</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">height</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span><span class="p">):</span>
@@ -1670,12 +1646,11 @@ <h2 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="doc d
     <span class="bp">self</span><span class="o">.</span><span class="n">height</span> <span class="o">=</span> <span class="n">height</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">inter</span> <span class="o">=</span> <span class="n">inter</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1688,36 +1663,33 @@ <h2 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="doc d
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">preprocess</span>
+            <span class="doc doc-object-name doc-function-name">preprocess</span>
 
 
 <a href="#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">preprocess</span><span class="p">(</span><span class="n">image</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">preprocess</span><span class="p">(</span><span class="n">image</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Returns the resized input image taking into account the image aspect ratio</p>
-
+    <div class="doc doc-contents ">
 
+      <p>Returns the resized input image taking into account the image aspect ratio</p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>image</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input image to be resized</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>image</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input image to be resized</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">cv2</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">AspectAwarePreprocessor</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">img</span> <span class="o">=</span> <span class="n">cv2</span><span class="o">.</span><span class="n">imread</span><span class="p">(</span><span class="s2">&quot;tests/test_data_utils/images/galaxy1.png&quot;</span><span class="p">)</span>
@@ -1730,21 +1702,20 @@ <h3 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess" c
 </code></pre></div>
 
 
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Resized image according to its original image aspect ratio</p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Resized image according to its original image aspect ratio</p>
-      </div>
-      </li>
-  </ul>
-
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">37</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">37</span>
 <span class="normal">38</span>
 <span class="normal">39</span>
 <span class="normal">40</span>
@@ -1832,8 +1803,8 @@ <h3 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess" c
 
     <span class="k">return</span> <span class="n">resized_image</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -1841,8 +1812,7 @@ <h3 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess" c
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -1851,51 +1821,50 @@ <h3 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess" c
 
 
 <h2 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="doc doc-heading">
-          <span class="doc doc-object-name doc-class-name">SimplePreprocessor</span>
+            <span class="doc doc-object-name doc-class-name">SimplePreprocessor</span>
 
 
 <a href="#pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">SimplePreprocessor</span><span class="p">(</span><span class="n">width</span><span class="p">,</span> <span class="n">height</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">SimplePreprocessor</span><span class="p">(</span><span class="n">width</span><span class="p">,</span> <span class="n">height</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
+    <div class="doc doc-contents first">
 
-  
-      <p>Class to resize an image to a certain width and height</p>
 
+      <p>Class to resize an image to a certain width and height</p>
 
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>width</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>output width</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>height</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>output height</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>inter</code></b>
-        –
-        <div class="doc-md-description">
-          <p><code>opencv</code> interpolation method. See <code>opencv</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>width</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>output width</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>height</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>output height</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>inter</code></b>
+          –
+          <div class="doc-md-description">
+            <p><code>opencv</code> interpolation method. See <code>opencv</code>
 <code>InterpolationFlags</code>.</p>
-        </div>
-      </li>
-  </ul>
+          </div>
+        </li>
+    </ul>
 
-                <details class="quote">
-                  <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
-                  <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 97</span>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 97</span>
 <span class="normal"> 98</span>
 <span class="normal"> 99</span>
 <span class="normal">100</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">width</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">height</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span><span class="p">):</span>
@@ -1903,12 +1872,11 @@ <h2 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="doc doc-he
     <span class="bp">self</span><span class="o">.</span><span class="n">height</span> <span class="o">=</span> <span class="n">height</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">inter</span> <span class="o">=</span> <span class="n">inter</span>
 </code></pre></div></td></tr></table></div>
-                </details>
+                  </details>
 
-  
 
-  <div class="doc doc-children">
 
+  <div class="doc doc-children">
 
 
 
@@ -1921,49 +1889,46 @@ <h2 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="doc doc-he
 <div class="doc doc-object doc-function">
 
 
-
 <h3 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">preprocess</span>
+            <span class="doc doc-object-name doc-function-name">preprocess</span>
 
 
 <a href="#pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">preprocess</span><span class="p">(</span><span class="n">image</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">preprocess</span><span class="p">(</span><span class="n">image</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents ">
-  
-      <p>Returns the resized input image</p>
-
+    <div class="doc doc-contents ">
 
+      <p>Returns the resized input image</p>
 
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>image</code></b>
-            (<code><span title="numpy.ndarray">ndarray</span></code>)
-        –
-        <div class="doc-md-description">
-          <p>Input image to be resized</p>
-        </div>
-      </li>
-  </ul>
 
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>image</code></b>
+              (<code><span title="numpy.ndarray">ndarray</span></code>)
+          –
+          <div class="doc-md-description">
+            <p>Input image to be resized</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Resized image</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Resized image</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">102</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">102</span>
 <span class="normal">103</span>
 <span class="normal">104</span>
 <span class="normal">105</span>
@@ -2001,8 +1966,8 @@ <h3 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess" class=
 
     <span class="k">return</span> <span class="n">resized_image</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2010,8 +1975,7 @@ <h3 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess" class=
 
   </div>
 
-  </div>
-
+    </div>
 
 </div>
 
@@ -2086,7 +2050,7 @@ <h3 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess" class=
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -2104,7 +2068,7 @@ <h3 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess" class=
     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/utils/index.html b/mkdocs/site/pytorch-widedeep/utils/index.html
index 304b4615..0582c272 100644
--- a/mkdocs/site/pytorch-widedeep/utils/index.html
+++ b/mkdocs/site/pytorch-widedeep/utils/index.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -978,8 +978,6 @@
         
       
         
-      
-        
       
         
       
@@ -1317,32 +1315,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1359,11 +1336,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1380,11 +1357,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1401,11 +1378,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1422,11 +1399,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1443,11 +1420,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1464,11 +1441,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1485,11 +1462,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="../../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1626,7 +1603,7 @@ <h1 id="the-utils-module">The <code>utils</code> module<a class="headerlink" hre
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -1644,7 +1621,7 @@ <h1 id="the-utils-module">The <code>utils</code> module<a class="headerlink" hre
     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/utils/text_utils.html b/mkdocs/site/pytorch-widedeep/utils/text_utils.html
index 27cc7e51..fd281126 100644
--- a/mkdocs/site/pytorch-widedeep/utils/text_utils.html
+++ b/mkdocs/site/pytorch-widedeep/utils/text_utils.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../../assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="../../assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -372,7 +372,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -1054,8 +1054,6 @@
         
       
         
-      
-        
       
         
       
@@ -1393,32 +1391,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1435,11 +1412,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1456,11 +1433,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="../../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1477,11 +1454,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="../../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1498,11 +1475,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1519,11 +1496,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1540,11 +1517,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="../../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1561,11 +1538,11 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="../../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1624,105 +1601,100 @@ <h1 id="text-utils">Text utils<a class="headerlink" href="#text-utils" title="Pe
 <p>Collection of helper function that facilitate processing text.</p>
 
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h2 id="pytorch_widedeep.utils.text_utils.simple_preprocess" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">simple_preprocess</span>
+            <span class="doc doc-object-name doc-function-name">simple_preprocess</span>
 
 
 <a href="#pytorch_widedeep.utils.text_utils.simple_preprocess" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">simple_preprocess</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">simple_preprocess</span><span class="p">(</span>
     <span class="n">doc</span><span class="p">,</span> <span class="n">lower</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">deacc</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">max_len</span><span class="o">=</span><span class="mi">15</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-  
+    <div class="doc doc-contents first">
+
       <p>This is <code>Gensim</code>'s <code>simple_preprocess</code> with a <code>lower</code> param to
 indicate wether or not to lower case all the token in the doc</p>
 <p>For more information see: <code>Gensim</code> <a href="https://radimrehurek.com/gensim/utils.html">utils module</a></p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>doc</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>Input document.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>lower</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>doc</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>Input document.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>lower</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Lower case tokens in the input doc</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>deacc</code></b>
-            (<code>bool</code>, default:
-                <code>False</code>
+          –
+          <div class="doc-md-description">
+            <p>Lower case tokens in the input doc</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>deacc</code></b>
+              (<code>bool</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>min_len</code></b>
-            (<code>int</code>, default:
-                <code>2</code>
+          –
+          <div class="doc-md-description">
+            <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>min_len</code></b>
+              (<code>int</code>, default:
+                  <code>2</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>max_len</code></b>
-            (<code>int</code>, default:
-                <code>15</code>
+          –
+          <div class="doc-md-description">
+            <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>max_len</code></b>
+              (<code>int</code>, default:
+                  <code>15</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">simple_preprocess</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">simple_preprocess</span><span class="p">(</span><span class="s1">&#39;Machine learning is great&#39;</span><span class="p">)</span>
 <span class="go">[&#39;Machine&#39;, &#39;learning&#39;, &#39;is&#39;, &#39;great&#39;]</span>
 </code></pre></div>
 
 
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="typing.List">List</span>[str]</code>
+          –
+          <div class="doc-md-description">
+            <p>List with the processed tokens</p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="typing.List">List</span>[str]</code>
-        –
-        <div class="doc-md-description">
-          <p>List with the processed tokens</p>
-      </div>
-      </li>
-  </ul>
-
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">16</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">16</span>
 <span class="normal">17</span>
 <span class="normal">18</span>
 <span class="normal">19</span>
@@ -1808,70 +1780,66 @@ <h2 id="pytorch_widedeep.utils.text_utils.simple_preprocess" class="doc doc-head
     <span class="p">]</span>
     <span class="k">return</span> <span class="n">tokens</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">get_texts</span>
+            <span class="doc doc-object-name doc-function-name">get_texts</span>
 
 
 <a href="#pytorch_widedeep.utils.text_utils.get_texts" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="n">already_processed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">n_cpus</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="n">already_processed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">n_cpus</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-  
+    <div class="doc doc-contents first">
+
       <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a
 series of very convenients things during the tokenization process</p>
 <p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>texts</code></b>
-            (<code><span title="typing.List">List</span>[str]</code>)
-        –
-        <div class="doc-md-description">
-          <p>List of str with the texts (or documents). One str per document</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>already_processed</code></b>
-            (<code><span title="typing.Optional">Optional</span>[bool]</code>, default:
-                <code>False</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>texts</code></b>
+              (<code><span title="typing.List">List</span>[str]</code>)
+          –
+          <div class="doc-md-description">
+            <p>List of str with the texts (or documents). One str per document</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>already_processed</code></b>
+              (<code><span title="typing.Optional">Optional</span>[bool]</code>, default:
+                  <code>False</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Boolean indicating if the text is already processed and we simply want
+          –
+          <div class="doc-md-description">
+            <p>Boolean indicating if the text is already processed and we simply want
 to tokenize it. This parameter is thought for those cases where the
 input sequences might not be text (but IDs, or anything else) and we
 just want to tokenize it</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>n_cpus</code></b>
-            (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
-                <code>None</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>n_cpus</code></b>
+              (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
+                  <code>None</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>number of CPUs to used during the tokenization process</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>number of CPUs to used during the tokenization process</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">get_texts</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;Machine learning is great&#39;</span><span class="p">,</span> <span class="s1">&#39;but building stuff is even better&#39;</span><span class="p">]</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
@@ -1879,63 +1847,62 @@ <h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
 </code></pre></div>
 
 
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="typing.List">List</span>[<span title="typing.List">List</span>[str]]</code>
+          –
+          <div class="doc-md-description">
+            <p>List of lists, one list per '<em>document</em>' containing its corresponding tokens</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+              <code>information_source: **NOTE**:</code>
+          –
+          <div class="doc-md-description">
+            
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+              <code>`get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</code>
+          –
+          <div class="doc-md-description">
+            
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+              <code>Such tokenizer uses a series of convenient processing steps, including</code>
+          –
+          <div class="doc-md-description">
+            
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+              <code>the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</code>
+          –
+          <div class="doc-md-description">
+            
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+              <code>indicate the next word begins with a capital in the original text. For more</code>
+          –
+          <div class="doc-md-description">
+            
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+              <code>details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</code>
+          –
+          <div class="doc-md-description">
+            
+          </div>
+        </li>
+    </ul>
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="typing.List">List</span>[<span title="typing.List">List</span>[str]]</code>
-        –
-        <div class="doc-md-description">
-          <p>List of lists, one list per '<em>document</em>' containing its corresponding tokens</p>
-      </div>
-      </li>
-      <li class="field-body">
-            <code>information_source: **NOTE**:</code>
-        –
-        <div class="doc-md-description">
-          
-      </div>
-      </li>
-      <li class="field-body">
-            <code>`get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</code>
-        –
-        <div class="doc-md-description">
-          
-      </div>
-      </li>
-      <li class="field-body">
-            <code>Such tokenizer uses a series of convenient processing steps, including</code>
-        –
-        <div class="doc-md-description">
-          
-      </div>
-      </li>
-      <li class="field-body">
-            <code>the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</code>
-        –
-        <div class="doc-md-description">
-          
-      </div>
-      </li>
-      <li class="field-body">
-            <code>indicate the next word begins with a capital in the original text. For more</code>
-        –
-        <div class="doc-md-description">
-          
-      </div>
-      </li>
-      <li class="field-body">
-            <code>details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</code>
-        –
-        <div class="doc-md-description">
-          
-      </div>
-      </li>
-  </ul>
-
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 61</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 61</span>
 <span class="normal"> 62</span>
 <span class="normal"> 63</span>
 <span class="normal"> 64</span>
@@ -2035,75 +2002,71 @@ <h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
     <span class="n">tok</span> <span class="o">=</span> <span class="n">Tokenizer</span><span class="p">(</span><span class="n">n_cpus</span><span class="o">=</span><span class="n">num_cpus</span><span class="p">)</span><span class="o">.</span><span class="n">process_all</span><span class="p">(</span><span class="n">processed_texts</span><span class="p">)</span>
     <span class="k">return</span> <span class="n">tok</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h2 id="pytorch_widedeep.utils.text_utils.pad_sequences" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">pad_sequences</span>
+            <span class="doc doc-object-name doc-function-name">pad_sequences</span>
 
 
 <a href="#pytorch_widedeep.utils.text_utils.pad_sequences" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">pad_sequences</span><span class="p">(</span><span class="n">seq</span><span class="p">,</span> <span class="n">maxlen</span><span class="p">,</span> <span class="n">pad_first</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">pad_sequences</span><span class="p">(</span><span class="n">seq</span><span class="p">,</span> <span class="n">maxlen</span><span class="p">,</span> <span class="n">pad_first</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-  
+    <div class="doc doc-contents first">
+
       <p>Given a List of tokenized and <code>numericalised</code> sequences it will return
 padded sequences according to the input parameters.</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>seq</code></b>
-            (<code><span title="typing.List">List</span>[int]</code>)
-        –
-        <div class="doc-md-description">
-          <p>List of int with the <code>numericalised</code> tokens</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>maxlen</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>Maximum length of the padded sequences</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pad_first</code></b>
-            (<code>bool</code>, default:
-                <code>True</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>seq</code></b>
+              (<code><span title="typing.List">List</span>[int]</code>)
+          –
+          <div class="doc-md-description">
+            <p>List of int with the <code>numericalised</code> tokens</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>maxlen</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>Maximum length of the padded sequences</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pad_first</code></b>
+              (<code>bool</code>, default:
+                  <code>True</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>Indicates whether the padding index will be added at the beginning or the
+          –
+          <div class="doc-md-description">
+            <p>Indicates whether the padding index will be added at the beginning or the
 end of the sequences</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>pad_idx</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>pad_idx</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
+          </div>
+        </li>
+    </ul>
 
 
-<p><strong>Examples:</strong></p>
+<p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">pad_sequences</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">seq</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">]</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">pad_sequences</span><span class="p">(</span><span class="n">seq</span><span class="p">,</span> <span class="n">maxlen</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
@@ -2111,21 +2074,20 @@ <h2 id="pytorch_widedeep.utils.text_utils.pad_sequences" class="doc doc-heading"
 </code></pre></div>
 
 
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>numpy array with the padded sequences</p>
+          </div>
+        </li>
+    </ul>
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>numpy array with the padded sequences</p>
-      </div>
-      </li>
-  </ul>
-
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">113</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">113</span>
 <span class="normal">114</span>
 <span class="normal">115</span>
 <span class="normal">116</span>
@@ -2211,89 +2173,85 @@ <h2 id="pytorch_widedeep.utils.text_utils.pad_sequences" class="doc doc-heading"
             <span class="n">res</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">seq</span><span class="p">)</span> <span class="p">:]</span> <span class="o">=</span> <span class="n">seq</span>
         <span class="k">return</span> <span class="n">res</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
-
 <div class="doc doc-object doc-function">
 
 
-
 <h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc doc-heading">
-          <span class="doc doc-object-name doc-function-name">build_embeddings_matrix</span>
+            <span class="doc doc-object-name doc-function-name">build_embeddings_matrix</span>
 
 
 <a href="#pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">build_embeddings_matrix</span><span class="p">(</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">build_embeddings_matrix</span><span class="p">(</span>
     <span class="n">vocab</span><span class="p">,</span> <span class="n">word_vectors_path</span><span class="p">,</span> <span class="n">min_freq</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span>
 <span class="p">)</span>
 </code></pre></div>
 
-  <div class="doc doc-contents first">
-  
+    <div class="doc doc-contents first">
+
       <p>Build the embedding matrix using pretrained word vectors.</p>
 <p>Returns pretrained word embeddings. If a word in our vocabulary is not
 among the pretrained embeddings it will be assigned the mean pretrained
 word-embeddings vector</p>
 
 
-
-  <p>Parameters:</p>
-  <ul>
-      <li class="field-body">
-        <b><code>vocab</code></b>
-            (<code><span title="typing.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a>, <span title="pytorch_widedeep.utils.fastai_transforms.ChunkVocab">ChunkVocab</span>]</code>)
-        –
-        <div class="doc-md-description">
-          <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>word_vectors_path</code></b>
-            (<code>str</code>)
-        –
-        <div class="doc-md-description">
-          <p>path to the pretrained word embeddings</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>min_freq</code></b>
-            (<code>int</code>)
-        –
-        <div class="doc-md-description">
-          <p>minimum frequency required for a word to be in the vocabulary</p>
-        </div>
-      </li>
-      <li class="field-body">
-        <b><code>verbose</code></b>
-            (<code>int</code>, default:
-                <code>1</code>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+          <b><code>vocab</code></b>
+              (<code><span title="typing.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a>, <span title="pytorch_widedeep.utils.fastai_transforms.ChunkVocab">ChunkVocab</span>]</code>)
+          –
+          <div class="doc-md-description">
+            <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>word_vectors_path</code></b>
+              (<code>str</code>)
+          –
+          <div class="doc-md-description">
+            <p>path to the pretrained word embeddings</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>min_freq</code></b>
+              (<code>int</code>)
+          –
+          <div class="doc-md-description">
+            <p>minimum frequency required for a word to be in the vocabulary</p>
+          </div>
+        </li>
+        <li class="doc-section-item field-body">
+          <b><code>verbose</code></b>
+              (<code>int</code>, default:
+                  <code>1</code>
 )
-        –
-        <div class="doc-md-description">
-          <p>level of verbosity. Set to 0 for no verbosity</p>
-        </div>
-      </li>
-  </ul>
-
+          –
+          <div class="doc-md-description">
+            <p>level of verbosity. Set to 0 for no verbosity</p>
+          </div>
+        </li>
+    </ul>
 
 
-  <p>Returns:</p>
-  <ul>
-      <li class="field-body">
-            <code><span title="numpy.ndarray">ndarray</span></code>
-        –
-        <div class="doc-md-description">
-          <p>Pretrained word embeddings</p>
-      </div>
-      </li>
-  </ul>
+<p><span class="doc-section-title">Returns:</span></p>
+    <ul>
+        <li class="doc-section-item field-body">
+              <code><span title="numpy.ndarray">ndarray</span></code>
+          –
+          <div class="doc-md-description">
+            <p>Pretrained word embeddings</p>
+          </div>
+        </li>
+    </ul>
 
-          <details class="quote">
-            <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
-            <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">158</span>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">158</span>
 <span class="normal">159</span>
 <span class="normal">160</span>
 <span class="normal">161</span>
@@ -2427,8 +2385,8 @@ <h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc do
 
     <span class="k">return</span> <span class="n">embedding_matrix</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float32&quot;</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-          </details>
-  </div>
+            </details>
+    </div>
 
 </div>
 
@@ -2503,7 +2461,7 @@ <h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc do
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -2521,7 +2479,7 @@ <h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc do
     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../../assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="../../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="../../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/quick_start.html b/mkdocs/site/quick_start.html
index 3f453050..73b1df19 100644
--- a/mkdocs/site/quick_start.html
+++ b/mkdocs/site/quick_start.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.13">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.31">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="assets/stylesheets/main.7e359304.min.css">
+      <link rel="stylesheet" href="assets/stylesheets/main.3cba04c6.min.css">
       
         
         <link rel="stylesheet" href="assets/stylesheets/palette.06af60db.min.css">
@@ -176,7 +176,7 @@
       
     </form>
     <div class="md-search__output">
-      <div class="md-search__scrollwrap" data-md-scrollfix>
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
         <div class="md-search-result" data-md-component="search-result">
           <div class="md-search-result__meta">
             Initializing search
@@ -193,7 +193,7 @@
         <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -370,7 +370,7 @@
       <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
   <div class="md-source__icon md-icon">
     
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
   </div>
   <div class="md-source__repository">
     pytorch_widedeep
@@ -983,8 +983,6 @@
         
       
         
-      
-        
       
         
       
@@ -1322,32 +1320,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/15_DIR-LDS_and_FDS.html" class="md-nav__link">
-        
-  
-  <span class="md-ellipsis">
-    15_DIR-LDS_and_FDS
-  </span>
-  
-
-      </a>
-    </li>
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
-    <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
+      <a href="examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 1
+    15_Self-Supervised Pre-Training pt 1
   </span>
   
 
@@ -1364,11 +1341,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
+      <a href="examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    16_Self-Supervised Pre-Training pt 2
+    15_Self-Supervised Pre-Training pt 2
   </span>
   
 
@@ -1385,11 +1362,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/17_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
+      <a href="examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    17_Usign-a-custom-hugging-face-model
+    16_Usign-a-custom-hugging-face-model
   </span>
   
 
@@ -1406,11 +1383,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/18_feature_importance_via_attention_weights.html" class="md-nav__link">
+      <a href="examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    18_feature_importance_via_attention_weights
+    17_feature_importance_via_attention_weights
   </span>
   
 
@@ -1427,11 +1404,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
+      <a href="examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt1
+    18_wide_and_deep_for_recsys_pt1
   </span>
   
 
@@ -1448,11 +1425,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
+      <a href="examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    19_wide_and_deep_for_recsys_pt2
+    18_wide_and_deep_for_recsys_pt2
   </span>
   
 
@@ -1469,11 +1446,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/20_load_from_folder_functionality.html" class="md-nav__link">
+      <a href="examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    20_load_from_folder_functionality
+    19_load_from_folder_functionality
   </span>
   
 
@@ -1490,11 +1467,11 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/21_Using_huggingface_within_widedeep.html" class="md-nav__link">
+      <a href="examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    21-Using-huggingface-within-widedeep
+    20-Using-huggingface-within-widedeep
   </span>
   
 
@@ -1724,7 +1701,7 @@ <h1 id="quick-start">Quick Start<a class="headerlink" href="#quick-start" title=
       
     
     <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
-      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262Zm288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
     </a>
   
 </div>
@@ -1742,7 +1719,7 @@ <h1 id="quick-start">Quick Start<a class="headerlink" href="#quick-start" title=
     <script id="__config" type="application/json">{"base": ".", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="assets/javascripts/bundle.c8d2eff1.min.js"></script>
+      <script src="assets/javascripts/bundle.fe8b6f2b.min.js"></script>
       
         <script src="stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/search/search_index.json b/mkdocs/site/search/search_index.json
index 77524a09..0f52f1ad 100644
--- a/mkdocs/site/search/search_index.json
+++ b/mkdocs/site/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"index.html","title":"Home","text":""},{"location":"index.html#pytorch-widedeep","title":"pytorch-widedeep","text":"<p>A flexible package for multimodal-deep-learning to combine tabular data with text and images using Wide and Deep models in Pytorch</p> <p>Documentation: https://pytorch-widedeep.readthedocs.io</p> <p>Companion posts and tutorials: infinitoml</p> <p>Experiments and comparison with <code>LightGBM</code>: TabularDL vs LightGBM</p> <p>Slack: if you want to contribute or just want to chat with us, join slack</p> <p>The content of this document is organized as follows:</p> <ul> <li>pytorch-widedeep<ul> <li>Introduction</li> <li>Architectures</li> <li>The <code>deeptabular</code> component</li> <li>Text and Images</li> <li>Acknowledgments</li> <li>License</li> <li>Cite<ul> <li>BibTex</li> <li>APA</li> </ul> </li> </ul> </li> </ul>"},{"location":"index.html#introduction","title":"Introduction","text":"<p><code>pytorch-widedeep</code> is based on Google's Wide and Deep Algorithm, adjusted for multi-modal datasets.</p> <p>In general terms, <code>pytorch-widedeep</code> is a package to use deep learning with tabular data. In particular, is intended to facilitate the combination of text and images with corresponding tabular data using wide and deep models. With that in mind there are a number of architectures that can be implemented with the library. The main components of those architectures are shown in the Figure below:</p> <p> </p> <p>In math terms, and following the notation in the paper, the expression for the architecture without a <code>deephead</code> component can be formulated as:</p> <p> </p> <p>Where \u03c3 is the sigmoid function, 'W' are the weight matrices applied to the wide model and to the final activations of the deep models, 'a' are these final activations, \u03c6(x) are the cross product transformations of the original features 'x', and , and 'b' is the bias term. In case you are wondering what are \"cross product transformations\", here is a quote taken directly from the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>It is perfectly possible to use custom models (and not necessarily those in the library) as long as the the custom models have a property called <code>output_dim</code> with the size of the last layer of activations, so that <code>WideDeep</code> can be constructed. Examples on how to use custom components can be found in the Examples folder and the section below.</p>"},{"location":"index.html#architectures","title":"Architectures","text":"<p>The <code>pytorch-widedeep</code> library offers a number of different architectures. In this section we will show some of them in their simplest form (i.e. with default param values in most cases) with their corresponding code snippets. Note that all the snippets below shoud run locally. For a more detailed explanation of the different components and their parameters, please refer to the documentation.</p> <p>For the examples below we will be using a toy dataset generated as follows:</p> <pre><code>import os\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom faker import Faker\n\n\ndef create_and_save_random_image(image_number, size=(32, 32)):\n\n    if not os.path.exists(\"images\"):\n        os.makedirs(\"images\")\n\n    array = np.random.randint(0, 256, (size[0], size[1], 3), dtype=np.uint8)\n\n    image = Image.fromarray(array)\n\n    image_name = f\"image_{image_number}.png\"\n    image.save(os.path.join(\"images\", image_name))\n\n    return image_name\n\n\nfake = Faker()\n\ncities = [\"New York\", \"Los Angeles\", \"Chicago\", \"Houston\"]\nnames = [\"Alice\", \"Bob\", \"Charlie\", \"David\", \"Eva\"]\n\ndata = {\n    \"city\": [random.choice(cities) for _ in range(100)],\n    \"name\": [random.choice(names) for _ in range(100)],\n    \"age\": [random.uniform(18, 70) for _ in range(100)],\n    \"height\": [random.uniform(150, 200) for _ in range(100)],\n    \"sentence\": [fake.sentence() for _ in range(100)],\n    \"other_sentence\": [fake.sentence() for _ in range(100)],\n    \"image_name\": [create_and_save_random_image(i) for i in range(100)],\n    \"target\": [random.choice([0, 1]) for _ in range(100)],\n}\n\ndf = pd.DataFrame(data)\n</code></pre> <p>This will create a 100 rows dataframe and a dir in your local folder, called <code>images</code> with 100 random images (or images with just noise).</p> <p>Perhaps the simplest architecture would be just one component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> or <code>deepimage</code> on their own, which is also possible, but let's start the examples with a standard Wide and Deep architecture. From there, how to build a model comprised only of one component will be straightforward.</p> <p>Note that the examples shown below would be almost identical using any of the models available in the library. For example, <code>TabMlp</code> can be replaced by <code>TabResnet</code>, <code>TabNet</code>, <code>TabTransformer</code>, etc. Similarly, <code>BasicRNN</code> can be replaced by <code>AttentiveRNN</code>, <code>StackedAttentiveRNN</code>, or <code>HFModel</code> with their corresponding parameters and preprocessor in the case of the Hugging Face models.</p> <p>1. Wide and Tabular component (aka deeptabular)</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Wide\nwide_cols = [\"city\"]\ncrossed_cols = [(\"city\", \"name\")]\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\nwide = Wide(input_dim=np.unique(X_wide).shape[0])\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# WideDeep\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>2. Tabular and Text data</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>3. Tabular and text with a FC head on top via the <code>head_hidden_dims</code> param   in <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn, head_hidden_dims=[32, 16])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>4. Tabular and multiple text columns that are passed directly to   <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=[rnn_1, rnn_2])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>5. Tabular data and multiple text columns that are fused via a the library's   <code>ModelFuser</code> class</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\n\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\nmodels_fuser = ModelFuser(models=[rnn_1, rnn_2], fusion_method=\"mult\")\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=models_fuser)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>6. Tabular and multiple text columns, with an image column. The text columns   are fused via the library's <code>ModelFuser</code> and then all fused via the   deephead paramenter in <code>WideDeep</code> which is a custom <code>ModelFuser</code> coded by   the user</p> <p>This is perhaps the less elegant solution as it involves a custom component by the user and slicing the 'incoming' tensor. In the future, we will include a <code>TextAndImageModelFuser</code> to make this process more straightforward. Still, is not really complicated and it is a good example of how to use custom components in <code>pytorch-widedeep</code>.</p> <p>Note that the only requirement for the custom component is that it has a property called <code>output_dim</code> that returns the size of the last layer of activations. In other words, it does not need to inherit from <code>BaseWDModelComponent</code>. This base class simply checks the existence of such property and avoids some typing errors internally.</p> <p> </p> <pre><code>import torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[16, 8],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nmodels_fuser = ModelFuser(\n    models=[rnn_1, rnn_2],\n    fusion_method=\"mult\",\n)\n\n# Image\nimage_preprocessor = ImagePreprocessor(img_col=\"image_name\", img_path=\"images\")\nX_img = image_preprocessor.fit_transform(df)\nvision = Vision(pretrained_model_setup=\"resnet18\", head_hidden_dims=[16, 8])\n\n# deephead (custom model fuser)\nclass MyModelFuser(BaseWDModelComponent):\n    \"\"\"\n    Simply a Linear + Relu sequence on top of the text + images followed by a\n    Linear -&gt; Relu -&gt; Linear for the concatenation of tabular slice of the\n    tensor and the output of the text and image sequential model\n    \"\"\"\n    def __init__(\n        self,\n        tab_incoming_dim: int,\n        text_incoming_dim: int,\n        image_incoming_dim: int,\n        output_units: int,\n    ):\n\n        super(MyModelFuser, self).__init__()\n\n        self.tab_incoming_dim = tab_incoming_dim\n        self.text_incoming_dim = text_incoming_dim\n        self.image_incoming_dim = image_incoming_dim\n        self.output_units = output_units\n        self.text_and_image_fuser = torch.nn.Sequential(\n            torch.nn.Linear(text_incoming_dim + image_incoming_dim, output_units),\n            torch.nn.ReLU(),\n        )\n        self.out = torch.nn.Sequential(\n            torch.nn.Linear(output_units + tab_incoming_dim, output_units * 4),\n            torch.nn.ReLU(),\n            torch.nn.Linear(output_units * 4, output_units),\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        tab_slice = slice(0, self.tab_incoming_dim)\n        text_slice = slice(\n            self.tab_incoming_dim, self.tab_incoming_dim + self.text_incoming_dim\n        )\n        image_slice = slice(\n            self.tab_incoming_dim + self.text_incoming_dim,\n            self.tab_incoming_dim + self.text_incoming_dim + self.image_incoming_dim,\n        )\n        X_tab = X[:, tab_slice]\n        X_text = X[:, text_slice]\n        X_img = X[:, image_slice]\n        X_text_and_image = self.text_and_image_fuser(torch.cat([X_text, X_img], dim=1))\n        return self.out(torch.cat([X_tab, X_text_and_image], dim=1))\n\n    @property\n    def output_dim(self):\n        return self.output_units\n\ndeephead = MyModelFuser(\n    tab_incoming_dim=tab_mlp.output_dim,\n    text_incoming_dim=models_fuser.output_dim,\n    image_incoming_dim=vision.output_dim,\n    output_units=8,\n)\n\n# WideDeep\nmodel = WideDeep(\n    deeptabular=tab_mlp,\n    deeptext=models_fuser,\n    deepimage=vision,\n    deephead=deephead,\n)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    X_img=X_img,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>7. Tabular with a multi-target loss</p> <p>This one is \"a bonus\" to illustrate the use of multi-target losses, more than actually a different architecture.</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# let's add a second target to the dataframe\ndf[\"target2\"] = [random.choice([0, 1]) for _ in range(100)]\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# 'pred_dim=2' because we have two binary targets. For other types of targets,\n#  please, see the documentation\nmodel = WideDeep(deeptabular=tab_mlp, pred_dim=2).\n\nloss = MultiTargetClassificationLoss(binary_config=[0, 1], reduction=\"mean\")\n\n# When a multi-target loss is used, 'custom_loss_function' must not be None.\n# See the docs\ntrainer = Trainer(model, objective=\"multitarget\", custom_loss_function=loss)\n\ntrainer.fit(\n    X_tab=X_tab,\n    target=df[[\"target\", \"target2\"]].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre>"},{"location":"index.html#the-deeptabular-component","title":"The <code>deeptabular</code> component","text":"<p>It is important to emphasize again that each individual component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code>, can be used independently and in isolation. For example, one could use only <code>wide</code>, which is in simply a linear model. In fact, one of the most interesting functionalities in<code>pytorch-widedeep</code> would be the use of the <code>deeptabular</code> component on its own, i.e. what one might normally refer as Deep Learning for Tabular Data. Currently, <code>pytorch-widedeep</code> offers the following different models for that component:</p> <ol> <li>Wide: a simple linear model where the nonlinearities are captured via cross-product transformations, as explained before.</li> <li>TabMlp: a simple MLP that receives embeddings representing the categorical features, concatenated with the continuous features, which can also be embedded.</li> <li>TabResnet: similar to the previous model but the embeddings are passed through a series of ResNet blocks built with dense layers.</li> <li>TabNet: details on TabNet can be found in TabNet: Attentive Interpretable Tabular Learning</li> </ol> <p>Two simpler attention based models that we call:</p> <ol> <li>ContextAttentionMLP: MLP with at attention mechanism \"on top\" that is based on     Hierarchical Attention Networks for Document Classification</li> <li>SelfAttentionMLP: MLP with an attention mechanism that is a simplified     version of a transformer block that we refer as \"query-key self-attention\".</li> </ol> <p>The <code>Tabformer</code> family, i.e. Transformers for Tabular data:</p> <ol> <li>TabTransformer: details on the TabTransformer can be found in TabTransformer: Tabular Data Modeling Using Contextual Embeddings.</li> <li>SAINT: Details on SAINT can be found in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training.</li> <li>FT-Transformer: details on the FT-Transformer can be found in Revisiting Deep Learning Models for Tabular Data.</li> <li>TabFastFormer: adaptation of the FastFormer for tabular data. Details on the Fasformer can be found in FastFormers: Highly Efficient Transformer Models for Natural Language Understanding</li> <li>TabPerceiver: adaptation of the Perceiver for tabular data. Details on the Perceiver can be found in Perceiver: General Perception with Iterative Attention</li> </ol> <p>And probabilistic DL models for tabular data based on Weight Uncertainty in Neural Networks:</p> <ol> <li>BayesianWide: Probabilistic adaptation of the <code>Wide</code> model.</li> <li>BayesianTabMlp: Probabilistic adaptation of the <code>TabMlp</code> model</li> </ol> <p>Note that while there are scientific publications for the TabTransformer, SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own adaptation of those algorithms for tabular data.</p> <p>In addition, Self-Supervised pre-training can be used for all <code>deeptabular</code> models, with the exception of the <code>TabPerceiver</code>. Self-Supervised pre-training can be used via two methods or routines which we refer as: encoder-decoder method and constrastive-denoising method. Please, see the documentation and the examples for details on this functionality, and all other options in the library.</p>"},{"location":"index.html#text-and-images","title":"Text and Images","text":"<p>For the text component, <code>deeptext</code>, the library offers the following models:</p> <ol> <li>BasicRNN: a simple RNN 2. AttentiveRNN: a RNN with an attention mechanism based on the Hierarchical Attention Networks for DocumentClassification</li> <li>StackedAttentiveRNN: a stack of AttentiveRNNs</li> <li>HFModel: a wrapper around Hugging Face Transfomer-based models. At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks. If there is demand for other models, they will be included in the future.</li> </ol> <p>For the image component, <code>deepimage</code>, the library supports models from the following families: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3',  'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'.  These are  offered via <code>torchvision</code> and wrapped up in the <code>Vision</code> class.</p>"},{"location":"index.html#acknowledgments","title":"Acknowledgments","text":"<p>This library takes from a series of other libraries, so I think it is just fair to mention them here in the README (specific mentions are also included in the code).</p> <p>The <code>Callbacks</code> and <code>Initializers</code> structure and code is inspired by the <code>torchsample</code> library, which in itself partially inspired by <code>Keras</code>.</p> <p>The <code>TextProcessor</code> class in this library uses the <code>fastai</code>'s <code>Tokenizer</code> and <code>Vocab</code>. The code at <code>utils.fastai_transforms</code> is a minor adaptation of their code so it functions within this library. To my experience their <code>Tokenizer</code> is the best in class.</p> <p>The <code>ImageProcessor</code> class in this library uses code from the fantastic Deep Learning for Computer Vision (DL4CV) book by Adrian Rosebrock.</p>"},{"location":"index.html#license","title":"License","text":"<p>This work is dual-licensed under Apache 2.0 and MIT (or any later version). You can choose between one of them if you use this work.</p> <p><code>SPDX-License-Identifier: Apache-2.0 AND MIT</code></p>"},{"location":"index.html#cite","title":"Cite","text":""},{"location":"index.html#bibtex","title":"BibTex","text":"<pre><code>@article{Zaurin_pytorch-widedeep_A_flexible_2023,\nauthor = {Zaurin, Javier Rodriguez and Mulinka, Pavol},\ndoi = {10.21105/joss.05027},\njournal = {Journal of Open Source Software},\nmonth = jun,\nnumber = {86},\npages = {5027},\ntitle = {{pytorch-widedeep: A flexible package for multimodal deep learning}},\nurl = {https://joss.theoj.org/papers/10.21105/joss.05027},\nvolume = {8},\nyear = {2023}\n}\n</code></pre>"},{"location":"index.html#apa","title":"APA","text":"<pre><code>Zaurin, J. R., &amp; Mulinka, P. (2023). pytorch-widedeep: A flexible package for\nmultimodal deep learning. Journal of Open Source Software, 8(86), 5027.\nhttps://doi.org/10.21105/joss.05027\n</code></pre>"},{"location":"contributing.html","title":"Contributing","text":"<p>Pytorch-widedeep is being developed and used by many active community members. Your help is very valuable to make it better for everyone.</p> <ul> <li>Check for the Roadmap or Open an issue to report problems or recommend new features and submit a draft pull requests, which will be changed to pull request after intial review</li> <li>Contribute to the tests to make it more reliable.</li> <li>Contribute to the documentation to make it clearer for everyone.</li> <li>Contribute to the examples to share your experience with other users.</li> <li>Join the dicussion on slack</li> </ul>"},{"location":"installation.html","title":"Installation","text":"<p>This section explains how to install <code>pytorch-widedeep</code>.</p> <p>For the latest stable release, execute:</p> <pre><code>pip install pytorch-widedeep\n</code></pre> <p>For the bleeding-edge version, execute:</p> <pre><code>pip install git+https://github.com/jrzaurin/pytorch-widedeep.git\n</code></pre> <p>For developer install</p> <pre><code># Clone the repository\ngit clone https://github.com/jrzaurin/pytorch-widedeep\ncd pytorch-widedeep\n\n# Install in dev mode\npip install -e .\n</code></pre>"},{"location":"installation.html#dependencies","title":"Dependencies","text":"<ul> <li>pandas&gt;=1.3.5</li> <li>numpy&gt;=1.21.6</li> <li>scipy&gt;=1.7.3,&lt;=1.12.0</li> <li>scikit-learn&gt;=1.0.2</li> <li>gensim</li> <li>spacy</li> <li>opencv-contrib-python</li> <li>imutils</li> <li>tqdm</li> <li>torch &gt;= 2.0.0</li> <li>torchvision &gt;= 0.15.0</li> <li>einops</li> <li>wrapt</li> <li>torchmetrics</li> <li>pyarrow</li> <li>fastparquet&gt;=0.8.1</li> <li>transformers</li> <li>sentence-transformers</li> <li>sentencepiece</li> </ul>"},{"location":"quick_start.html","title":"Quick Start","text":"<p>This is an example of a binary classification with the adult census dataset using a combination of a wide and deep model (in this case a so called <code>deeptabular</code> model) with defaults settings.</p> <pre><code>import numpy as np\nimport torch\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n\n\ndf = load_adult(as_frame=True)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\n# Define the 'column set up'\nwide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital-status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital-gain\",\n    \"capital-loss\",\n    \"native-country\",\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\ntarget = \"income_label\"\ntarget = df_train[target].values\n\n# prepare the data\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df_train)\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]\n)\nX_tab = tab_preprocessor.fit_transform(df_train)\n\n# build the model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# train and validate\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=5,\n    batch_size=256,\n)\n\n# predict on test\nX_wide_te = wide_preprocessor.transform(df_test)\nX_tab_te = tab_preprocessor.transform(df_test)\npreds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)\n\n# Save and load\n\n# Option 1: this will also save training history and lr history if the\n# LRHistory callback is used\ntrainer.save(path=\"model_weights\", save_state_dict=True)\n\n# Option 2: save as any other torch model\ntorch.save(model.state_dict(), \"model_weights/wd_model.pt\")\n\n# From here in advance, Option 1 or 2 are the same. I assume the user has\n# prepared the data and defined the new model components:\n# 1. Build the model\nmodel_new = WideDeep(wide=wide, deeptabular=tab_mlp)\nmodel_new.load_state_dict(torch.load(\"model_weights/wd_model.pt\"))\n\n# 2. Instantiate the trainer\ntrainer_new = Trainer(model_new, objective=\"binary\")\n\n# 3. Either start the fit or directly predict\npreds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab, batch_size=32)\n</code></pre>"},{"location":"examples/01_preprocessors_and_utils.html","title":"01_preprocessors_and_utils","text":"<p>For example</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport pytorch_widedeep as wd\n\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import WidePreprocessor\n</pre> import numpy as np import pandas as pd import pytorch_widedeep as wd  from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import WidePreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre>wide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n</pre> wide_cols = [     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native-country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")] In\u00a0[4]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_wide = wide_preprocessor.transform(new_df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_wide = wide_preprocessor.transform(new_df) In\u00a0[5]: Copied! <pre>X_wide\n</pre> X_wide Out[5]: <pre>array([[  1,  17,  23, ...,  89,  91, 316],\n       [  2,  18,  23, ...,  89,  92, 317],\n       [  3,  18,  24, ...,  89,  93, 318],\n       ...,\n       [  2,  20,  23, ...,  90, 103, 323],\n       [  2,  17,  23, ...,  89, 103, 323],\n       [  2,  21,  29, ...,  90, 115, 324]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[6]: Copied! <pre>X_wide[0]\n</pre> X_wide[0] Out[6]: <pre>array([  1,  17,  23,  32,  47,  89,  91, 316])</pre> In\u00a0[7]: Copied! <pre>wide_preprocessor.inverse_transform(X_wide[:1])\n</pre> wide_preprocessor.inverse_transform(X_wide[:1]) Out[7]: education relationship workclass occupation native-country gender education_occupation native-country_occupation 0 11th Own-child Private Machine-op-inspct United-States Male 11th-Machine-op-inspct United-States-Machine-op-inspct <p>As we can see, <code>wide_preprocessor</code> numerically encodes the <code>wide_cols</code> and the <code>crossed_cols</code>, which can be recovered using the method <code>inverse_transform</code>.</p> In\u00a0[8]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[9]: Copied! <pre># cat_embed_cols = [(column_name, embed_dim), ...]\ncat_embed_cols = [\n    (\"education\", 10),\n    (\"relationship\", 8),\n    (\"workclass\", 10),\n    (\"occupation\", 10),\n    (\"native-country\", 10),\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\n</pre> # cat_embed_cols = [(column_name, embed_dim), ...] cat_embed_cols = [     (\"education\", 10),     (\"relationship\", 8),     (\"workclass\", 10),     (\"occupation\", 10),     (\"native-country\", 10), ] continuous_cols = [\"age\", \"hours-per-week\"] In\u00a0[10]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_deep = deep_preprocessor.transform(new_df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_deep = deep_preprocessor.transform(new_df) In\u00a0[11]: Copied! <pre>X_tab\n</pre> X_tab Out[11]: <pre>array([[ 1.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -9.95128932e-01,  4.00000000e+01],\n       [ 2.00000000e+00,  2.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -4.69415091e-02,  5.00000000e+01],\n       [ 3.00000000e+00,  2.00000000e+00,  2.00000000e+00, ...,\n         1.00000000e+00, -7.76316450e-01,  4.00000000e+01],\n       ...,\n       [ 2.00000000e+00,  4.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00,  1.41180837e+00,  4.00000000e+01],\n       [ 2.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -1.21394141e+00,  2.00000000e+01],\n       [ 2.00000000e+00,  5.00000000e+00,  7.00000000e+00, ...,\n         1.00000000e+00,  9.74183408e-01,  4.00000000e+01]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[12]: Copied! <pre>X_tab[0]\n</pre> X_tab[0] Out[12]: <pre>array([ 1.        ,  1.        ,  1.        ,  1.        ,  1.        ,\n       -0.99512893, 40.        ])</pre> In\u00a0[13]: Copied! <pre>tab_preprocessor.inverse_transform(X_tab[:1])\n</pre> tab_preprocessor.inverse_transform(X_tab[:1]) Out[13]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 25.0 40.0 <p>The <code>TabPreprocessor</code> will have a series of useful attributes that can later be used when instantiating the different Tabular Models, such us for example, the column indexes (used to slice the tensors, internally in the models) or the categorical embeddings set up</p> In\u00a0[14]: Copied! <pre>tab_preprocessor.column_idx\n</pre> tab_preprocessor.column_idx Out[14]: <pre>{'education': 0,\n 'relationship': 1,\n 'workclass': 2,\n 'occupation': 3,\n 'native-country': 4,\n 'age': 5,\n 'hours-per-week': 6}</pre> In\u00a0[15]: Copied! <pre># column name, num unique, embedding dim\ntab_preprocessor.cat_embed_input\n</pre> # column name, num unique, embedding dim tab_preprocessor.cat_embed_input Out[15]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10)]</pre> <p>As I mentioned, there is more one can do, such as for example, quantize (or bucketize) the continuous cols. For this we could use the <code>quantization_setup</code> param. This parameter accepts a number of different inputs and uses <code>pd.cut</code> under the hood to quantize the continuous cols. For more info, please, read the docs. Let's use it here to quantize \"age\" and \"hours-per-week\" in 4 and 5 \"buckets\" respectively</p> In\u00a0[16]: Copied! <pre>quantization_setup = {\n    \"age\": 4,\n    \"hours-per-week\": 5,\n}  # you can also pass a list of floats with the boundaries if you wanted\nquant_tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    quantization_setup=quantization_setup,\n)\nqX_tab = quant_tab_preprocessor.fit_transform(df)\n</pre> quantization_setup = {     \"age\": 4,     \"hours-per-week\": 5, }  # you can also pass a list of floats with the boundaries if you wanted quant_tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     quantization_setup=quantization_setup, ) qX_tab = quant_tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>qX_tab\n</pre> qX_tab Out[17]: <pre>array([[1, 1, 1, ..., 1, 1, 2],\n       [2, 2, 1, ..., 1, 2, 3],\n       [3, 2, 2, ..., 1, 1, 2],\n       ...,\n       [2, 4, 1, ..., 1, 3, 2],\n       [2, 1, 1, ..., 1, 1, 1],\n       [2, 5, 7, ..., 1, 2, 2]])</pre> <p>Note that the continuous columns that have been bucketised into quantiles are treated as any other categorical column</p> In\u00a0[18]: Copied! <pre>quant_tab_preprocessor.cat_embed_input\n</pre> quant_tab_preprocessor.cat_embed_input Out[18]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10),\n ('age', 4, 4),\n ('hours-per-week', 5, 4)]</pre> <p>Where the column 'age' has now 4 categories, which will be encoded using embeddings of 4 dims. Note that, as any other categorical columns, the categorical \"counter\" starts with 1. This is because all incoming values that are lower/higher than the existing lowest/highest value in the train (or already seen) dataset, will be encoded as 0.</p> In\u00a0[19]: Copied! <pre>np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]])\n</pre> np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]]) Out[19]: <pre>array([1, 2, 3, 4])</pre> <p>Finally, if we now wanted to <code>inverse_transform</code> the transformed array into the original dataframe, we could still do it, but the continuous, bucketized columns will be transformed back to the middle of their quantile/bucket range</p> In\u00a0[20]: Copied! <pre>df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab)\n</pre> df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> In\u00a0[21]: Copied! <pre>df.head(2)\n</pre> df.head(2) Out[21]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K In\u00a0[22]: Copied! <pre>df_decoded.head(2)\n</pre> df_decoded.head(2) Out[22]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 26.0885 30.4 1 HS-grad Husband Private Farming-fishing United-States 44.3750 50.0 <p>there is one final comment to make regarding to the <code>inverse_transform</code> functionality. As we mentioned before, the encoding <code>0</code> is reserved for values that fall outside the range covered by the data we used to run the <code>fit</code> method. For example</p> In\u00a0[23]: Copied! <pre>df.age.min(), df.age.max()\n</pre> df.age.min(), df.age.max() Out[23]: <pre>(17, 90)</pre> <p>All future age values outside that range will be encoded as 0 and decoded as <code>NaN</code></p> In\u00a0[24]: Copied! <pre>tmp_df = df.head(1).copy()\ntmp_df.loc[:, \"age\"] = 5\n</pre> tmp_df = df.head(1).copy() tmp_df.loc[:, \"age\"] = 5 In\u00a0[25]: Copied! <pre>tmp_df\n</pre> tmp_df Out[25]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 5 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K In\u00a0[26]: Copied! <pre># quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90\ntmp_qX_tab = quant_tab_preprocessor.transform(tmp_df)\n</pre> # quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90 tmp_qX_tab = quant_tab_preprocessor.transform(tmp_df) In\u00a0[27]: Copied! <pre>tmp_qX_tab\n</pre> tmp_qX_tab Out[27]: <pre>array([[1, 1, 1, 1, 1, 0, 2]])</pre> In\u00a0[28]: Copied! <pre>quant_tab_preprocessor.inverse_transform(tmp_qX_tab)\n</pre> quant_tab_preprocessor.inverse_transform(tmp_qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> Out[28]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States NaN 30.4 In\u00a0[29]: Copied! <pre>from pytorch_widedeep.preprocessing import TextPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TextPreprocessor In\u00a0[30]: Copied! <pre># The airbnb dataset, which you could get from here:\n# http://insideairbnb.com/get-the-data.html, is too big to be included in\n# our datasets module (when including images). Therefore, go there,\n# download it, and use the download_images.py script to get the images\n# and the airbnb_data_processing.py to process the data. We'll find\n# better datasets in the future ;). Note that here we are only using a\n# small sample to illustrate the use, so PLEASE ignore the results, just\n# focus on usage\ndf = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\n</pre> # The airbnb dataset, which you could get from here: # http://insideairbnb.com/get-the-data.html, is too big to be included in # our datasets module (when including images). Therefore, go there, # download it, and use the download_images.py script to get the images # and the airbnb_data_processing.py to process the data. We'll find # better datasets in the future ;). Note that here we are only using a # small sample to illustrate the use, so PLEASE ignore the results, just # focus on usage df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") In\u00a0[31]: Copied! <pre>texts = df.description.tolist()\ntexts[:2]\n</pre> texts = df.description.tolist() texts[:2] Out[31]: <pre>[\"My bright double bedroom with a large window has a relaxed feeling! It comfortably fits one or two and is centrally located just two blocks from Finsbury Park. Enjoy great restaurants in the area and easy access to easy transport tubes, trains and buses. Babies and children of all ages are welcome. Hello Everyone, I'm offering my lovely double bedroom in Finsbury Park area (zone 2) for let in a shared apartment.  You will share the apartment with me and it is fully furnished with a self catering kitchen. Two people can easily sleep well as the room has a queen size bed. I also have a travel cot for a baby for guest with small children.  I will require a deposit up front as a security gesture on both our parts and will be given back to you when you return the keys.  I trust anyone who will be responding to this add would treat my home with care and respect .  Best Wishes  Alina Guest will have access to the self catering kitchen and bathroom. There is the flat is equipped wifi internet,\",\n \"Lots of windows and light.  St Luke's Gardens are at the end of the block, and the river not too far the other way. Ten minutes walk if you go slowly. Buses to everywhere round the corner and shops, restaurants, pubs, the cinema and Waitrose . Bright Chelsea Apartment  This is a bright one bedroom ground floor apartment in an interesting listed building. There is one double bedroom and a living room/kitchen The apartment has a full  bathroom and the kitchen is fully equipped. Two wardrobes are available exclusively for guests and bedside tables and two long drawers. This sunny convenient compact flat is just around the corner from the Waitrose supermarket and all sorts of shops, cinemas, restaurants and pubs.  This is a lovely part of London. There is a fun farmers market in the King's Road at the weekend.  Buses to everywhere are just round the corner, and two underground stations are within ten minutes walk. There is a very nice pub round by St. Luke's gardens, 4 mins slow walk, the \"]</pre> In\u00a0[32]: Copied! <pre>text_preprocessor = TextPreprocessor(text_col=\"description\")\nX_text = text_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_text = text_preprocessor.transform(new_df)\n</pre> text_preprocessor = TextPreprocessor(text_col=\"description\") X_text = text_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_text = text_preprocessor.transform(new_df) <pre>The vocabulary contains 2192 tokens\n</pre> In\u00a0[33]: Copied! <pre>print(X_text[0])\n</pre> print(X_text[0]) <pre>[  29   48   37  367  818   17  910   17  177   15  122  349   53  879\n 1174  126  393   40  911    0   23  228   71  819    9   53   55 1380\n  225   11   18  308   18 1564   10  755    0  942  239   53   55    0\n   11   36 1013  277 1974   70   62   15 1475    9  943    5  251    5\n    0    5    0    5  177   53   37   75   11   10  294  726   32    9\n   42    5   25   12   10   22   12  136  100  145]\n</pre> In\u00a0[34]: Copied! <pre>from pytorch_widedeep.preprocessing import ImagePreprocessor\n</pre> from pytorch_widedeep.preprocessing import ImagePreprocessor In\u00a0[35]: Copied! <pre>image_preprocessor = wd.preprocessing.ImagePreprocessor(\n    img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\"\n)\nX_images = image_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_images = image_preprocessor.transform(new_df)\n</pre> image_preprocessor = wd.preprocessing.ImagePreprocessor(     img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\" ) X_images = image_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_images = image_preprocessor.transform(new_df) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 667.89it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[36]: Copied! <pre>X_images[0].shape\n</pre> X_images[0].shape Out[36]: <pre>(224, 224, 3)</pre>"},{"location":"examples/01_preprocessors_and_utils.html#processors-and-utils","title":"Processors and Utils\u00b6","text":"<p>Description of the main tools and utilities that one needs to prepare the data for a <code>WideDeep</code> model constructor.</p>"},{"location":"examples/01_preprocessors_and_utils.html#the-preprocessing-module","title":"The <code>preprocessing</code>  module\u00b6","text":"<p>There are 4 preprocessors, corresponding to 4 main components of the <code>WideDeep</code> model. These are</p> <ul> <li><code>WidePreprocessor</code></li> <li><code>TabPreprocessor</code></li> <li><code>TextPreprocessor</code></li> <li><code>ImagePreprocessor</code></li> </ul> <p>Behind the scenes, these preprocessors use a series of helper funcions and classes that are in the <code>utils</code> module. If you were interested please go and have a look to the documentation</p>"},{"location":"examples/01_preprocessors_and_utils.html#1-widepreprocessor","title":"1. WidePreprocessor\u00b6","text":"<p>The <code>wide</code> component of the model is a linear model that in principle, could be implemented as a linear layer receiving the result of on one-hot encoding categorical columns. However, this is not memory efficient. Therefore, we implement a liner layer as an Embedding layer plus a bias. I will explain in a bit more detail later.</p> <p>With that in mind, <code>WidePreprocessor</code> simply encodes the categories numerically so that they are the indexes of the lookup table that is an Embedding layer.</p>"},{"location":"examples/01_preprocessors_and_utils.html#2-tabpreprocessor","title":"2. TabPreprocessor\u00b6","text":"<p>The <code>TabPreprocessor</code> has a lot of different functionalities. Let's explore some of them in detail. In its basic use, the <code>TabPreprocessor</code> simply label encodes the categorical columns and normalises the numerical ones (unless otherwised specified).</p>"},{"location":"examples/01_preprocessors_and_utils.html#3-textpreprocessor","title":"3. TextPreprocessor\u00b6","text":"<p>This preprocessor returns the tokenised, padded sequences that will be directly fed to the stack of LSTMs.</p>"},{"location":"examples/01_preprocessors_and_utils.html#4-imagepreprocessor","title":"4. ImagePreprocessor\u00b6","text":"<p><code>ImagePreprocessor</code> simply resizes the images, being aware of the aspect ratio.</p>"},{"location":"examples/02_model_components.html","title":"02_model_components","text":"In\u00a0[1]: Copied! <pre>import torch\nimport pandas as pd\nimport numpy as np\n\nfrom torch import nn\n</pre> import torch import pandas as pd import numpy as np  from torch import nn In\u00a0[2]: Copied! <pre>df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]})\ndf.head()\n</pre> df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]}) df.head() Out[2]: color size 0 r s 1 b n 2 g l <p>one hot encoded, the first observation would be</p> In\u00a0[3]: Copied! <pre>obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\")\n</pre> obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\") <p>if we simply numerically encode (label encode or <code>le</code>) the values:</p> In\u00a0[4]: Copied! <pre>obs_0_le = (np.array([0, 3])).astype(\"int64\")\n</pre> obs_0_le = (np.array([0, 3])).astype(\"int64\") <p>Note that in the functioning implementation of the package we start from 1, saving 0 for padding, i.e. unseen values.</p> <p>Now, let's see if the two implementations are equivalent</p> In\u00a0[5]: Copied! <pre># we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1\nlin = nn.Linear(6, 1)\n</pre> # we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1 lin = nn.Linear(6, 1) In\u00a0[6]: Copied! <pre>emb = nn.Embedding(6, 1)\nemb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight))\n</pre> emb = nn.Embedding(6, 1) emb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight)) In\u00a0[7]: Copied! <pre>lin(torch.tensor(obs_0_oh))\n</pre> lin(torch.tensor(obs_0_oh)) Out[7]: <pre>tensor([-0.5181], grad_fn=&lt;ViewBackward0&gt;)</pre> In\u00a0[8]: Copied! <pre>emb(torch.tensor(obs_0_le)).sum() + lin.bias\n</pre> emb(torch.tensor(obs_0_le)).sum() + lin.bias Out[8]: <pre>tensor([-0.5181], grad_fn=&lt;AddBackward0&gt;)</pre> <p>And this is precisely how the linear model <code>Wide</code> is implemented</p> In\u00a0[9]: Copied! <pre>from pytorch_widedeep.models import Wide\n</pre> from pytorch_widedeep.models import Wide <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[10]: Copied! <pre># ?Wide\n</pre> # ?Wide In\u00a0[11]: Copied! <pre>wide = Wide(input_dim=10, pred_dim=1)\nwide\n</pre> wide = Wide(input_dim=10, pred_dim=1) wide Out[11]: <pre>Wide(\n  (wide_linear): Embedding(11, 1, padding_idx=0)\n)</pre> <p>Note that even though the input dim is 10, the Embedding layer has 11 weights. Again, this is because we save <code>0</code> for padding, which is used for unseen values during the encoding process.</p> <p>As I mentioned, <code>deeptabular</code> has enough complexity on its own and it will be described in a separated notebook. Let's then jump to <code>deeptext</code>.</p> In\u00a0[12]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp In\u00a0[13]: Copied! <pre>data = {\n    \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),\n    \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),\n    \"cont1\": np.random.rand(20),\n    \"cont2\": np.random.rand(20),\n}\n\ndf = pd.DataFrame(data)\n</pre> data = {     \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),     \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),     \"cont1\": np.random.rand(20),     \"cont2\": np.random.rand(20), }  df = pd.DataFrame(data) In\u00a0[14]: Copied! <pre>df.head()\n</pre> df.head() Out[14]: cat1 cat2 cont1 cont2 0 A Y 0.789347 0.561789 1 C X 0.050822 0.061538 2 A Y 0.863784 0.241967 3 C X 0.917848 0.644658 4 C Y 0.042328 0.417303 In\u00a0[15]: Copied! <pre># see the docs for details on all params/options\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"cat1\", \"cat2\"],\n    continuous_cols=[\"cont1\", \"cont2\"],\n    embedding_rule=\"fastai\",\n)\n</pre> # see the docs for details on all params/options tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"cat1\", \"cat2\"],     continuous_cols=[\"cont1\", \"cont2\"],     embedding_rule=\"fastai\", ) In\u00a0[16]: Copied! <pre>X_tab = tab_preprocessor.fit_transform(df)\n</pre> X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre># toy example just to build a model.\ntabmlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=4,\n    mlp_hidden_dims=[8, 4],\n    mlp_linear_first=True,\n)\ntabmlp\n</pre> # toy example just to build a model. tabmlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=4,     mlp_hidden_dims=[8, 4],     mlp_linear_first=True, ) tabmlp Out[17]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_cat1): Embedding(4, 3, padding_idx=0)\n      (emb_layer_cat2): Embedding(3, 2, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=4, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=13, out_features=8, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=8, out_features=4, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Lets describe a bit the model: first we have what we call a <code>DiffSizeCatEmbeddings</code>, where categorical columns with different number of unique categories will be encoded with embeddings of different dimensions. Then the continuous columns will not be normalised (the normalised layer is just the identity) and they will be embedded via a \"standard\" method, using a so-called <code>ContLinear</code> layer. This layer displays some <code>INFO</code> that tells us what it is (<code>ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]</code>). There are two other options available to embed the continuous cols based on the paper On Embeddings for Numerical Features in Tabular Deep Learning. These are <code>PieceWise</code> and <code>Periodic</code> and all available via the <code>embed_continuous_method</code> param, which can adopt values <code>\"standard\", \"piecewise\"</code> and <code>\"periodic\"</code>. The embedded categorical and continuous columns will be then concatenated ($3 + 2 + (4 * 2) = 13$ input dims) and passed to an MLP.</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import BasicRNN\n</pre> from pytorch_widedeep.models import BasicRNN In\u00a0[19]: Copied! <pre>basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4)\n</pre> basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/torch/nn/modules/rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n  warnings.warn(\"dropout option adds dropout after all but last \"\n</pre> In\u00a0[20]: Copied! <pre>basic_rnn\n</pre> basic_rnn Out[20]: <pre>BasicRNN(\n  (word_embed): Embedding(4, 4, padding_idx=0)\n  (rnn): LSTM(4, 4, batch_first=True, dropout=0.1)\n  (rnn_mlp): Identity()\n)</pre> <p>You could, if you wanted, add a Fully Connected Head (FC-Head) on top of it</p> In\u00a0[21]: Copied! <pre>from pytorch_widedeep.models import Vision\n</pre> from pytorch_widedeep.models import Vision In\u00a0[22]: Copied! <pre>resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0)\n</pre> resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0) In\u00a0[23]: Copied! <pre>resnet\n</pre> resnet Out[23]: <pre>Vision(\n  (features): Sequential(\n    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (2): ReLU(inplace=True)\n    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n    (4): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (5): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (6): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (7): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (8): AdaptiveAvgPool2d(output_size=(1, 1))\n  )\n)</pre>"},{"location":"examples/02_model_components.html#model-components","title":"Model Components\u00b6","text":"<p>The main components of a <code>WideDeep</code> (i.e. Multimodal) model are tabular data, text and images, which are feed into the model via so called <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> model components</p>"},{"location":"examples/02_model_components.html#1-wide","title":"1. <code>wide</code>\u00b6","text":"<p>The <code>wide</code> component is a Linear layer \"plugged\" into the output neuron(s). Here, the non-linearities are captured via crossed columns. Crossed columns are, quoting directly the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>The only particularity of our implementation is that we have implemented the linear layer via an Embedding layer plus a bias. While the implementations are equivalent, the latter is faster and far more memory efficient, since we do not need to one hot encode the categorical features.</p> <p>Let's assume we the following dataset:</p>"},{"location":"examples/02_model_components.html#2-deeptabular","title":"2. <code>deeptabular</code>\u00b6","text":"<p>The <code>deeptabular</code> model alone is what normally would be referred as Deep Learning for tabular data. As mentioned a number of times throughout the library, each component can be used independently. Therefore, if you wanted to use any of the models below alone, it is perfectly possible. There are just a couple of simple requirement that will be covered in a later notebook.</p> <p>By the time of writing, there are a number of models available in <code>pytorch-widedeep</code> to do DL for tabular data. These are:</p> <ol> <li><code>TabMlp</code></li> <li><code>ContextAttentionMLP</code></li> <li><code>SelfAttentionMLP</code></li> <li><code>TabResnet</code></li> <li><code>Tabnet</code></li> <li><code>TabTransformer</code></li> <li><code>FT-Tabransformer</code></li> <li><code>SAINT</code></li> <li><code>TabFastFormer</code></li> <li><code>TabPerceiver</code></li> </ol> <p>Let's have a look to one of them. For more information on each of these models, please, have a look to the documentation</p>"},{"location":"examples/02_model_components.html#3-deeptext","title":"3. <code>deeptext</code>\u00b6","text":"<p>At the time of writing, <code>pytorch-widedeep</code> offers three models that can be passed to <code>WideDeep</code> as the <code>deeptext</code> component. These are:</p> <ol> <li>BasicRNN</li> <li>AttentiveRNN</li> <li>StackedAttentiveRNN</li> </ol> <p>For details on each of these models, please, have a look to the documentation of the package.</p> <p>We will soon integrate with Hugginface, but let me insist. It is perfectly possible to use custom models for each component, please, have a look to the corresponding notebook. In general, simply, build them and pass them as the corresponding parameters. Note that the custom models MUST return a last layer of activations (i.e. not the final prediction) so that  these activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with the size of these last layers of activations.</p> <p>Let's have a look to the <code>BasicRNN</code> model</p>"},{"location":"examples/02_model_components.html#4-deepimage","title":"4. <code>deepimage</code>\u00b6","text":"<p>At the time of writing <code>pytorch-widedeep</code> is integrated with torchvision via the <code>Vision</code> class. This means that the it is possible to use a variant of the following architectures:</p> <ol> <li>resnet</li> <li>shufflenet</li> <li>resnext</li> <li>wide_resnet</li> <li>regnet</li> <li>densenet</li> <li>mobilenet</li> <li>mnasnet</li> <li>efficientnet</li> <li>squeezenet</li> </ol> <p>The user can choose which layers will be trainable. Alternatively, in none of these architectures is useful, one could use a simple, fully trained CNN (please see the package documentation) or pass a custom model.</p> <p>let's have a look</p>"},{"location":"examples/03_binary_classification_with_defaults.html","title":"03_binary_classification_with_defaults","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy, Precision\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import Wide, TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy, Precision from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\n</pre> df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True) In\u00a0[5]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre># TARGET\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # TARGET target_col = \"income_label\" target = df[target_col].values <p>let's see what the preprocessors do</p> In\u00a0[8]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[9]: Copied! <pre># # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary\n# wide_preprocessor.encoding_dict\n</pre> # # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary # wide_preprocessor.encoding_dict In\u00a0[10]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) In\u00a0[11]: Copied! <pre># check the docs to understand the useful attributes that the tab_preprocessor has. For example,\n# as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input\n# that specifies the categortical columns that will be represented as embeddings, the number\n# of different categories per feature, and the dimension of the embeddings as defined by some\n# of the internal rules of thumb that the preprocessor has (have a look to the docs)\ntab_preprocessor.cat_embed_input\n</pre> # check the docs to understand the useful attributes that the tab_preprocessor has. For example, # as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input # that specifies the categortical columns that will be represented as embeddings, the number # of different categories per feature, and the dimension of the embeddings as defined by some # of the internal rules of thumb that the preprocessor has (have a look to the docs) tab_preprocessor.cat_embed_input Out[11]: <pre>[('workclass', 9, 5),\n ('education', 16, 8),\n ('marital_status', 7, 5),\n ('occupation', 15, 7),\n ('relationship', 6, 4),\n ('race', 5, 4),\n ('gender', 2, 2),\n ('capital_gain', 123, 24),\n ('capital_loss', 99, 21),\n ('native_country', 42, 13)]</pre> In\u00a0[12]: Copied! <pre>print(X_wide)\nprint(X_wide.shape)\n</pre> print(X_wide) print(X_wide.shape) <pre>[[  1  10  26 ...  61 103 328]\n [  1  11  27 ...  61 104 329]\n [  2  12  27 ...  61 105 330]\n ...\n [  1  11  28 ...  61 115 335]\n [  1  11  26 ...  61 115 335]\n [  7  11  27 ...  61 127 336]]\n(48842, 10)\n</pre> In\u00a0[13]: Copied! <pre>print(X_tab)\nprint(X_tab.shape)\n</pre> print(X_tab) print(X_tab.shape) <pre>[[ 1.          1.          1.         ...  1.         -0.99512893\n  -0.03408696]\n [ 1.          2.          2.         ...  1.         -0.04694151\n   0.77292975]\n [ 2.          3.          2.         ...  1.         -0.77631645\n  -0.03408696]\n ...\n [ 1.          2.          3.         ...  1.          1.41180837\n  -0.03408696]\n [ 1.          2.          1.         ...  1.         -1.21394141\n  -1.64812038]\n [ 7.          2.          2.         ...  1.          0.97418341\n  -0.03408696]]\n(48842, 12)\n</pre> In\u00a0[14]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) <p>Let's first find out how a linear model performs</p> In\u00a0[15]: Copied! <pre>wide\n</pre> wide Out[15]: <pre>Wide(\n  (wide_linear): Embedding(809, 1, padding_idx=0)\n)</pre> <p>Before being passed to the Trainer, the models need to be \"constructed\" with the <code>WideDeep</code> constructor class. For the particular case of the wide/linear model, not much really happens</p> In\u00a0[16]: Copied! <pre>lin_model = WideDeep(wide=wide)\n</pre> lin_model = WideDeep(wide=wide) In\u00a0[17]: Copied! <pre>lin_model\n</pre> lin_model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n)</pre> In\u00a0[18]: Copied! <pre>lin_trainer = Trainer(\n    model=lin_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),\n    metrics=[Accuracy, Precision],\n)\n</pre> lin_trainer = Trainer(     model=lin_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),     metrics=[Accuracy, Precision], ) In\u00a0[19]: Copied! <pre>lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 109.04it/s, loss=0.426, metrics={'acc': 0.7983, 'prec': 0.6152}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 102.46it/s, loss=0.366, metrics={'acc': 0.832, 'prec': 0.6916}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.27it/s, loss=0.364, metrics={'acc': 0.8305, 'prec': 0.6933}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 150.46it/s, loss=0.361, metrics={'acc': 0.8357, 'prec': 0.6982}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 133.19it/s, loss=0.359, metrics={'acc': 0.8329, 'prec': 0.6994}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 145.75it/s, loss=0.361, metrics={'acc': 0.836, 'prec': 0.7009}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.91it/s, loss=0.358, metrics={'acc': 0.8333, 'prec': 0.7005}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 155.08it/s, loss=0.361, metrics={'acc': 0.8364, 'prec': 0.702}]\n</pre> <p>Bear in mind that <code>wide</code> is a linear model where the non-linearities are captured via the crossed columns. For the crossed-columns to be effective one needs proper business knowledge. There is no magic formula to produce them</p> <p>Let's have a look to the tabular model by itself</p> In\u00a0[20]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_model = WideDeep(deeptabular=tab_mlp) In\u00a0[21]: Copied! <pre>tab_model\n</pre> tab_model Out[21]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=95, out_features=400, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=400, out_features=200, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>You can see how the <code>WideDeep</code> class has added a final prediction layer that collects the activations from the last layer of the model and plugs them into the output neuron. If this was a multiclass classification problem, the prediction dimension (i.e. the size of that final layer) needs to be specified via the <code>pred_dim</code> when instantiating the <code>WideDeep</code> class, as we will see later</p> In\u00a0[22]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[23]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 97.00it/s, loss=0.37, metrics={'acc': 0.8267, 'prec': 0.7037}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 134.91it/s, loss=0.313, metrics={'acc': 0.8588, 'prec': 0.7577}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 86.86it/s, loss=0.319, metrics={'acc': 0.8514, 'prec': 0.761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 73.13it/s, loss=0.296, metrics={'acc': 0.8675, 'prec': 0.7685}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 79.07it/s, loss=0.305, metrics={'acc': 0.8574, 'prec': 0.7646}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 130.11it/s, loss=0.289, metrics={'acc': 0.8696, 'prec': 0.7765}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 87.39it/s, loss=0.296, metrics={'acc': 0.8622, 'prec': 0.7769}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 90.63it/s, loss=0.285, metrics={'acc': 0.8697, 'prec': 0.7741}]\n</pre> <p>The best result I ever obtained with <code>LightGBM</code> on this dataset is 0.8782...so we are pretty close.</p> <p>Let's combine the <code>wide</code> and <code>tab_mlp</code> components see if it helps</p> In\u00a0[24]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\nwd_model = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) wd_model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[25]: Copied! <pre>wd_trainer = Trainer(\n    model=wd_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> wd_trainer = Trainer(     model=wd_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[26]: Copied! <pre>wd_trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2\n)\n</pre> wd_trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 77.48it/s, loss=0.418, metrics={'acc': 0.8047, 'prec': 0.6154}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 110.51it/s, loss=0.321, metrics={'acc': 0.8521, 'prec': 0.7059}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 82.70it/s, loss=0.333, metrics={'acc': 0.8428, 'prec': 0.7141}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 112.52it/s, loss=0.299, metrics={'acc': 0.866, 'prec': 0.7447}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 74.34it/s, loss=0.312, metrics={'acc': 0.8533, 'prec': 0.7404}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 89.86it/s, loss=0.29, metrics={'acc': 0.8683, 'prec': 0.7496}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 65.32it/s, loss=0.301, metrics={'acc': 0.8591, 'prec': 0.7542}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 86.81it/s, loss=0.286, metrics={'acc': 0.8712, 'prec': 0.7552}]\n</pre> <p>For this particular case, the combination of both did not lead to better results that using just the tab_mlp model, when using only 4 epochs.</p> <p>Note that we have use a <code>TabMlp</code> model, but we could use any other model in the library using the same syntax</p> In\u00a0[27]: Copied! <pre>from pytorch_widedeep.models import TabTransformer\n</pre> from pytorch_widedeep.models import TabTransformer <p>The parameters for the <code>TabTransformer</code> are this</p> <pre><code>column_idx: Dict[str, int],\ncat_embed_input: Optional[List[Tuple[str, int]]] = None,\ncat_embed_dropout: Optional[float] = None,\nuse_cat_bias: Optional[bool] = None,\ncat_embed_activation: Optional[str] = None,\nshared_embed: Optional[bool] = None,\nadd_shared_embed: Optional[bool] = None,\nfrac_shared_embed: Optional[float] = None,\ncontinuous_cols: Optional[List[str]] = None,\ncont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\nembed_continuous: Optional[bool] = None,\nembed_continuous_method: Optional[Literal[\"standard\", \"piecewise\", \"periodic\"]] = None,\ncont_embed_dropout: Optional[float] = None,\ncont_embed_activation: Optional[str] = None,\nquantization_setup: Optional[Dict[str, List[float]]] = None,\nn_frequencies: Optional[int] = None,\nsigma: Optional[float] = None,\nshare_last_layer: Optional[bool] = None,\nfull_embed_dropout: Optional[bool] = None,\ninput_dim: int = 32,\nn_heads: int = 8,\nuse_qkv_bias: bool = False,\nn_blocks: int = 4,\nattn_dropout: float = 0.2,\nff_dropout: float = 0.1,\nff_factor: int = 4,\ntransformer_activation: str = \"gelu\",\nuse_linear_attention: bool = False,\nuse_flash_attention: bool = False,\nmlp_hidden_dims: Optional[List[int]] = None,\nmlp_activation: str = \"relu\",\nmlp_dropout: float = 0.1,\nmlp_batchnorm: bool = False,\nmlp_batchnorm_last: bool = False,\nmlp_linear_first: bool = True,\n</code></pre> <p>Please, see the documentation for details on each one of them, for now let's see how one could use a <code>TabTransformer</code> model in a few lines of code</p> In\u00a0[28]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_norm_layer=\"layernorm\",\n    cont_embed_dropout=0.2,\n    cont_embed_activation=\"leaky_relu\",\n    n_heads=4,\n    ff_dropout=0.2,\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n    mlp_linear_first=\"True\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_norm_layer=\"layernorm\",     cont_embed_dropout=0.2,     cont_embed_activation=\"leaky_relu\",     n_heads=4,     ff_dropout=0.2,     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\",     mlp_linear_first=\"True\", ) In\u00a0[29]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_transformer)\n</pre> tab_model = WideDeep(deeptabular=tab_transformer) In\u00a0[30]: Copied! <pre>tab_model\n</pre> tab_model Out[30]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabTransformer(\n      (cat_embed): SameSizeCatEmbeddings(\n        (embed): Embedding(325, 32, padding_idx=0)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.2)\n        (activation_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dropout): Dropout(p=0.2, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block2): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block3): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=384, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[31]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[32]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:11&lt;00:00, 27.57it/s, loss=0.359, metrics={'acc': 0.8334, 'prec': 0.7082}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 57.89it/s, loss=0.33, metrics={'acc': 0.8536, 'prec': 0.7152}]\n</pre>"},{"location":"examples/03_binary_classification_with_defaults.html#simple-binary-classification-with-defaults","title":"Simple Binary Classification with defaults\u00b6","text":"<p>In this notebook we will train a Wide and Deep model and simply a \"Deep\" model using the well known adult dataset</p>"},{"location":"examples/03_binary_classification_with_defaults.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/03_binary_classification_with_defaults.html#defining-the-model","title":"Defining the model\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html","title":"04_regression_with_images_and_text","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\nfrom torchvision.transforms import ToTensor, Normalize\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\n</pre> import numpy as np import pandas as pd import os import torch from torchvision.transforms import ToTensor, Normalize  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n</pre> text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df) <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\n</pre> In\u00a0[8]: Copied! <pre>image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>Reading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 638.00it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[9]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# DeepText: a stack of 2 LSTMs\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_matrix=text_preprocessor.embedding_matrix,\n    n_layers=2,\n    hidden_dim=64,\n    rnn_dropout=0.5,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # DeepText: a stack of 2 LSTMs basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_matrix=text_preprocessor.embedding_matrix,     n_layers=2,     hidden_dim=64,     rnn_dropout=0.5, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4) <p>Combine them all with the \"collector\" class <code>WideDeep</code></p> In\u00a0[10]: Copied! <pre>model = WideDeep(\n    wide=wide,\n    deeptabular=tab_mlp,\n    deeptext=basic_rnn,\n    deepimage=resnet,\n    head_hidden_dims=[256, 128],\n)\n</pre> model = WideDeep(     wide=wide,     deeptabular=tab_mlp,     deeptext=basic_rnn,     deepimage=resnet,     head_hidden_dims=[256, 128], ) In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"rmse\")\n</pre> trainer = Trainer(model, objective=\"rmse\") In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.28it/s, loss=115]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=94.1]\n</pre> <p>Both, the Text and Image components allow FC-heads on their own (have a look to the documentation).</p> <p>Now let's go \"kaggle crazy\". Let's use different optimizers, initializers and schedulers for different components. Moreover, let's use a different learning rate for different parameter groups, for the <code>deeptabular</code> component</p> In\u00a0[13]: Copied! <pre>deep_params = []\nfor childname, child in model.named_children():\n    if childname == \"deeptabular\":\n        for n, p in child.named_parameters():\n            if \"embed_layer\" in n:\n                deep_params.append({\"params\": p, \"lr\": 1e-4})\n            else:\n                deep_params.append({\"params\": p, \"lr\": 1e-3})\n</pre> deep_params = [] for childname, child in model.named_children():     if childname == \"deeptabular\":         for n, p in child.named_parameters():             if \"embed_layer\" in n:                 deep_params.append({\"params\": p, \"lr\": 1e-4})             else:                 deep_params.append({\"params\": p, \"lr\": 1e-3}) In\u00a0[14]: Copied! <pre>wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03)\ndeep_opt = torch.optim.Adam(deep_params)\ntext_opt = torch.optim.AdamW(model.deeptext.parameters())\nimg_opt = torch.optim.AdamW(model.deepimage.parameters())\nhead_opt = torch.optim.Adam(model.deephead.parameters())\n</pre> wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03) deep_opt = torch.optim.Adam(deep_params) text_opt = torch.optim.AdamW(model.deeptext.parameters()) img_opt = torch.optim.AdamW(model.deepimage.parameters()) head_opt = torch.optim.Adam(model.deephead.parameters()) In\u00a0[15]: Copied! <pre>wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\ndeep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\ntext_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\nimg_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\nhead_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5)\n</pre> wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5) deep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5) img_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) head_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5) In\u00a0[16]: Copied! <pre># remember, one optimizer per model components, for lr_schedures and initializers is not neccesary\noptimizers = {\n    \"wide\": wide_opt,\n    \"deeptabular\": deep_opt,\n    \"deeptext\": text_opt,\n    \"deepimage\": img_opt,\n    \"deephead\": head_opt,\n}\nschedulers = {\n    \"wide\": wide_sch,\n    \"deeptabular\": deep_sch,\n    \"deeptext\": text_sch,\n    \"deepimage\": img_sch,\n    \"deephead\": head_sch,\n}\n\n# Now...we have used pretrained word embeddings, so you do not want to\n# initialise these  embeddings. However you might still want to initialise the\n# other layers in the DeepText component. No probs, you can do that with the\n# parameter pattern and your knowledge on regular  expressions. Here we are\n# telling to the KaimingNormal initializer to NOT touch the  parameters whose\n# name contains the string word_embed.\ninitializers = {\n    \"wide\": KaimingNormal,\n    \"deeptabular\": KaimingNormal,\n    \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),\n    \"deepimage\": KaimingNormal,\n}\n\nmean = [0.406, 0.456, 0.485]  # BGR\nstd = [0.225, 0.224, 0.229]  # BGR\ntransforms = [ToTensor, Normalize(mean=mean, std=std)]\ncallbacks = [\n    LRHistory(n_epochs=10),\n    EarlyStopping,\n    ModelCheckpoint(filepath=\"model_weights/wd_out\"),\n]\n</pre> # remember, one optimizer per model components, for lr_schedures and initializers is not neccesary optimizers = {     \"wide\": wide_opt,     \"deeptabular\": deep_opt,     \"deeptext\": text_opt,     \"deepimage\": img_opt,     \"deephead\": head_opt, } schedulers = {     \"wide\": wide_sch,     \"deeptabular\": deep_sch,     \"deeptext\": text_sch,     \"deepimage\": img_sch,     \"deephead\": head_sch, }  # Now...we have used pretrained word embeddings, so you do not want to # initialise these  embeddings. However you might still want to initialise the # other layers in the DeepText component. No probs, you can do that with the # parameter pattern and your knowledge on regular  expressions. Here we are # telling to the KaimingNormal initializer to NOT touch the  parameters whose # name contains the string word_embed. initializers = {     \"wide\": KaimingNormal,     \"deeptabular\": KaimingNormal,     \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),     \"deepimage\": KaimingNormal, }  mean = [0.406, 0.456, 0.485]  # BGR std = [0.225, 0.224, 0.229]  # BGR transforms = [ToTensor, Normalize(mean=mean, std=std)] callbacks = [     LRHistory(n_epochs=10),     EarlyStopping,     ModelCheckpoint(filepath=\"model_weights/wd_out\"), ] In\u00a0[17]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"rmse\",\n    initializers=initializers,\n    optimizers=optimizers,\n    lr_schedulers=schedulers,\n    callbacks=callbacks,\n    transforms=transforms,\n)\n</pre> trainer = Trainer(     model,     objective=\"rmse\",     initializers=initializers,     optimizers=optimizers,     lr_schedulers=schedulers,     callbacks=callbacks,     transforms=transforms, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/initializers.py:34: UserWarning: No initializer found for deephead\n  warnings.warn(\n</pre> In\u00a0[18]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.25it/s, loss=101]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=90.6]\n</pre> <pre>Model weights after training corresponds to the those of the final epoch which might not be the best performing weights. Use the 'ModelCheckpoint' Callback to restore the best epoch weights.\n</pre> <p>we have only run one epoch, but let's check that the LRHistory callback records the lr values for each group</p> In\u00a0[19]: Copied! <pre>trainer.lr_history\n</pre> trainer.lr_history Out[19]: <pre>{'lr_wide_0': [0.03, 0.03],\n 'lr_deeptabular_0': [0.0001, 0.0001],\n 'lr_deeptabular_1': [0.0001, 0.0001],\n 'lr_deeptabular_2': [0.0001, 0.0001],\n 'lr_deeptabular_3': [0.0001, 0.0001],\n 'lr_deeptabular_4': [0.0001, 0.0001],\n 'lr_deeptabular_5': [0.0001, 0.0001],\n 'lr_deeptabular_6': [0.0001, 0.0001],\n 'lr_deeptabular_7': [0.0001, 0.0001],\n 'lr_deeptabular_8': [0.0001, 0.0001],\n 'lr_deeptabular_9': [0.001, 0.001],\n 'lr_deeptabular_10': [0.001, 0.001],\n 'lr_deeptabular_11': [0.001, 0.001],\n 'lr_deeptabular_12': [0.001, 0.001],\n 'lr_deeptext_0': [0.001, 0.001],\n 'lr_deepimage_0': [0.001, 0.001],\n 'lr_deephead_0': [0.001, 0.001]}</pre>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-images-and-text","title":"Regression with Images and Text\u00b6","text":"<p>In this notebook we will go through a series of examples on how to combine all Wide &amp; Deep components.</p> <p>To that aim I will use the Airbnb listings dataset for London, which you can download from here. I use this dataset simply because it contains tabular data, images and text.</p> <p>I have taken a sample of 1000 listings to keep the data tractable in this notebook. Also, I have preprocessed the data and prepared it for this exercise. All preprocessing steps can be found in the notebook <code>airbnb_data_preprocessing.ipynb</code> in this <code>examples</code> folder.</p>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-the-defaults","title":"Regression with the defaults\u00b6","text":"<p>The set up</p>"},{"location":"examples/04_regression_with_images_and_text.html#prepare-the-data","title":"Prepare the data\u00b6","text":"<p>I will focus here on how to prepare the data and run the model. Check notebooks 1 and 2 to see what's going on behind the scences</p> <p>Preparing the data is rather simple</p>"},{"location":"examples/04_regression_with_images_and_text.html#build-the-model-components","title":"Build the model components\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html#build-the-trainer-and-fit","title":"Build the trainer and fit\u00b6","text":""},{"location":"examples/05_save_and_load_model_and_artifacts.html","title":"05_save_and_load_model_and_artifacts","text":"In\u00a0[1]: Copied! <pre>import pickle\nimport numpy as np\nimport pandas as pd\nimport torch\nimport shutil\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom sklearn.model_selection import train_test_split\n</pre> import pickle import numpy as np import pandas as pd import torch import shutil  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from sklearn.model_selection import train_test_split <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>train, valid = train_test_split(df, test_size=0.2, stratify=df.target)\n# the test data will be used lately as if it was \"fresh\", new data coming after some time...\nvalid, test = train_test_split(valid, test_size=0.5, stratify=valid.target)\n</pre> train, valid = train_test_split(df, test_size=0.2, stratify=df.target) # the test data will be used lately as if it was \"fresh\", new data coming after some time... valid, test = train_test_split(valid, test_size=0.5, stratify=valid.target) In\u00a0[5]: Copied! <pre>print(f\"train shape: {train.shape}\")\nprint(f\"valid shape: {valid.shape}\")\nprint(f\"test shape: {test.shape}\")\n</pre> print(f\"train shape: {train.shape}\") print(f\"valid shape: {valid.shape}\") print(f\"test shape: {test.shape}\") <pre>train shape: (39073, 15)\nvalid shape: (4884, 15)\ntest shape: (4885, 15)\n</pre> In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab_train = tab_preprocessor.fit_transform(train)\ny_train = train.target.values\nX_tab_valid = tab_preprocessor.transform(valid)\ny_valid = valid.target.values\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab_train = tab_preprocessor.fit_transform(train) y_train = train.target.values X_tab_valid = tab_preprocessor.transform(valid) y_valid = valid.target.values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[8]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[9]: Copied! <pre>model\n</pre> model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(122, 23, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(97, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=8, embed_dropout=0.0)\n        (dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=108, out_features=64, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre>early_stopping = EarlyStopping()\nmodel_checkpoint = ModelCheckpoint(\n    filepath=\"tmp_dir/adult_tabmlp_model\",\n    save_best_only=True,\n    verbose=1,\n    max_save=1,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    callbacks=[early_stopping, model_checkpoint],\n    metrics=[Accuracy],\n)\n\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=4,\n    batch_size=256,\n)\n</pre> early_stopping = EarlyStopping() model_checkpoint = ModelCheckpoint(     filepath=\"tmp_dir/adult_tabmlp_model\",     save_best_only=True,     verbose=1,     max_save=1, )  trainer = Trainer(     model,     objective=\"binary\",     callbacks=[early_stopping, model_checkpoint],     metrics=[Accuracy], )  trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=4,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 76.25it/s, loss=0.452, metrics={'acc': 0.7867}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 125.36it/s, loss=0.335, metrics={'acc': 0.8532}]\n</pre> <pre>\nEpoch 1: val_loss improved from inf to 0.33532 Saving model to tmp_dir/adult_tabmlp_model_1.p\n</pre> <pre>epoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 76.98it/s, loss=0.355, metrics={'acc': 0.8401}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 106.51it/s, loss=0.303, metrics={'acc': 0.8665}]\n</pre> <pre>\nEpoch 2: val_loss improved from 0.33532 to 0.30273 Saving model to tmp_dir/adult_tabmlp_model_2.p\n</pre> <pre>epoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.71it/s, loss=0.332, metrics={'acc': 0.849}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 107.80it/s, loss=0.288, metrics={'acc': 0.8757}]\n</pre> <pre>\nEpoch 3: val_loss improved from 0.30273 to 0.28791 Saving model to tmp_dir/adult_tabmlp_model_3.p\n</pre> <pre>epoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 79.02it/s, loss=0.32, metrics={'acc': 0.8541}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 127.07it/s, loss=0.282, metrics={'acc': 0.8763}]</pre> <pre>\nEpoch 4: val_loss improved from 0.28791 to 0.28238 Saving model to tmp_dir/adult_tabmlp_model_4.p\nModel weights restored to best epoch: 4\n</pre> <pre>\n</pre> In\u00a0[11]: Copied! <pre>torch.save(model, \"tmp_dir/model_saved_option_1.pt\")\n</pre> torch.save(model, \"tmp_dir/model_saved_option_1.pt\") In\u00a0[12]: Copied! <pre>torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\")\n</pre> torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\") In\u00a0[13]: Copied! <pre>trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\")\n</pre> trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\") <p>or the state dict</p> In\u00a0[14]: Copied! <pre>trainer.save(\n    path=\"tmp_dir/\",\n    model_filename=\"model_state_dict_saved_option_2.pt\",\n    save_state_dict=True,\n)\n</pre> trainer.save(     path=\"tmp_dir/\",     model_filename=\"model_state_dict_saved_option_2.pt\",     save_state_dict=True, ) In\u00a0[15]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\n</pre> In\u00a0[16]: Copied! <pre>%%bash\n\nls tmp_dir/history/\n</pre> %%bash  ls tmp_dir/history/ <pre>train_eval_history.json\n</pre> <p>Note that since we have used the <code>ModelCheckpoint</code> Callback, <code>adult_tabmlp_model_2.p</code> is the model state dict of the model at epoch 2, i.e. same as <code>model_state_dict_saved_option_1.p</code> or <code>model_state_dict_saved_option_2.p</code>.</p> In\u00a0[17]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:\n    pickle.dump(tab_preprocessor, dp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:     pickle.dump(tab_preprocessor, dp) In\u00a0[18]: Copied! <pre>with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:\n    pickle.dump(early_stopping, es)\n</pre> with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:     pickle.dump(early_stopping, es) In\u00a0[19]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\neary_stop.pkl\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\ntab_preproc.pkl\n</pre> <p>And that is pretty much all you need to resume training or directly predict, let's see</p> In\u00a0[20]: Copied! <pre>test.head()\n</pre> test.head() Out[20]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 10103 43 Private 198282 HS-grad 9 Married-civ-spouse Craft-repair Husband White Male 0 0 40 United-States 1 31799 20 Private 228686 11th 7 Married-civ-spouse Other-service Husband White Male 0 0 40 United-States 0 19971 26 Private 291968 HS-grad 9 Married-civ-spouse Transport-moving Husband White Male 0 0 44 United-States 0 3039 48 Private 175958 Bachelors 13 Divorced Prof-specialty Not-in-family White Male 0 0 30 United-States 0 20725 18 Private 232024 11th 7 Never-married Machine-op-inspct Own-child White Male 0 0 55 United-States 0 In\u00a0[21]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:\n    tab_preprocessor_new = pickle.load(tp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:     tab_preprocessor_new = pickle.load(tp) In\u00a0[22]: Copied! <pre>X_test_tab = tab_preprocessor_new.transform(test)\ny_test = test.target\n</pre> X_test_tab = tab_preprocessor_new.transform(test) y_test = test.target In\u00a0[23]: Copied! <pre>tab_mlp_new = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nnew_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp_new = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) new_model = WideDeep(deeptabular=tab_mlp) In\u00a0[24]: Copied! <pre>new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\"))\n</pre> new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\")) Out[24]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[25]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\", ) In\u00a0[26]: Copied! <pre>preds = trainer.predict(X_tab=X_test_tab, batch_size=32)\n</pre> preds = trainer.predict(X_tab=X_test_tab, batch_size=32) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:00&lt;00:00, 309.83it/s]\n</pre> In\u00a0[27]: Copied! <pre>from sklearn.metrics import accuracy_score\n</pre> from sklearn.metrics import accuracy_score In\u00a0[28]: Copied! <pre>accuracy_score(y_test, preds)\n</pre> accuracy_score(y_test, preds) Out[28]: <pre>0.8595701125895598</pre> In\u00a0[29]: Copied! <pre>shutil.rmtree(\"tmp_dir/\")\n</pre> shutil.rmtree(\"tmp_dir/\")"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-and-load-model-and-artifacts","title":"Save and load model and artifacts\u00b6","text":"<p>In this notebook I will show the different options to save and load a model, as well as some additional objects produced during training.</p> <p>On a given day, you train a model...</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-1","title":"Save model: option 1\u00b6","text":"<p>save (and load) a model as you woud do with any other torch model</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-2","title":"Save model: option 2\u00b6","text":"<p>use the <code>trainer</code>. The <code>trainer</code> will also save the training history and the learning rate history (if learning rate schedulers are used)</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-preprocessors-and-callbacks","title":"Save preprocessors and callbacks\u00b6","text":"<p>...just pickle them</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#run-new-experiment-prepare-new-dataset-load-model-and-predict","title":"Run New experiment: prepare new dataset, load model, and predict\u00b6","text":""},{"location":"examples/06_finetune_and_warmup.html","title":"06_finetune_and_warmup","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\n# For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[3]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[4]: Copied! <pre># TARGET\ntarget = df[target_col].values\n\n# WIDE\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# DEEP\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # TARGET target = df[target_col].values  # WIDE wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # DEEP tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[5]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[6]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[7]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 74.26it/s, loss=0.399, metrics={'acc': 0.8163}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 91.03it/s, loss=0.296, metrics={'acc': 0.8677}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 81.31it/s, loss=0.3, metrics={'acc': 0.8614}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 106.45it/s, loss=0.285, metrics={'acc': 0.8721}]\n</pre> In\u00a0[8]: Copied! <pre>trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\")\n</pre> trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\") <p>Now time goes by...and we want to fine-tune the model to another, new dataset (for example, a dataset that is identical to the one you used to train the previous model but for another country).</p> <p>Here I will use the same dataset just for illustration purposes, but the flow would be identical to that new dataset</p> In\u00a0[9]: Copied! <pre>wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp_1 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_1 = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp_1 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_1 = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[10]: Copied! <pre>model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\"))\n</pre> model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\")) Out[10]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[11]: Copied! <pre>trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy]) In\u00a0[12]: Copied! <pre>trainer_1.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=2,\n    batch_size=256,\n    finetune=True,\n    finetune_epochs=2,\n)\n</pre> trainer_1.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     n_epochs=2,     batch_size=256,     finetune=True,     finetune_epochs=2, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 97.37it/s, loss=0.39, metrics={'acc': 0.8152}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 104.04it/s, loss=0.359, metrics={'acc': 0.824}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 83.83it/s, loss=0.297, metrics={'acc': 0.8365}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 82.78it/s, loss=0.283, metrics={'acc': 0.8445}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 72.84it/s, loss=0.281, metrics={'acc': 0.8716}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 77.46it/s, loss=0.273, metrics={'acc': 0.8744}]\n</pre> <p>Note that, as I describe above, in scenario 2, we can just use this to warm up models before they joined training begins:</p> In\u00a0[13]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[14]: Copied! <pre>trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[15]: Copied! <pre>trainer_2.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    warmup=True,\n    warmup_epochs=2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_2.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     warmup=True,     warmup_epochs=2,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 102.49it/s, loss=0.52, metrics={'acc': 0.7519}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 98.15it/s, loss=0.381, metrics={'acc': 0.7891}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 82.97it/s, loss=0.356, metrics={'acc': 0.8043}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 80.27it/s, loss=0.295, metrics={'acc': 0.8195}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 77.27it/s, loss=0.291, metrics={'acc': 0.8667}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 89.57it/s, loss=0.289, metrics={'acc': 0.8665}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.69it/s, loss=0.283, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 91.81it/s, loss=0.284, metrics={'acc': 0.869}]\n</pre> <p>We need to explicitly indicate</p> <ol> <li><p>That we want fine-tune</p> </li> <li><p>The components that we want to individually fine-tune</p> </li> <li><p>In case of gradual fine-tuning, the routine (\"felbo\" or \"howard\")</p> </li> <li><p>The layers we want to fine-tune.</p> </li> </ol> <p>For example</p> In\u00a0[16]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[17]: Copied! <pre>model\n</pre> model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>let's first train as usual</p> In\u00a0[18]: Copied! <pre>trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[19]: Copied! <pre>trainer_3.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256\n)\n</pre> trainer_3.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.23it/s, loss=0.382, metrics={'acc': 0.8239}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.72it/s, loss=0.331, metrics={'acc': 0.8526}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.35it/s, loss=0.33, metrics={'acc': 0.8465}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 68.15it/s, loss=0.312, metrics={'acc': 0.8604}]\n</pre> In\u00a0[20]: Copied! <pre>trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\")\n</pre> trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\") <p>Now we are going to fine-tune the model components, and in the case of the <code>deeptabular</code> component, we will fine-tune the resnet-blocks and the linear layer but NOT the embeddings.</p> <p>For this, we need to access the model component's children: <code>deeptabular</code> $\\rightarrow$ <code>tab_resnet</code> $\\rightarrow$ <code>dense_resnet</code> $\\rightarrow$ <code>blocks</code></p> In\u00a0[21]: Copied! <pre>wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet_3 = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel_3 = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet_3 = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model_3 = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[22]: Copied! <pre>model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\"))\n</pre> model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\")) Out[22]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[23]: Copied! <pre>model_3\n</pre> model_3 Out[23]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre>tab_lin_layer = list(model_3.deeptabular.children())[1]\n</pre> tab_lin_layer = list(model_3.deeptabular.children())[1] In\u00a0[25]: Copied! <pre>tab_lin_layer\n</pre> tab_lin_layer Out[25]: <pre>Linear(in_features=200, out_features=1, bias=True)</pre> In\u00a0[26]: Copied! <pre>tab_deep_layers = []\nfor n1, c1 in model_3.deeptabular.named_children():\n    if (\n        n1 == \"0\"\n    ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class\n        for n2, c2 in c1.named_children():\n            if n2 == \"encoder\":  # TabResnet\n                for _, c3 in c2.named_children():\n                    for n4, c4 in c3.named_children():  # dense_resnet\n                        if \"block\" in n4:\n                            tab_deep_layers.append((n4, c4))\n</pre> tab_deep_layers = [] for n1, c1 in model_3.deeptabular.named_children():     if (         n1 == \"0\"     ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class         for n2, c2 in c1.named_children():             if n2 == \"encoder\":  # TabResnet                 for _, c3 in c2.named_children():                     for n4, c4 in c3.named_children():  # dense_resnet                         if \"block\" in n4:                             tab_deep_layers.append((n4, c4)) In\u00a0[27]: Copied! <pre>tab_deep_layers\n</pre> tab_deep_layers Out[27]: <pre>[('block_0',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  )),\n ('block_1',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  ))]</pre> <p>Now remember, we need to pass ONLY LAYERS (before I included the name for clarity) the layers in WARM UP ORDER, therefore:</p> In\u00a0[28]: Copied! <pre>tab_deep_layers = [el[1] for el in tab_deep_layers][::-1]\n</pre> tab_deep_layers = [el[1] for el in tab_deep_layers][::-1] In\u00a0[29]: Copied! <pre>tab_layers = [tab_lin_layer] + tab_deep_layers[::-1]\n</pre> tab_layers = [tab_lin_layer] + tab_deep_layers[::-1] In\u00a0[30]: Copied! <pre>tab_layers\n</pre> tab_layers Out[30]: <pre>[Linear(in_features=200, out_features=1, bias=True),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n ),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )]</pre> <p>And now simply</p> In\u00a0[31]: Copied! <pre>trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy]) In\u00a0[32]: Copied! <pre>trainer_4.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    deeptabular_gradual=True,\n    deeptabular_layers=tab_layers,\n    deeptabular_max_lr=0.01,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_4.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     deeptabular_gradual=True,     deeptabular_layers=tab_layers,     deeptabular_max_lr=0.01,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 95.17it/s, loss=0.504, metrics={'acc': 0.7523}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 99.83it/s, loss=0.384, metrics={'acc': 0.789}]\n</pre> <pre>Training deeptabular, layer 1 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.31it/s, loss=0.317, metrics={'acc': 0.8098}]\n</pre> <pre>Training deeptabular, layer 2 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 65.97it/s, loss=0.312, metrics={'acc': 0.8214}]\n</pre> <pre>Training deeptabular, layer 3 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 63.92it/s, loss=0.306, metrics={'acc': 0.8284}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 57.26it/s, loss=0.292, metrics={'acc': 0.8664}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.56it/s, loss=0.292, metrics={'acc': 0.8696}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 53.61it/s, loss=0.282, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 80.59it/s, loss=0.289, metrics={'acc': 0.8719}]\n</pre> <p>Finally, there is one more use case I would like to consider. The case where we train only one component and we just want to fine-tune and stop the training afterwards, since there is no joined training. This is a simple as</p> In\u00a0[33]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[34]: Copied! <pre>trainer_5 = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_5 = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[35]: Copied! <pre>trainer_5.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256\n)\n</pre> trainer_5.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.69it/s, loss=0.365, metrics={'acc': 0.8331}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 92.56it/s, loss=0.299, metrics={'acc': 0.8673}]\n</pre> In\u00a0[36]: Copied! <pre>trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\")\n</pre> trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\") In\u00a0[37]: Copied! <pre>tab_mlp_5 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_5 = WideDeep(deeptabular=tab_mlp_5)\n</pre> tab_mlp_5 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_5 = WideDeep(deeptabular=tab_mlp_5) In\u00a0[38]: Copied! <pre>model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\"))\n</pre> model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\")) Out[38]: <pre>&lt;All keys matched successfully&gt;</pre> <p>...times go by...</p> In\u00a0[39]: Copied! <pre>trainer_6 = Trainer(\n    model_5,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_6 = Trainer(     model_5,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[40]: Copied! <pre>trainer_6.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    finetune_max_lr=0.01,\n    stop_after_finetuning=True,\n    batch_size=256,\n)\n</pre> trainer_6.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     finetune_max_lr=0.01,     stop_after_finetuning=True,     batch_size=256, ) <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.86it/s, loss=0.298, metrics={'acc': 0.8652}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 75.45it/s, loss=0.286, metrics={'acc': 0.8669}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 1 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 76.29it/s, loss=0.282, metrics={'acc': 0.8698}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.93it/s, loss=0.281, metrics={'acc': 0.8749}]\n</pre> In\u00a0[42]: Copied! <pre>import shutil\n\nshutil.rmtree(\"models_dir/\")\nshutil.rmtree(\"model_weights/\")\n</pre> import shutil  shutil.rmtree(\"models_dir/\") shutil.rmtree(\"model_weights/\") In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/06_finetune_and_warmup.html#the-finetunewarm-up-option","title":"The FineTune/Warm Up option\u00b6","text":"<p>Let's place ourselves in two possible scenarios.</p> <ol> <li><p>Let's assume we have run a model and we want to just transfer the learnings (you know...transfer-learning) to another dataset, or simply we have received new data and we do not want to start the training of each component from scratch. Simply, we want to load the pre-trained weights and fine-tune.</p> </li> <li><p>We just want to \"warm up\" individual model components individually before the joined training begins.</p> </li> </ol> <p>This can be done with the <code>finetune</code> set of parameters. There are 3 fine-tuning routines:</p> <ol> <li>Fine-tune all trainable layers at once with a triangular one-cycle learning rate (referred as slanted triangular learning rates in Howard &amp; Ruder 2018)</li> <li>Gradual fine-tuning inspired by the work of Felbo et al., 2017</li> <li>Gradual fine-tuning based on the work of Howard &amp; Ruder 2018</li> </ol> <p>Currently fine-tunning is only supported without a fully connected head, i.e. if <code>deephead=None</code>. In addition, <code>Felbo</code> and <code>Howard</code> routines only applied, of course, to the <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> models. The <code>wide</code> component can also be fine-tuned, but only in an \"all at once\" mode.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-or-warm-up-all-at-once","title":"Fine-tune or warm-up all at once\u00b6","text":"<p>Here, the model components will be trained for <code>finetune_epochs</code> using a triangular one-cycle learning rate (slanted triangular learning rate) ranging from <code>finetune_max_lr/10</code> to <code>finetune_max_lr</code> (default is 0.01). 10% of the training steps are used to increase the learning rate which then decreases for the remaining 90%.</p> <p>Here all trainable layers are fine-tuned.</p> <p>Let's have a look to one example.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-gradually-the-felbo-and-the-howard-routines","title":"Fine-tune Gradually: The \"felbo\"  and the \"howard\" routines\u00b6","text":"<p>The Felbo routine can be illustrated as follows:</p> <p> </p> <p>Figure 1. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while freezing the already warmed up layer(s). Repeat untill all individual layers are warmed. Then warm one last epoch with all warmed layers trainable. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that this is not identical to the Fine-Tunning routine described in Felbo et al, 2017, this is why I used the word 'inspired'.</p> <p>The Howard routine can be illustrated as follows:</p> <p> </p> <p>Figure 2. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while keeping the already warmed up layer(s) trainable. Repeat. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that I write \"fine-tune (or train) the last layer for one epoch [...]\". However, in practice the user will have to specify the order of the layers to be fine-tuned. This is another reason why I wrote that the fine-tune routines I have implemented are inspired by the work of Felbo and Howard and not identical to their implemenations.</p> <p>The <code>felbo</code> and <code>howard</code> routines can be accessed with via the <code>fine-tune</code> parameters.</p>"},{"location":"examples/07_custom_components.html","title":"07_custom_components","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\n\nfrom torch import Tensor\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import os import torch  from torch import Tensor from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n# it does not make sense to standarised Latitude and Longitude\nalready_standard = [\"latitude\", \"longitude\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"] # it does not make sense to standarised Latitude and Longitude already_standard = [\"latitude\", \"longitude\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n\ntext_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n\nimage_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df)  text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df)  image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\nReading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:02&lt;00:00, 497.80it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> <p>Now we are ready to build a wide and deep model. Three of the four components we will use are included in this package, and they will be combined with a custom <code>deeptext</code> component. Then the fit process will run with a custom loss function.</p> <p>Let's have a look</p> In\u00a0[6]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0) In\u00a0[7]: Copied! <pre>class MyDeepText(nn.Module):\n    def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):\n        super(MyDeepText, self).__init__()\n\n        # word/token embeddings\n        self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)\n\n        # stack of RNNs\n        self.rnn = nn.GRU(\n            embed_dim,\n            hidden_dim,\n            num_layers=2,\n            bidirectional=True,\n            batch_first=True,\n        )\n\n        # Remember, this MUST be defined. If not WideDeep will through an error\n        self.output_dim = hidden_dim * 2\n\n    def forward(self, X):\n        embed = self.word_embed(X.long())\n        o, h = self.rnn(embed)\n        return torch.cat((h[-2], h[-1]), dim=1)\n</pre> class MyDeepText(nn.Module):     def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):         super(MyDeepText, self).__init__()          # word/token embeddings         self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)          # stack of RNNs         self.rnn = nn.GRU(             embed_dim,             hidden_dim,             num_layers=2,             bidirectional=True,             batch_first=True,         )          # Remember, this MUST be defined. If not WideDeep will through an error         self.output_dim = hidden_dim * 2      def forward(self, X):         embed = self.word_embed(X.long())         o, h = self.rnn(embed)         return torch.cat((h[-2], h[-1]), dim=1) In\u00a0[8]: Copied! <pre>mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos))\n</pre> mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos)) In\u00a0[9]: Copied! <pre>model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet)\n</pre> model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet) In\u00a0[10]: Copied! <pre>class RMSELoss(nn.Module):\n    def __init__(self):\n        \"\"\"root mean squared error\"\"\"\n        super().__init__()\n        self.mse = nn.MSELoss()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        return torch.sqrt(self.mse(input, target))\n</pre> class RMSELoss(nn.Module):     def __init__(self):         \"\"\"root mean squared error\"\"\"         super().__init__()         self.mse = nn.MSELoss()      def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:         return torch.sqrt(self.mse(input, target)) <p>and now we just instantiate the <code>Trainer</code> as usual. Needless to say, but this runs with 1000 random observations, so loss and metric values are meaningless. This is just an example</p> In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss())\n</pre> trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss()) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:23&lt;00:00,  1.07it/s, loss=126]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:05&lt;00:00,  1.24it/s, loss=97.4]\n</pre> <p>In addition to model components and loss functions, we can also use custom callbacks or custom metrics. The former need to be of type <code>Callback</code> and the latter need to be of type <code>Metric</code>. See:</p> <pre>pytorch-widedeep.callbacks\n</pre> <p>and</p> <pre>pytorch-widedeep.metrics\n</pre> <p>For this example let me use the adult dataset. Again, we first prepare the data as usual</p> In\u00a0[13]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[13]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[14]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[14]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[15]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[16]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[18]: Copied! <pre>from pytorch_widedeep.metrics import Metric\n</pre> from pytorch_widedeep.metrics import Metric In\u00a0[19]: Copied! <pre>class Accuracy(Metric):\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n\n        # \u00a0metric name needs to be defined\n        self._name = \"acc\"\n\n    def reset(self):\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</pre> class Accuracy(Metric):     def __init__(self, top_k: int = 1):         super(Accuracy, self).__init__()          self.top_k = top_k         self.correct_count = 0         self.total_count = 0          # \u00a0metric name needs to be defined         self._name = \"acc\"      def reset(self):         self.correct_count = 0         self.total_count = 0      def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:         num_classes = y_pred.size(1)          if num_classes == 1:             y_pred = y_pred.round()             y_true = y_true         elif num_classes &gt; 1:             y_pred = y_pred.topk(self.top_k, 1)[1]             y_true = y_true.view(-1, 1).expand_as(y_pred)          self.correct_count += y_pred.eq(y_true).sum().item()         self.total_count += len(y_pred)         accuracy = float(self.correct_count) / float(self.total_count)         return np.array(accuracy) In\u00a0[20]: Copied! <pre># have a look to the class\nfrom pytorch_widedeep.callbacks import Callback\n</pre> # have a look to the class from pytorch_widedeep.callbacks import Callback In\u00a0[21]: Copied! <pre>class SillyCallback(Callback):\n    def on_train_begin(self, logs=None):\n        # recordings will be the trainer object attributes\n        self.trainer.silly_callback = {}\n\n        self.trainer.silly_callback[\"beginning\"] = []\n        self.trainer.silly_callback[\"end\"] = []\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.trainer.silly_callback[\"beginning\"].append(epoch + 1)\n\n    def on_epoch_end(self, epoch, logs=None, metric=None):\n        self.trainer.silly_callback[\"end\"].append(epoch + 1)\n</pre> class SillyCallback(Callback):     def on_train_begin(self, logs=None):         # recordings will be the trainer object attributes         self.trainer.silly_callback = {}          self.trainer.silly_callback[\"beginning\"] = []         self.trainer.silly_callback[\"end\"] = []      def on_epoch_begin(self, epoch, logs=None):         self.trainer.silly_callback[\"beginning\"].append(epoch + 1)      def on_epoch_end(self, epoch, logs=None, metric=None):         self.trainer.silly_callback[\"end\"].append(epoch + 1) <p>and now, as usual:</p> In\u00a0[22]: Copied! <pre>trainer = Trainer(\n    model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback]\n)\n</pre> trainer = Trainer(     model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback] ) In\u00a0[23]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:06&lt;00:00, 94.39it/s, loss=0.411, metrics={'acc': 0.814}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 121.91it/s, loss=0.327, metrics={'acc': 0.8449}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:07&lt;00:00, 85.39it/s, loss=0.324, metrics={'acc': 0.8495}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.68it/s, loss=0.298, metrics={'acc': 0.8612}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 74.35it/s, loss=0.302, metrics={'acc': 0.8593}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 100.51it/s, loss=0.29, metrics={'acc': 0.8665}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 73.83it/s, loss=0.292, metrics={'acc': 0.8637}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 105.98it/s, loss=0.286, metrics={'acc': 0.8695}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 72.15it/s, loss=0.286, metrics={'acc': 0.866}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.27it/s, loss=0.284, metrics={'acc': 0.8698}]\n</pre> In\u00a0[24]: Copied! <pre>trainer.silly_callback\n</pre> trainer.silly_callback Out[24]: <pre>{'beginning': [1, 2, 3, 4, 5], 'end': [1, 2, 3, 4, 5]}</pre>"},{"location":"examples/07_custom_components.html#custom-components","title":"Custom components\u00b6","text":"<p>As I mentioned earlier in the example notebooks, and also in the <code>README</code>, it is possible to customise almost every component in <code>pytorch-widedeep</code>.</p> <p>Let's now go through a couple of simple examples to illustrate how that could be done.</p> <p>First let's load and process the data \"as usual\", let's start with a regression and the airbnb dataset.</p>"},{"location":"examples/07_custom_components.html#custom-deeptext","title":"Custom <code>deeptext</code>\u00b6","text":"<p>Standard Pytorch model</p>"},{"location":"examples/07_custom_components.html#custom-loss-function","title":"Custom loss function\u00b6","text":"<p>Loss functions must simply inherit pytorch's <code>nn.Module</code>. For example, let's say we want to use <code>RMSE</code> (note that this is already available in the package, but I will pass it here as a custom loss for illustration purposes)</p>"},{"location":"examples/07_custom_components.html#custom-metric","title":"Custom metric\u00b6","text":"<p>Let's say we want to use our own accuracy metric (again, this is already available in the package, but I will pass it here as a custom loss for illustration purposes).</p> <p>This could be done as:</p>"},{"location":"examples/07_custom_components.html#custom-callback","title":"Custom Callback\u00b6","text":"<p>Let's code a callback that records the current epoch at the beginning and the end of each epoch (silly, but you know, this is just an example)</p>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html","title":"08_custom_dataLoader_imbalanced_dataset","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\n# drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n\ndf_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n\ncontinuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> df = load_bio_kdd04(as_frame=True) # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)  df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 )  continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[3]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[4]: Copied! <pre># Define the model\ninput_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n\ndeeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=hidden_layers,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> # Define the model input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist()  deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols,     mlp_hidden_dims=hidden_layers, ) model = WideDeep(deeptabular=deeptabular) model Out[4]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[5]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\n\n# # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision],\n    verbose=1,\n)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False)  # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1)  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)  # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision],     verbose=1, ) In\u00a0[6]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=1,\n    batch_size=32,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n\npd.DataFrame(trainer.history)\n\ndf_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=1,     batch_size=32,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) )  pd.DataFrame(trainer.history)  df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 325/325 [00:02&lt;00:00, 153.99it/s, loss=0.163, metrics={'acc': 0.9363, 'prec': [0.9358]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:02&lt;00:00, 205.93it/s, loss=0.1, metrics={'acc': 0.9501, 'prec': [0.1447]}]\n</pre> <pre>Training time[s]: 0:00:04\n</pre> <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:01&lt;00:00, 368.16it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.95      0.97     14446\n           1       0.15      0.95      0.25       130\n\n    accuracy                           0.95     14576\n   macro avg       0.57      0.95      0.61     14576\nweighted avg       0.99      0.95      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13736,   840]))\n</pre>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#custom-dataloader-for-imbalanced-dataset","title":"Custom DataLoader for Imbalanced dataset\u00b6","text":""},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/09_extracting_embeddings.html","title":"09_extracting_embeddings","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import FTTransformer, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep import Tab2Vec\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import FTTransformer, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep import Tab2Vec from pytorch_widedeep.datasets import load_adult In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"educational_num\"], axis=1, inplace=True)\n\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"educational_num\"], axis=1, inplace=True)  df.head() Out[3]: age workclass fnlwgt education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_cols, cont_cols = [], []\nfor col in df.columns:\n    # 50 is just a random number I choose here for this example\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":\n        cat_cols.append(col)\n    elif col != \"target\":\n        cont_cols.append(col)\ntarget_col = \"target\"\n</pre> cat_cols, cont_cols = [], [] for col in df.columns:     # 50 is just a random number I choose here for this example     if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":         cat_cols.append(col)     elif col != \"target\":         cont_cols.append(col) target_col = \"target\" In\u00a0[5]: Copied! <pre>target = df[target_col].values\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> target = df[target_col].values  tab_preprocessor = TabPreprocessor(     embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[6]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    n_blocks=2,\n    n_heads=4,\n    input_dim=16,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     n_blocks=2,     n_heads=4,     input_dim=16, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/utils/general_utils.py:12: DeprecationWarning: The 'embed_continuous' parameter is deprecated and will be removed in the next release. Please use 'embed_continuous_method' instead See the documentation for more details.\n  return func(*args, **kwargs)\n</pre> In\u00a0[7]: Copied! <pre>model = WideDeep(deeptabular=ft_transformer)\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2)\n</pre> model = WideDeep(deeptabular=ft_transformer) trainer = Trainer(model, objective=\"binary\", metrics=[Accuracy]) trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 41.47it/s, loss=221, metrics={'acc': 0.686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 86.24it/s, loss=9.28, metrics={'acc': 0.76}]\n</pre> In\u00a0[8]: Copied! <pre>t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor)\n</pre> t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor) In\u00a0[9]: Copied! <pre># assuming is a test set with target col\nX_vec, y = t2v.transform(df.sample(100), target_col=\"target\")\n</pre> # assuming is a test set with target col X_vec, y = t2v.transform(df.sample(100), target_col=\"target\") In\u00a0[10]: Copied! <pre># X vec is the dataframe turned into the embeddings\nX_vec.shape\n</pre> # X vec is the dataframe turned into the embeddings X_vec.shape Out[10]: <pre>(100, 208)</pre> <p><code>208 = input_dim (16) * n_cols (13)</code></p> In\u00a0[11]: Copied! <pre># ...or if we don't have target col\nX_vec = t2v.transform(df.sample(100))\n</pre> # ...or if we don't have target col X_vec = t2v.transform(df.sample(100))"},{"location":"examples/09_extracting_embeddings.html#extracting-embeddings","title":"Extracting embeddings\u00b6","text":"<p>This notebook is a simple guide to extracting learned feature embeddings using Tab2Vec</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html","title":"10_3rd_party_integration-RayTune_WnB","text":"In\u00a0[11]: Copied! <pre>from typing import Optional, Dict\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport wandb\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.callbacks import (\n    EarlyStopping,\n    ModelCheckpoint,\n    Callback,\n)\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\nfrom ray import tune\nfrom ray.tune.schedulers import AsyncHyperBandScheduler\nfrom ray.tune import JupyterNotebookReporter\nfrom ray.air.integrations.wandb import WandbLoggerCallback\n\n# from ray.tune.integration.wandb import wandb_mixin\n\nimport tracemalloc\n\ntracemalloc.start()\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> from typing import Optional, Dict import os  import numpy as np import pandas as pd import torch import wandb from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.callbacks import (     EarlyStopping,     ModelCheckpoint,     Callback, ) from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  from ray import tune from ray.tune.schedulers import AsyncHyperBandScheduler from ray.tune import JupyterNotebookReporter from ray.air.integrations.wandb import WandbLoggerCallback  # from ray.tune.integration.wandb import wandb_mixin  import tracemalloc  tracemalloc.start()  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[12]: Copied! <pre>class RayTuneReporter(Callback):\n    r\"\"\"Callback that allows reporting history and lr_history values to RayTune\n    during Hyperparameter tuning\n\n    Callbacks are passed as input parameters to the ``Trainer`` class. See\n    :class:`pytorch_widedeep.trainer.Trainer`\n\n    For examples see the examples folder at:\n\n        .. code-block:: bash\n\n            /examples/12_HyperParameter_tuning_w_RayTune.ipynb\n    \"\"\"\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        report_dict = {}\n        for k, v in self.trainer.history.items():\n            report_dict.update({k: v[-1]})\n        if hasattr(self.trainer, \"lr_history\"):\n            for k, v in self.trainer.lr_history.items():\n                report_dict.update({k: v[-1]})\n        tune.report(report_dict)\n\n\nclass WnBReportBest(Callback):\n    r\"\"\"Callback that allows reporting best performance of a run to WnB\n    during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint\n    with added WnB and removed checkpoint saving.\n\n    Callbacks are passed as input parameters to the ``Trainer`` class.\n\n    Parameters\n    ----------\n    wb: obj\n        Weights&amp;Biases API interface to report single best result usable for\n        comparisson of multiple paramater combinations by, for example,\n        `parallel coordinates\n        &lt;https://docs.wandb.ai/ref/app/features/panels/parallel-coordinates&gt;`_.\n        E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically `'val_loss'` or metric name\n        (e.g. `'val_acc'`)\n    mode: str, default=\"auto\"\n        If ``save_best_only=True``, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For `'acc'`, this should be `'max'`, for\n        `'loss'` this should be `'min'`, etc. In `'auto'` mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        wb: object,\n        monitor: str = \"val_loss\",\n        mode: str = \"auto\",\n    ):\n        super(WnBReportBest, self).__init__()\n\n        self.monitor = monitor\n        self.mode = mode\n        self.wb = wb\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"WnBReportBest mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if self._is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is not None:\n            if self.monitor_op(current, self.best):\n                self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]\n                self.best = current\n                self.best_epoch = epoch\n\n    @staticmethod\n    def _is_metric(monitor: str):\n        \"copied from pytorch_widedeep.callbacks\"\n        if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):\n            return True\n        else:\n            return False\n</pre> class RayTuneReporter(Callback):     r\"\"\"Callback that allows reporting history and lr_history values to RayTune     during Hyperparameter tuning      Callbacks are passed as input parameters to the ``Trainer`` class. See     :class:`pytorch_widedeep.trainer.Trainer`      For examples see the examples folder at:          .. code-block:: bash              /examples/12_HyperParameter_tuning_w_RayTune.ipynb     \"\"\"      def on_epoch_end(         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         report_dict = {}         for k, v in self.trainer.history.items():             report_dict.update({k: v[-1]})         if hasattr(self.trainer, \"lr_history\"):             for k, v in self.trainer.lr_history.items():                 report_dict.update({k: v[-1]})         tune.report(report_dict)   class WnBReportBest(Callback):     r\"\"\"Callback that allows reporting best performance of a run to WnB     during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint     with added WnB and removed checkpoint saving.      Callbacks are passed as input parameters to the ``Trainer`` class.      Parameters     ----------     wb: obj         Weights&amp;Biases API interface to report single best result usable for         comparisson of multiple paramater combinations by, for example,         `parallel coordinates         `_.         E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.     monitor: str, default=\"loss\"         quantity to monitor. Typically `'val_loss'` or metric name         (e.g. `'val_acc'`)     mode: str, default=\"auto\"         If ``save_best_only=True``, the decision to overwrite the current save         file is made based on either the maximization or the minimization of         the monitored quantity. For `'acc'`, this should be `'max'`, for         `'loss'` this should be `'min'`, etc. In `'auto'` mode, the         direction is automatically inferred from the name of the monitored         quantity.      \"\"\"      def __init__(         self,         wb: object,         monitor: str = \"val_loss\",         mode: str = \"auto\",     ):         super(WnBReportBest, self).__init__()          self.monitor = monitor         self.mode = mode         self.wb = wb          if self.mode not in [\"auto\", \"min\", \"max\"]:             warnings.warn(                 \"WnBReportBest mode %s is unknown, \"                 \"fallback to auto mode.\" % (self.mode),                 RuntimeWarning,             )             self.mode = \"auto\"         if self.mode == \"min\":             self.monitor_op = np.less             self.best = np.Inf         elif self.mode == \"max\":             self.monitor_op = np.greater  # type: ignore[assignment]             self.best = -np.Inf         else:             if self._is_metric(self.monitor):                 self.monitor_op = np.greater  # type: ignore[assignment]                 self.best = -np.Inf             else:                 self.monitor_op = np.less                 self.best = np.Inf      def on_epoch_end(  # noqa: C901         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         logs = logs or {}         current = logs.get(self.monitor)         if current is not None:             if self.monitor_op(current, self.best):                 self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]                 self.best = current                 self.best_epoch = epoch      @staticmethod     def _is_metric(monitor: str):         \"copied from pytorch_widedeep.callbacks\"         if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):             return True         else:             return False In\u00a0[13]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[13]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[14]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[14]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[15]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[16]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[17]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[18]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[19]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[20]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular) model Out[20]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre># Metrics from torchmetrics\naccuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\")\nprecision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\")\nf1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\")\nrecall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\")\n</pre> # Metrics from torchmetrics accuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\") precision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\") f1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\") recall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\") <p>Note:</p> <p>Following cells includes usage of both <code>RayTuneReporter</code> and <code>WnBReportBest</code> callbacks. In case you want to use just <code>RayTuneReporter</code>, remove following:</p> <ul> <li>wandb from config</li> <li><code>WandbLoggerCallback</code></li> <li><code>WnBReportBest</code></li> <li><code>@wandb_mixin</code> decorator</li> </ul> <p>We do not see strong reason to use WnB without RayTune for a single paramater combination run, but it is possible:</p> <ul> <li>option01: define paramaters in config only for a single value <code>tune.grid_search([1000])</code> (single value RayTune run)</li> <li>option02: define WnB callback that reports currnet validation/training loss, metrics, etc. at the end of batch, ie. do not report to WnB at <code>epoch_end</code> as in <code>WnBReportBest</code> but at the <code>on_batch_end</code>, see <code>pytorch_widedeep.callbacks.Callback</code></li> </ul> In\u00a0[26]: Copied! <pre>config = {\n    \"batch_size\": tune.grid_search([1000, 5000]),\n    \"wandb\": {\n        \"project\": \"test\",\n        # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",\n        \"api_key\": \"WNB_API_KEY\",\n    },\n}\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\n\n@wandb_mixin\ndef training_function(config, X_train, X_val):\n    early_stopping = EarlyStopping()\n    model_checkpoint = ModelCheckpoint(save_best_only=True)\n    # Hyperparameters\n    batch_size = config[\"batch_size\"]\n    trainer = Trainer(\n        model,\n        objective=\"binary_focal_loss\",\n        callbacks=[\n            RayTuneReporter,\n            WnBReportBest(wb=wandb),\n            early_stopping,\n            model_checkpoint,\n        ],\n        lr_schedulers={\"deeptabular\": deep_sch},\n        initializers={\"deeptabular\": XavierNormal},\n        optimizers={\"deeptabular\": deep_opt},\n        metrics=[accuracy, precision, recall, f1],\n        verbose=0,\n    )\n\n    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)\n\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n\nasha_scheduler = AsyncHyperBandScheduler(\n    time_attr=\"training_iteration\",\n    metric=\"_metric/val_loss\",\n    mode=\"min\",\n    max_t=100,\n    grace_period=10,\n    reduction_factor=3,\n    brackets=1,\n)\n\nanalysis = tune.run(\n    tune.with_parameters(training_function, X_train=X_train, X_val=X_val),\n    resources_per_trial={\"cpu\": 1, \"gpu\": 0},\n    progress_reporter=JupyterNotebookReporter(overwrite=True),\n    scheduler=asha_scheduler,\n    config=config,\n    callbacks=[\n        WandbLoggerCallback(\n            project=config[\"wandb\"][\"project\"],\n            # api_key_file=config[\"wandb\"][\"api_key_file\"],\n            api_key=config[\"wandb\"][\"api_key\"],\n            log_config=True,\n        )\n    ],\n)\n</pre> config = {     \"batch_size\": tune.grid_search([1000, 5000]),     \"wandb\": {         \"project\": \"test\",         # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",         \"api_key\": \"WNB_API_KEY\",     }, }  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)   @wandb_mixin def training_function(config, X_train, X_val):     early_stopping = EarlyStopping()     model_checkpoint = ModelCheckpoint(save_best_only=True)     # Hyperparameters     batch_size = config[\"batch_size\"]     trainer = Trainer(         model,         objective=\"binary_focal_loss\",         callbacks=[             RayTuneReporter,             WnBReportBest(wb=wandb),             early_stopping,             model_checkpoint,         ],         lr_schedulers={\"deeptabular\": deep_sch},         initializers={\"deeptabular\": XavierNormal},         optimizers={\"deeptabular\": deep_opt},         metrics=[accuracy, precision, recall, f1],         verbose=0,     )      trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)   X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}  asha_scheduler = AsyncHyperBandScheduler(     time_attr=\"training_iteration\",     metric=\"_metric/val_loss\",     mode=\"min\",     max_t=100,     grace_period=10,     reduction_factor=3,     brackets=1, )  analysis = tune.run(     tune.with_parameters(training_function, X_train=X_train, X_val=X_val),     resources_per_trial={\"cpu\": 1, \"gpu\": 0},     progress_reporter=JupyterNotebookReporter(overwrite=True),     scheduler=asha_scheduler,     config=config,     callbacks=[         WandbLoggerCallback(             project=config[\"wandb\"][\"project\"],             # api_key_file=config[\"wandb\"][\"api_key_file\"],             api_key=config[\"wandb\"][\"api_key\"],             log_config=True,         )     ], ) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmp60pfyl1kwandb'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpnjv2rg1wwandb-artifacts'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpgebu5k1kwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpxy9y2yriwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n</pre> In\u00a0[14]: Copied! <pre>analysis.results\n</pre> analysis.results Out[14]: <pre>{'fc9a8_00000': {'_metric': {'train_loss': 0.006297602537127896,\n   'train_Accuracy': 0.9925042986869812,\n   'train_Precision': 0.9939393997192383,\n   'train_Recall': 0.15814851224422455,\n   'train_F1Score': 0.2728785574436188,\n   'val_loss': 0.005045663565397263,\n   'val_Accuracy': 0.9946483969688416,\n   'val_Precision': 1.0,\n   'val_Recall': 0.39534884691238403,\n   'val_F1Score': 0.5666667222976685},\n  'time_this_iter_s': 2.388202428817749,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00000',\n  'experiment_id': 'baad1d4f3d924b48b9ece1b9f26c80cc',\n  'date': '2022-07-31_14-06-51',\n  'timestamp': 1659276411,\n  'time_total_s': 12.656474113464355,\n  'pid': 1813,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 1000},\n  'time_since_restore': 12.656474113464355,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.8006253242492676,\n  'experiment_tag': '0_batch_size=1000'},\n 'fc9a8_00001': {'_metric': {'train_loss': 0.02519632239515583,\n   'train_Accuracy': 0.9910891652107239,\n   'train_Precision': 0.25,\n   'train_Recall': 0.0009643201483413577,\n   'train_F1Score': 0.0019212296465411782,\n   'val_loss': 0.02578434906899929,\n   'val_Accuracy': 0.9911492466926575,\n   'val_Precision': 0.0,\n   'val_Recall': 0.0,\n   'val_F1Score': 0.0},\n  'time_this_iter_s': 4.113586902618408,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00001',\n  'experiment_id': 'f2e54a6a5780429fbf0db0746853347e',\n  'date': '2022-07-31_14-06-56',\n  'timestamp': 1659276416,\n  'time_total_s': 12.926990509033203,\n  'pid': 1962,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 5000},\n  'time_since_restore': 12.926990509033203,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.9253025054931641,\n  'experiment_tag': '1_batch_size=5000'}}</pre> <p>Using Weights and Biases logging you can create parallel coordinates graphs that map parametr combinations to the best(lowest) loss achieved during the training of the networks</p> <p></p> <p>local visualization of raytune reults using tensorboard</p> In\u00a0[23]: Copied! <pre>%load_ext tensorboard\n%tensorboard --logdir ~/ray_results\n</pre> %load_ext tensorboard %tensorboard --logdir ~/ray_results"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#3rd-party-integration-raytune-weights-biases","title":"3rd party integration - RayTune, Weights &amp; Biases\u00b6","text":"<p>This notebook provides guideline for integration of external library functions in the model training process through <code>Callback</code> objects, a popular concept of using objects as arguments for other objects.</p> <p>[DISCLAIMER]</p> <p>We show integration of RayTune (a hyperparameter tuning framework) and Weights &amp; Biases (ML projects experiment tracking and versioning solution) in the <code>pytorch_widedeep</code> model training process. We did not include <code>RayTuneReporter</code> and <code>WnBReportBest</code> in the library code to minimize the dependencies on other libraries that are not directly included in the model design and training.</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/11_auc_multiclass.html","title":"11_auc_multiclass","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import AUROC\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_ecoli\nfrom pytorch_widedeep.utils import LabelEncoder\n\nfrom sklearn.model_selection import train_test_split\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import AUROC from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_ecoli from pytorch_widedeep.utils import LabelEncoder  from sklearn.model_selection import train_test_split  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_ecoli(as_frame=True)\ndf.head()\n</pre> df = load_ecoli(as_frame=True) df.head() Out[2]: SequenceName mcg gvh lip chg aac alm1 alm2 class 0 AAT_ECOLI 0.49 0.29 0.48 0.5 0.56 0.24 0.35 cp 1 ACEA_ECOLI 0.07 0.40 0.48 0.5 0.54 0.35 0.44 cp 2 ACEK_ECOLI 0.56 0.40 0.48 0.5 0.49 0.37 0.46 cp 3 ACKA_ECOLI 0.59 0.49 0.48 0.5 0.52 0.45 0.36 cp 4 ADI_ECOLI 0.23 0.32 0.48 0.5 0.55 0.25 0.35 cp In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"class\"].value_counts()\n</pre> # imbalance of the classes df[\"class\"].value_counts() Out[3]: <pre>class\ncp     143\nim      77\npp      52\nimU     35\nom      20\nomL      5\nimS      2\nimL      2\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre>df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])]\ndf.reset_index(inplace=True, drop=True)\n</pre> df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])] df.reset_index(inplace=True, drop=True) In\u00a0[5]: Copied! <pre>encoder = LabelEncoder([\"class\"])\ndf_enc = encoder.fit_transform(df)\ndf_enc[\"class\"] = df_enc[\"class\"] - 1\n</pre> encoder = LabelEncoder([\"class\"]) df_enc = encoder.fit_transform(df) df_enc[\"class\"] = df_enc[\"class\"] - 1 In\u00a0[6]: Copied! <pre># drop columns we won't need in this example\ndf_enc = df_enc.drop(columns=[\"SequenceName\"])\n</pre> # drop columns we won't need in this example df_enc = df_enc.drop(columns=[\"SequenceName\"]) In\u00a0[7]: Copied! <pre>df_train, df_valid = train_test_split(\n    df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1 ) In\u00a0[8]: Copied! <pre>continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist()\n</pre> continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist() In\u00a0[9]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"class\"].values\ny_valid = df_valid[\"class\"].values\ny_test = df_test[\"class\"].values\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"class\"].values y_valid = df_valid[\"class\"].values y_test = df_test[\"class\"].values  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n  self._check_inputs(cat_embed_cols)\n</pre> In\u00a0[10]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique())\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique()) model Out[10]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=7, out_features=200, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=200, out_features=100, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=100, out_features=5, bias=True)\n  )\n)</pre> In\u00a0[11]: Copied! <pre>auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\")\n</pre> auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\") In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"multiclass_focal_loss\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[auroc],\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"multiclass_focal_loss\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[auroc], )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 54.59it/s, loss=0.109, metrics={'MulticlassAUROC': 0.314}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 98.35it/s, loss=0.105, metrics={'MulticlassAUROC': 0.2558}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 91.55it/s, loss=0.105, metrics={'MulticlassAUROC': 0.3546}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 111.68it/s, loss=0.101, metrics={'MulticlassAUROC': 0.2737}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 62.55it/s, loss=0.1, metrics={'MulticlassAUROC': 0.3795}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 108.51it/s, loss=0.0966, metrics={'MulticlassAUROC': 0.3053}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 99.35it/s, loss=0.0965, metrics={'MulticlassAUROC': 0.3809}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 117.73it/s, loss=0.0962, metrics={'MulticlassAUROC': 0.3089}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 110.56it/s, loss=0.0967, metrics={'MulticlassAUROC': 0.3509}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 127.35it/s, loss=0.0958, metrics={'MulticlassAUROC': 0.3089}]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/11_auc_multiclass.html#auc-multiclass-computation","title":"AUC multiclass computation\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html","title":"12_ZILNLoss_origkeras_vs_pytorch_widedeep","text":"In\u00a0[1]: Copied! <pre># @title Copyright 2019 The Lifetime Value Authors.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n</pre> # @title Copyright 2019 The Lifetime Value Authors. # Licensed under the Apache License, Version 2.0 (the \"License\"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an \"AS IS\" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ Run in Google Colab View source on GitHub In\u00a0[3]: Copied! <pre>import os\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nfrom typing import Sequence\n\n# install and import ltv\n!pip install -q git+https://github.com/google/lifetime_value\nimport lifetime_value as ltv\n</pre> import os  import numpy as np import pandas as pd from scipy import stats import matplotlib.pyplot as plt import seaborn as sns import tensorflow as tf import tensorflow_probability as tfp from typing import Sequence  # install and import ltv !pip install -q git+https://github.com/google/lifetime_value import lifetime_value as ltv In\u00a0[\u00a0]: Copied! <pre>tfd = tfp.distributions\n%config InlineBackend.figure_format='retina'\nsns.set_style(\"whitegrid\")\n</pre> tfd = tfp.distributions %config InlineBackend.figure_format='retina' sns.set_style(\"whitegrid\") In\u00a0[\u00a0]: Copied! <pre>MODEL = \"dnn\"\nLOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln']\nLEARNING_RATE = 0.001  # @param { isTemplate: true}\nVERSION = 0  # @param { isTemplate: true, type: 'integer'}\nOUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'}\n</pre> MODEL = \"dnn\" LOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln'] LEARNING_RATE = 0.001  # @param { isTemplate: true} VERSION = 0  # @param { isTemplate: true, type: 'integer'} OUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'} <p>Download kdd_cup_98 data to /tmp/lifetime-value/kdd_cup_98</p> In\u00a0[\u00a0]: Copied! <pre>%%bash\nmkdir -p /tmp/lifetime-value/kdd_cup_98\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/\ncd /tmp/lifetime-value/kdd_cup_98/\nunzip cup98lrn.zip\nunzip cup98val.zip\n</pre> %%bash mkdir -p /tmp/lifetime-value/kdd_cup_98 wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/ cd /tmp/lifetime-value/kdd_cup_98/ unzip cup98lrn.zip unzip cup98val.zip In\u00a0[\u00a0]: Copied! <pre>df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\")\nnum_train = df_train.shape[0]\ndf_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\")\ndf_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\")\ndf_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\")\n</pre> df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\") num_train = df_train.shape[0] df_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\") df_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\") df_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\") In\u00a0[\u00a0]: Copied! <pre>df = pd.concat([df_train, df_eval], axis=0, sort=True)\n</pre> df = pd.concat([df_train, df_eval], axis=0, sort=True) In\u00a0[\u00a0]: Copied! <pre>y = df[\"TARGET_D\"][:num_train]\n</pre> y = df[\"TARGET_D\"][:num_train] In\u00a0[\u00a0]: Copied! <pre>def plot_hist_log_scale(y):\n    max_val = y.max() + 1.0\n    ax = pd.Series(y).hist(\n        figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)\n    )\n\n    plt.xlabel(\"Donation ($)\")\n    plt.ylabel(\"Count\")\n    # plt.title('Histogram of LTV')\n    plt.xticks(rotation=\"horizontal\")\n    plt.legend(loc=\"upper left\")\n    ax.set_xscale(\"log\")\n    ax.grid(False)\n    # Hide the right and top spines\n    ax.spines[\"right\"].set_visible(False)\n    ax.spines[\"top\"].set_visible(False)\n    # Only show ticks on the left and bottom spines\n    ax.yaxis.set_ticks_position(\"left\")\n    ax.xaxis.set_ticks_position(\"bottom\")\n    plt.show()\n\n    fig = ax.get_figure()\n    output_file = tf.io.gfile.GFile(\n        \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"\n    )\n    fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\")\n</pre> def plot_hist_log_scale(y):     max_val = y.max() + 1.0     ax = pd.Series(y).hist(         figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)     )      plt.xlabel(\"Donation ($)\")     plt.ylabel(\"Count\")     # plt.title('Histogram of LTV')     plt.xticks(rotation=\"horizontal\")     plt.legend(loc=\"upper left\")     ax.set_xscale(\"log\")     ax.grid(False)     # Hide the right and top spines     ax.spines[\"right\"].set_visible(False)     ax.spines[\"top\"].set_visible(False)     # Only show ticks on the left and bottom spines     ax.yaxis.set_ticks_position(\"left\")     ax.xaxis.set_ticks_position(\"bottom\")     plt.show()      fig = ax.get_figure()     output_file = tf.io.gfile.GFile(         \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"     )     fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\") In\u00a0[\u00a0]: Copied! <pre>plot_hist_log_scale(y[y &gt; 0])\n</pre> plot_hist_log_scale(y[y &gt; 0]) In\u00a0[\u00a0]: Copied! <pre>VOCAB_FEATURES = [\n    \"ODATEDW\",  # date of donor's first gift (YYMM)\n    \"OSOURCE\",  # donor acquisition mailing list\n    \"TCODE\",  # donor title code\n    \"STATE\",\n    \"ZIP\",\n    \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood\n    \"CLUSTER\",  # socio-economic status\n    \"GENDER\",\n    \"MAXADATE\",  # date of the most recent promotion received\n    \"MINRDATE\",\n    \"LASTDATE\",\n    \"FISTDATE\",\n    \"RFA_2A\",\n]\n</pre> VOCAB_FEATURES = [     \"ODATEDW\",  # date of donor's first gift (YYMM)     \"OSOURCE\",  # donor acquisition mailing list     \"TCODE\",  # donor title code     \"STATE\",     \"ZIP\",     \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood     \"CLUSTER\",  # socio-economic status     \"GENDER\",     \"MAXADATE\",  # date of the most recent promotion received     \"MINRDATE\",     \"LASTDATE\",     \"FISTDATE\",     \"RFA_2A\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\")\ndf[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x))\ndf[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5)\ndf[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\")\ndf[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\")\ndf[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\")\ndf[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\")\n</pre> df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\") df[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x)) df[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5) df[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\") df[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\") df[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\") df[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\") In\u00a0[\u00a0]: Copied! <pre>def label_encoding(y, frequency_threshold=100):\n    value_counts = pd.value_counts(y)\n    categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()\n    # 0 indicates the unknown category.\n    return pd.Categorical(y, categories=categories).codes + 1\n</pre> def label_encoding(y, frequency_threshold=100):     value_counts = pd.value_counts(y)     categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()     # 0 indicates the unknown category.     return pd.Categorical(y, categories=categories).codes + 1 In\u00a0[\u00a0]: Copied! <pre>for key in VOCAB_FEATURES:\n    df[key] = label_encoding(df[key])\n</pre> for key in VOCAB_FEATURES:     df[key] = label_encoding(df[key]) In\u00a0[\u00a0]: Copied! <pre>MAIL_ORDER_RESPONSES = [\n    \"MBCRAFT\",\n    \"MBGARDEN\",\n    \"MBBOOKS\",\n    \"MBCOLECT\",\n    \"MAGFAML\",\n    \"MAGFEM\",\n    \"MAGMALE\",\n    \"PUBGARDN\",\n    \"PUBCULIN\",\n    \"PUBHLTH\",\n    \"PUBDOITY\",\n    \"PUBNEWFN\",\n    \"PUBPHOTO\",\n    \"PUBOPP\",\n    \"RFA_2F\",\n]\n</pre> MAIL_ORDER_RESPONSES = [     \"MBCRAFT\",     \"MBGARDEN\",     \"MBBOOKS\",     \"MBCOLECT\",     \"MAGFAML\",     \"MAGFEM\",     \"MAGMALE\",     \"PUBGARDN\",     \"PUBCULIN\",     \"PUBHLTH\",     \"PUBDOITY\",     \"PUBNEWFN\",     \"PUBPHOTO\",     \"PUBOPP\",     \"RFA_2F\", ] In\u00a0[\u00a0]: Copied! <pre>INDICATOR_FEATURES = [\n    \"AGE\",  # age decile, 0 indicates unknown\n    \"NUMCHLD\",\n    \"INCOME\",\n    \"WEALTH1\",\n    \"HIT\",\n] + MAIL_ORDER_RESPONSES\n</pre> INDICATOR_FEATURES = [     \"AGE\",  # age decile, 0 indicates unknown     \"NUMCHLD\",     \"INCOME\",     \"WEALTH1\",     \"HIT\", ] + MAIL_ORDER_RESPONSES In\u00a0[\u00a0]: Copied! <pre>df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1\ndf[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1)\ndf[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes\n\nfor col in MAIL_ORDER_RESPONSES:\n    df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1\n</pre> df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1 df[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1) df[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes  for col in MAIL_ORDER_RESPONSES:     df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1 In\u00a0[\u00a0]: Copied! <pre>NUMERIC_FEATURES = [\n    # binary\n    \"MAILCODE\",  # bad address\n    \"NOEXCH\",  # do not exchange\n    \"RECINHSE\",  # donor has given to PVA's in house program\n    \"RECP3\",  # donor has given to PVA's P3 program\n    \"RECPGVG\",  # planned giving record\n    \"RECSWEEP\",  # sweepstakes record\n    \"HOMEOWNR\",  # home owner\n    \"CHILD03\",\n    \"CHILD07\",\n    \"CHILD12\",\n    \"CHILD18\",\n    # continuous\n    \"CARDPROM\",\n    \"NUMPROM\",\n    \"CARDPM12\",\n    \"NUMPRM12\",\n    \"RAMNTALL\",\n    \"NGIFTALL\",\n    \"MINRAMNT\",\n    \"MAXRAMNT\",\n    \"LASTGIFT\",\n    \"AVGGIFT\",\n]\n</pre> NUMERIC_FEATURES = [     # binary     \"MAILCODE\",  # bad address     \"NOEXCH\",  # do not exchange     \"RECINHSE\",  # donor has given to PVA's in house program     \"RECP3\",  # donor has given to PVA's P3 program     \"RECPGVG\",  # planned giving record     \"RECSWEEP\",  # sweepstakes record     \"HOMEOWNR\",  # home owner     \"CHILD03\",     \"CHILD07\",     \"CHILD12\",     \"CHILD18\",     # continuous     \"CARDPROM\",     \"NUMPROM\",     \"CARDPM12\",     \"NUMPRM12\",     \"RAMNTALL\",     \"NGIFTALL\",     \"MINRAMNT\",     \"MAXRAMNT\",     \"LASTGIFT\",     \"AVGGIFT\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\")\ndf[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\")\ndf[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\")\ndf[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\")\ndf[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\")\ndf[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\")\ndf[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\")\ndf[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\")\ndf[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\n\ndf[\"CARDPROM\"] = df[\"CARDPROM\"] / 100\ndf[\"NUMPROM\"] = df[\"NUMPROM\"] / 100\ndf[\"CARDPM12\"] = df[\"CARDPM12\"] / 100\ndf[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100\ndf[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"])\ndf[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"])\ndf[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"])\ndf[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"])\ndf[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"])\ndf[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"])\n</pre> df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\") df[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\") df[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\") df[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\") df[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\") df[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\") df[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\") df[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\") df[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")  df[\"CARDPROM\"] = df[\"CARDPROM\"] / 100 df[\"NUMPROM\"] = df[\"NUMPROM\"] / 100 df[\"CARDPM12\"] = df[\"CARDPM12\"] / 100 df[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100 df[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"]) df[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"]) df[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"]) df[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"]) df[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"]) df[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"]) In\u00a0[\u00a0]: Copied! <pre>CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES\nALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES\n</pre> CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES ALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES In\u00a0[\u00a0]: Copied! <pre>def dnn_split(df):\n    df_train = df.iloc[:num_train]\n    df_eval = df.iloc[num_train:]\n\n    def feature_dict(df):\n        features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}\n        features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values\n        return features\n\n    x_train, y_train = (\n        feature_dict(df_train),\n        df_train[\"TARGET_D\"].astype(\"float32\").values,\n    )\n    x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values\n\n    return x_train, x_eval, y_train, y_eval\n</pre> def dnn_split(df):     df_train = df.iloc[:num_train]     df_eval = df.iloc[num_train:]      def feature_dict(df):         features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}         features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values         return features      x_train, y_train = (         feature_dict(df_train),         df_train[\"TARGET_D\"].astype(\"float32\").values,     )     x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values      return x_train, x_eval, y_train, y_eval In\u00a0[\u00a0]: Copied! <pre>def embedding_dim(x):\n    return int(x**0.25) + 1\n\n\ndef embedding_layer(vocab_size):\n    return tf.keras.Sequential(\n        [\n            tf.keras.layers.Embedding(\n                input_dim=vocab_size,\n                output_dim=embedding_dim(vocab_size),\n                input_length=1,\n            ),\n            tf.keras.layers.Flatten(),\n        ]\n    )\n\n\ndef dnn_model(output_units):\n    numeric_input = tf.keras.layers.Input(\n        shape=(len(NUMERIC_FEATURES),), name=\"numeric\"\n    )\n\n    embedding_inputs = [\n        tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)\n        for key in CATEGORICAL_FEATURES\n    ]\n\n    embedding_outputs = [\n        embedding_layer(vocab_size=df[key].max() + 1)(input)\n        for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)\n    ]\n\n    deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)\n    deep_model = tf.keras.Sequential(\n        [\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(units=output_units),\n        ]\n    )\n    return tf.keras.Model(\n        inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)\n    )\n</pre> def embedding_dim(x):     return int(x**0.25) + 1   def embedding_layer(vocab_size):     return tf.keras.Sequential(         [             tf.keras.layers.Embedding(                 input_dim=vocab_size,                 output_dim=embedding_dim(vocab_size),                 input_length=1,             ),             tf.keras.layers.Flatten(),         ]     )   def dnn_model(output_units):     numeric_input = tf.keras.layers.Input(         shape=(len(NUMERIC_FEATURES),), name=\"numeric\"     )      embedding_inputs = [         tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)         for key in CATEGORICAL_FEATURES     ]      embedding_outputs = [         embedding_layer(vocab_size=df[key].max() + 1)(input)         for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)     ]      deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)     deep_model = tf.keras.Sequential(         [             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(units=output_units),         ]     )     return tf.keras.Model(         inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)     ) In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    loss = tf.keras.losses.MeanSquaredError()\n    output_units = 1\n\nif LOSS == \"ziln\":\n    loss = ltv.zero_inflated_lognormal_loss\n    output_units = 3\n</pre> if LOSS == \"mse\":     loss = tf.keras.losses.MeanSquaredError()     output_units = 1  if LOSS == \"ziln\":     loss = ltv.zero_inflated_lognormal_loss     output_units = 3 In\u00a0[\u00a0]: Copied! <pre>x_train, x_eval, y_train, y_eval = dnn_split(df)\nmodel = dnn_model(output_units)\n</pre> x_train, x_eval, y_train, y_eval = dnn_split(df) model = dnn_model(output_units) In\u00a0[\u00a0]: Copied! <pre>model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss)\n</pre> model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss) In\u00a0[\u00a0]: Copied! <pre>callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),\n    tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10),\n]\n</pre> callbacks = [     tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),     tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10), ] In\u00a0[\u00a0]: Copied! <pre>history = model.fit(\n    x=x_train,\n    y=y_train,\n    batch_size=2048,\n    epochs=200,\n    verbose=2,\n    callbacks=callbacks,\n    validation_data=(x_eval, y_eval),\n).history\n</pre> history = model.fit(     x=x_train,     y=y_train,     batch_size=2048,     epochs=200,     verbose=2,     callbacks=callbacks,     validation_data=(x_eval, y_eval), ).history In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    y_pred = model.predict(x=x_eval, batch_size=1024).flatten()\n\nif LOSS == \"ziln\":\n    logits = model.predict(x=x_eval, batch_size=1024)\n    y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten()\n</pre> if LOSS == \"mse\":     y_pred = model.predict(x=x_eval, batch_size=1024).flatten()  if LOSS == \"ziln\":     logits = model.predict(x=x_eval, batch_size=1024)     y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten() In\u00a0[\u00a0]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom torch.optim import NAdam\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from torch.optim.lr_scheduler import ReduceLROnPlateau from pytorch_widedeep.callbacks import EarlyStopping from torch.optim import NAdam In\u00a0[\u00a0]: Copied! <pre># CATEGORICAL_FEATURES\nNUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)]\nx_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"])\nx_train_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_train.items() if key not in [\"numeric\"]}\n)\n\nx_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"])\nx_eval_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_eval.items() if key not in [\"numeric\"]}\n)\n</pre> # CATEGORICAL_FEATURES NUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)] x_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"]) x_train_pyt_cat = pd.DataFrame(     {key: value for key, value in x_train.items() if key not in [\"numeric\"]} )  x_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"]) x_eval_pyt_cat = pd.DataFrame(     {key: value for key, value in x_eval.items() if key not in [\"numeric\"]} ) In\u00a0[\u00a0]: Copied! <pre>x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1)\nx_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1)\n</pre> x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1) x_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1) In\u00a0[\u00a0]: Copied! <pre>embed_input = [\n    (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES\n]\n</pre> embed_input = [     (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES ] In\u00a0[\u00a0]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=embed_input,\n    continuous_cols=NUMERICAL_FEATURES,\n    shared_embed=False,\n    scale=False,\n)\nX_tab_train = tab_preprocessor.fit_transform(x_train_pyt)\nX_tab_valid = tab_preprocessor.transform(x_eval_pyt)\nX_tab_test = tab_preprocessor.transform(x_eval_pyt)\n\n# target\ny_train = y_train\ny_valid = y_eval\ny_test = y_train\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\nX_test = {\"X_tab\": X_tab_test}\n\ndeeptabular = TabMlp(\n    mlp_hidden_dims=[128, 128, 64, 64],\n    column_idx=tab_preprocessor.column_idx,\n    embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=3)\n\ndeep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE)\ncallbacks = [EarlyStopping()]\ndeep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)\n\nobjective = \"ziln\"\n\ntrainer = Trainer(\n    model,\n    callbacks=callbacks,\n    lr_schedulers={\"deeptabular\": deep_sch},\n    objective=objective,\n    optimizers={\"deeptabular\": deep_opt},\n)\n\ntrainer.fit(\n    X_train=X_train,\n    X_val=X_val,\n    n_epochs=200,\n    batch_size=2048,\n)\n\ny_pred_pytorch = trainer.predict(X_test=X_test)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=embed_input,     continuous_cols=NUMERICAL_FEATURES,     shared_embed=False,     scale=False, ) X_tab_train = tab_preprocessor.fit_transform(x_train_pyt) X_tab_valid = tab_preprocessor.transform(x_eval_pyt) X_tab_test = tab_preprocessor.transform(x_eval_pyt)  # target y_train = y_train y_valid = y_eval y_test = y_train  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} X_test = {\"X_tab\": X_tab_test}  deeptabular = TabMlp(     mlp_hidden_dims=[128, 128, 64, 64],     column_idx=tab_preprocessor.column_idx,     embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, )  model = WideDeep(deeptabular=deeptabular, pred_dim=3)  deep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE) callbacks = [EarlyStopping()] deep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)  objective = \"ziln\"  trainer = Trainer(     model,     callbacks=callbacks,     lr_schedulers={\"deeptabular\": deep_sch},     objective=objective,     optimizers={\"deeptabular\": deep_opt}, )  trainer.fit(     X_train=X_train,     X_val=X_val,     n_epochs=200,     batch_size=2048, )  y_pred_pytorch = trainer.predict(X_test=X_test) In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>from sklearn.metrics import mean_squared_error\n\nmean_squared_error(y_pred, y_pred_pytorch)\n</pre> from sklearn.metrics import mean_squared_error  mean_squared_error(y_pred, y_pred_pytorch) In\u00a0[\u00a0]: Copied! <pre>unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0]\n</pre> unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0] In\u00a0[\u00a0]: Copied! <pre>num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs]\nnum_mailed\n</pre> num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs] num_mailed In\u00a0[\u00a0]: Copied! <pre>baseline_total_profit = np.sum(y_eval - 0.68)\nbaseline_total_profit\n</pre> baseline_total_profit = np.sum(y_eval - 0.68) baseline_total_profit In\u00a0[\u00a0]: Copied! <pre>total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs]\ntotal_profits\n</pre> total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs] total_profits In\u00a0[\u00a0]: Copied! <pre>gain = pd.DataFrame(\n    {\n        \"lorenz\": ltv.cumulative_true(y_eval, y_eval),\n        \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),\n        \"model\": ltv.cumulative_true(y_eval, y_pred),\n    }\n)\n</pre> gain = pd.DataFrame(     {         \"lorenz\": ltv.cumulative_true(y_eval, y_eval),         \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),         \"model\": ltv.cumulative_true(y_eval, y_pred),     } ) In\u00a0[\u00a0]: Copied! <pre>num_customers = np.float32(gain.shape[0])\ngain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers\n</pre> num_customers = np.float32(gain.shape[0]) gain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers In\u00a0[\u00a0]: Copied! <pre>ax = gain[\n    [\n        \"cumulative_customer\",\n        \"lorenz\",\n        \"baseline\",\n        \"model\",\n    ]\n].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)\n\nax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")\n\nax.set_xlabel(\"Cumulative Fraction of Customers\")\nax.set_xticks(np.arange(0, 1.1, 0.1))\nax.set_xlim((0, 1.0))\n\nax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\")\nax.set_yticks(np.arange(0, 1.1, 0.1))\nax.set_ylim((0, 1.05))\nax.set_title(\"Gain Chart\");\n</pre> ax = gain[     [         \"cumulative_customer\",         \"lorenz\",         \"baseline\",         \"model\",     ] ].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)  ax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")  ax.set_xlabel(\"Cumulative Fraction of Customers\") ax.set_xticks(np.arange(0, 1.1, 0.1)) ax.set_xlim((0, 1.0))  ax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\") ax.set_yticks(np.arange(0, 1.1, 0.1)) ax.set_ylim((0, 1.05)) ax.set_title(\"Gain Chart\"); In\u00a0[\u00a0]: Copied! <pre>gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]])\ngini\n</pre> gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]]) gini In\u00a0[\u00a0]: Copied! <pre>df_decile = ltv.decile_stats(y_eval, y_pred)\ndf_decile\n</pre> df_decile = ltv.decile_stats(y_eval, y_pred) df_decile In\u00a0[\u00a0]: Copied! <pre>ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)\n\nax.set_title(\"Decile Chart\")\nax.set_xlabel(\"Prediction bucket\")\nax.set_ylabel(\"Average bucket value\")\nax.legend([\"Label\", \"Prediction\"], loc=\"upper left\");\n</pre> ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)  ax.set_title(\"Decile Chart\") ax.set_xlabel(\"Prediction bucket\") ax.set_ylabel(\"Average bucket value\") ax.legend([\"Label\", \"Prediction\"], loc=\"upper left\"); In\u00a0[\u00a0]: Copied! <pre>def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:\n    \"\"\"Calculates spearmanr rank correlation coefficient.\n\n    See https://docs.scipy.org/doc/scipy/reference/stats.html.\n\n    Args:\n      x1: 1D array_like.\n      x2: 1D array_like.\n\n    Returns:\n      correlation: float.\n    \"\"\"\n    return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]\n\n\nspearman_corr = spearmanr(y_eval, y_pred)\nspearman_corr\n</pre> def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:     \"\"\"Calculates spearmanr rank correlation coefficient.      See https://docs.scipy.org/doc/scipy/reference/stats.html.      Args:       x1: 1D array_like.       x2: 1D array_like.      Returns:       correlation: float.     \"\"\"     return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]   spearman_corr = spearmanr(y_eval, y_pred) spearman_corr In\u00a0[\u00a0]: Copied! <pre>df_metrics = pd.DataFrame(\n    {\n        \"model\": MODEL,\n        \"loss_function\": LOSS,\n        \"train_loss\": history[\"loss\"][-1],\n        \"eval_loss\": history[\"val_loss\"][-1],\n        \"label_positive\": np.mean(y_eval &gt; 0),\n        \"label_mean\": y_eval.mean(),\n        \"pred_mean\": y_pred.mean(),\n        \"decile_mape\": df_decile[\"decile_mape\"].mean(),\n        \"baseline_gini\": gini[\"normalized\"][1],\n        \"gini\": gini[\"normalized\"][2],\n        \"spearman_corr\": spearman_corr,\n    },\n    index=[VERSION],\n)\n</pre> df_metrics = pd.DataFrame(     {         \"model\": MODEL,         \"loss_function\": LOSS,         \"train_loss\": history[\"loss\"][-1],         \"eval_loss\": history[\"val_loss\"][-1],         \"label_positive\": np.mean(y_eval &gt; 0),         \"label_mean\": y_eval.mean(),         \"pred_mean\": y_pred.mean(),         \"decile_mape\": df_decile[\"decile_mape\"].mean(),         \"baseline_gini\": gini[\"normalized\"][1],         \"gini\": gini[\"normalized\"][2],         \"spearman_corr\": spearman_corr,     },     index=[VERSION], ) In\u00a0[\u00a0]: Copied! <pre>for unit_cost, total_profit in zip(unit_costs, total_profits):\n    df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit\n</pre> for unit_cost, total_profit in zip(unit_costs, total_profits):     df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit In\u00a0[\u00a0]: Copied! <pre>df_metrics.T\n</pre> df_metrics.T In\u00a0[\u00a0]: Copied! <pre>output_path = OUTPUT_CSV_FOLDER\n</pre> output_path = OUTPUT_CSV_FOLDER In\u00a0[\u00a0]: Copied! <pre>if not os.path.isdir(output_path):\n    os.makedirs(output_path)\n</pre> if not os.path.isdir(output_path):     os.makedirs(output_path) In\u00a0[\u00a0]: Copied! <pre>output_file = os.path.join(\n    output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION)\n)\n</pre> output_file = os.path.join(     output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION) ) In\u00a0[\u00a0]: Copied! <pre>df_metrics.to_csv(output_file, index=False)\n</pre> df_metrics.to_csv(output_file, index=False)"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#zilnloss","title":"ZILNLoss\u00b6","text":"<p>[DISCLAIMER]</p> <p>Purpose of this notebook is to check if ZILNloss implemented originaly Keras give same results in pytorch-widedeep implemenatation</p>"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#kdd-cup-98-ltv-prediction","title":"KDD Cup 98 LTV Prediction\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#configs","title":"Configs\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#load-data","title":"Load data\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#label-distribution","title":"Label distribution\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#preprocess-features","title":"Preprocess features\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#vocab","title":"Vocab\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#indicator","title":"Indicator\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#numeric","title":"Numeric\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all","title":"All\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#traineval-split","title":"Train/eval split\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#model","title":"Model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#loss","title":"Loss\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#train","title":"Train\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#eval","title":"Eval\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#pytorch-widedeep-approach","title":"Pytorch-widedeep approach\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#appendix","title":"Appendix\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#total-profit","title":"Total Profit\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#gini-coefficient","title":"Gini Coefficient\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#calibration","title":"Calibration\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#rank-correlation","title":"Rank Correlation\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all-metrics-together","title":"All metrics together\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#save","title":"Save\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html","title":"13_model_uncertainty_prediction","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[2]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[3]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[5]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[6]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[7]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[8]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[9]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=1)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=1) model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre># # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n# f1 = F1_torchmetrics(average=None, num_classes=1)\n# recall = Recall_torchmetrics(average=None, num_classes=1)\n</pre> # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1) # f1 = F1_torchmetrics(average=None, num_classes=1) # recall = Recall_torchmetrics(average=None, num_classes=1) In\u00a0[11]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\nrecall = Recall(average=True)\nf1 = F1Score(average=False)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False) recall = Recall(average=True) f1 = F1Score(average=False) In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision, recall, f1],\n    verbose=1,\n)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision, recall, f1],     verbose=1, ) In\u00a0[13]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=3,\n    batch_size=50,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=3,     batch_size=50,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 130.75it/s, loss=0.187, metrics={'acc': 0.9214, 'prec': [0.9149], 'rec': 0.9318, 'f1': [0.9233]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 173.71it/s, loss=0.106, metrics={'acc': 0.9499, 'prec': [0.1435], 'rec': 0.938, 'f1': [0.249]}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 139.68it/s, loss=0.109, metrics={'acc': 0.9559, 'prec': [0.9537], 'rec': 0.9572, 'f1': [0.9554]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 177.53it/s, loss=0.0888, metrics={'acc': 0.9602, 'prec': [0.1755], 'rec': 0.9457, 'f1': [0.2961]}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 141.63it/s, loss=0.08, metrics={'acc': 0.9706, 'prec': [0.9648], 'rec': 0.9766, 'f1': [0.9707]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 165.36it/s, loss=0.0969, metrics={'acc': 0.9564, 'prec': [0.1636], 'rec': 0.9535, 'f1': [0.2792]}]</pre> <pre>Training time[s]: 0:00:10\n</pre> <pre>\n</pre> In\u00a0[14]: Copied! <pre>pd.DataFrame(trainer.history)\n</pre> pd.DataFrame(trainer.history) Out[14]: train_loss train_acc train_prec train_rec train_f1 val_loss val_acc val_prec val_rec val_f1 0 0.186707 0.921408 [0.9149412512779236] 0.931801 [0.9232940673828125] 0.106023 0.949914 [0.14353498816490173] 0.937984 [0.24897116422653198] 1 0.109498 0.955931 [0.9536514282226562] 0.957193 [0.9554190039634705] 0.088787 0.960206 [0.17553956806659698] 0.945736 [0.29611650109291077] 2 0.079979 0.970588 [0.9648183584213257] 0.976582 [0.9706646203994751] 0.096858 0.956432 [0.1635638326406479] 0.953488 [0.279228150844574] In\u00a0[15]: Copied! <pre>df_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:00&lt;00:00, 346.55it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13845,   731]))\n</pre> In\u00a0[16]: Copied! <pre>df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\n    \"Actual predicted values:\\n{}\".format(\n        np.unique(df_pred_unc[:, -1], return_counts=True)\n    )\n)\n</pre> df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(     \"Actual predicted values:\\n{}\".format(         np.unique(df_pred_unc[:, -1], return_counts=True)     ) ) <pre>predict_UncertaintyIter: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:03&lt;00:00,  3.25it/s]</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0.]), array([14576]))\n</pre> <pre>\n</pre> In\u00a0[17]: Copied! <pre>df_pred_unc\n</pre> df_pred_unc Out[17]: <pre>array([[9.98401165e-01, 1.59881881e-03, 0.00000000e+00],\n       [9.99941409e-01, 5.85634953e-05, 0.00000000e+00],\n       [9.97351170e-01, 2.64881272e-03, 0.00000000e+00],\n       ...,\n       [9.99494374e-01, 5.05603210e-04, 0.00000000e+00],\n       [9.99981642e-01, 1.83574630e-05, 0.00000000e+00],\n       [9.99996483e-01, 3.52600046e-06, 0.00000000e+00]])</pre>"},{"location":"examples/13_model_uncertainty_prediction.html#model-uncertainty-prediction","title":"Model Uncertainty prediction\u00b6","text":"<p>Note:</p> <p>This notebook extends the \"Custom DataLoader for Imbalanced dataset\" notebook</p>"},{"location":"examples/13_model_uncertainty_prediction.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#normal-prediction","title":"\"Normal\" prediction\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#prediction-using-uncertainty","title":"Prediction using uncertainty\u00b6","text":""},{"location":"examples/14_bayesian_models.html","title":"14_bayesian_models","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp\nfrom pytorch_widedeep.training.bayesian_trainer import BayesianTrainer\n</pre> import numpy as np import torch import pandas as pd  from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score  from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor from pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp from pytorch_widedeep.training.bayesian_trainer import BayesianTrainer <p>The first few things to do we know them very well, like with any other model described in any of the other notebooks</p> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"age_buckets\"] = pd.cut(\n    df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)\n)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"age_buckets\"] = pd.cut(     df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9) ) df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country age_buckets income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 3 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 4 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 0 In\u00a0[3]: Copied! <pre>train, test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> train, test = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>wide_cols = [\n    \"age_buckets\",\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native_country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n\ntarget = train[\"income_label\"].values\n</pre> wide_cols = [     \"age_buckets\",     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native_country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"]  target = train[\"income_label\"].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_tab = wide_preprocessor.fit_transform(train)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_tab = wide_preprocessor.fit_transform(train) In\u00a0[6]: Copied! <pre>model = BayesianWide(\n    input_dim=np.unique(X_tab).shape[0],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified\n)\n</pre> model = BayesianWide(     input_dim=np.unique(X_tab).shape[0],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified ) In\u00a0[7]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[8]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 124.32it/s, loss=163, metrics={'acc': 0.7813}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 238.67it/s, loss=141, metrics={'acc': 0.8219}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 132.81it/s, loss=140, metrics={'acc': 0.8285}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 190.16it/s, loss=140, metrics={'acc': 0.8298}]\n</pre> In\u00a0[9]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(train)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(train) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[10]: Copied! <pre>model = BayesianTabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    #     embed_continuous_method = \"standard\",\n    #     cont_embed_activation=\"leaky_relu\",\n    #     cont_embed_dim = 8,\n    mlp_hidden_dims=[128, 64],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,\n)\n</pre> model = BayesianTabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     #     embed_continuous_method = \"standard\",     #     cont_embed_activation=\"leaky_relu\",     #     cont_embed_dim = 8,     mlp_hidden_dims=[128, 64],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1, ) In\u00a0[11]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 28.74it/s, loss=2e+3, metrics={'acc': 0.8007}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 136.89it/s, loss=1.75e+3, metrics={'acc': 0.8418}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 29.41it/s, loss=1.73e+3, metrics={'acc': 0.8596}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 143.87it/s, loss=1.71e+3, metrics={'acc': 0.8569}]\n</pre> <p>These models are powerful beyond the success metrics because they give us a sense of uncertainty as we predict. Let's have a look</p> In\u00a0[13]: Copied! <pre>X_tab_test = tab_preprocessor.transform(test)\n</pre> X_tab_test = tab_preprocessor.transform(test) In\u00a0[14]: Copied! <pre>preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5)\n</pre> preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 33.92it/s]\n</pre> In\u00a0[15]: Copied! <pre>preds.shape\n</pre> preds.shape Out[15]: <pre>(5, 9769)</pre> <p>as we can see the prediction have shape <code>(5, 9769)</code>, one set of predictions each time we have internally run predict (i.e. sample the network and predict, defined by the parameter <code>n_samples</code>). This gives us an idea of how certain the model is about a certain prediction.</p> <p>Similarly, we could obtain the probabilities</p> In\u00a0[16]: Copied! <pre>probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5)\n</pre> probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 32.79it/s]\n</pre> In\u00a0[17]: Copied! <pre>probs.shape\n</pre> probs.shape Out[17]: <pre>(5, 9769, 2)</pre> <p>And we could see how the model performs each time we sampled the network</p> In\u00a0[18]: Copied! <pre>for p in preds:\n    print(accuracy_score(p, test[\"income_label\"].values))\n</pre> for p in preds:     print(accuracy_score(p, test[\"income_label\"].values)) <pre>0.8559729757395844\n0.8564847988535162\n0.8567918927218753\n0.8562800696079435\n0.8558706111167981\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/14_bayesian_models.html#the-bayesian-models","title":"The Bayesian Models\u00b6","text":"<p>Perhaps one of the most interesting functionality in the library is the access to full Bayesian models in almost exactly the same way one would use any of the other models in the library.</p> <p>Note however that the Bayesian models are ONLY available for tabular data and, at the moment, we do not support combining them to form a Wide and Deep model.</p> <p>The implementation in this library is based on the publication: Weight Uncertainty in Neural Networks, by Blundell et al., 2015. Code-wise, our implementation is inspired by a number of source:</p> <ol> <li>https://joshfeldman.net/WeightUncertainty/</li> <li>https://www.nitarshan.com/bayes-by-backprop/</li> <li>https://github.com/piEsposito/blitz-bayesian-deep-learning</li> <li>https://github.com/zackchase/mxnet-the-straight-dope/tree/master/chapter18_variational-methods-and-uncertainty</li> </ol> <p>The two Bayesian models available in the library are:</p> <ol> <li>BayesianWide: this is a linear model where the non-linearities are captured via crossed-columns</li> <li>BayesianMLP: this is a standard MLP that receives categorical embeddings and continuous cols (embedded or not) which are the passed through a series of dense layers. All parameters in the model are probabilistic.</li> </ol>"},{"location":"examples/14_bayesian_models.html#1-bayesianwide","title":"1. <code>BayesianWide</code>\u00b6","text":""},{"location":"examples/14_bayesian_models.html#2-bayesiantabmlp","title":"2. <code>BayesianTabMlp</code>\u00b6","text":""},{"location":"examples/15_DIR-LDS_and_FDS.html","title":"15_DIR-LDS_and_FDS","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom sklearn.metrics import mean_squared_error\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_california_housing\n\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib.pyplot as plt\nfrom scipy.ndimage import convolve1d\nfrom scipy.ndimage import gaussian_filter1d\nfrom scipy.signal.windows import triang\nfrom pytorch_widedeep.utils.deeptabular_utils import get_kernel_window, find_bin\nfrom pytorch_widedeep.models import fds_layer\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from sklearn.metrics import mean_squared_error from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_california_housing  from sklearn.model_selection import train_test_split  import matplotlib.pyplot as plt from scipy.ndimage import convolve1d from scipy.ndimage import gaussian_filter1d from scipy.signal.windows import triang from pytorch_widedeep.utils.deeptabular_utils import get_kernel_window, find_bin from pytorch_widedeep.models import fds_layer  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_california_housing(as_frame=True)\ndf.head()\n</pre> df = load_california_housing(as_frame=True) df.head() Out[2]: MedInc HouseAge AveRooms AveBedrms Population AveOccup Latitude Longitude MedHouseVal 0 8.3252 41.0 6.984127 1.023810 322.0 2.555556 37.88 -122.23 4.526 1 8.3014 21.0 6.238137 0.971880 2401.0 2.109842 37.86 -122.22 3.585 2 7.2574 52.0 8.288136 1.073446 496.0 2.802260 37.85 -122.24 3.521 3 5.6431 52.0 5.817352 1.073059 558.0 2.547945 37.85 -122.25 3.413 4 3.8462 52.0 6.281853 1.081081 565.0 2.181467 37.85 -122.25 3.422 In\u00a0[3]: Copied! <pre>ks = 5\nsigma = 2\nhalf_ks = (ks - 1) // 2\nbase_kernel = [0.0] * half_ks + [1.0] + [0.0] * half_ks\nkernel_window = gaussian_filter1d(base_kernel, sigma=sigma)\nplt.plot(kernel_window)\n</pre> ks = 5 sigma = 2 half_ks = (ks - 1) // 2 base_kernel = [0.0] * half_ks + [1.0] + [0.0] * half_ks kernel_window = gaussian_filter1d(base_kernel, sigma=sigma) plt.plot(kernel_window) Out[3]: <pre>[&lt;matplotlib.lines.Line2D at 0x16a0a43a0&gt;]</pre> In\u00a0[4]: Copied! <pre>lds = True\nkernel = \"gaussian\"\nks = 5\nsigma = 2\nreweight = \"sqrt\"\nY = df[\"MedHouseVal\"].values\nlds_y_max = None\nlds_y_min = None\ngranularity = 100\n\nfig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1)\n\ny_max = max(Y) if lds_y_max is None else lds_y_max\ny_min = min(Y) if lds_y_min is None else lds_y_min\nbin_edges = np.linspace(y_min, y_max, num=granularity, endpoint=True)\nvalue_dict = dict(zip(bin_edges[:-1], np.histogram(Y, bin_edges)[0]))\n\nax1.set_title(\"Histogram of values in the Y\")\nax1.bar(\n    value_dict.keys(),\n    value_dict.values(),\n    width=list(value_dict.keys())[1] - list(value_dict.keys())[0],\n)\n\nif reweight:\n    value_dict = dict(zip(value_dict.keys(), np.sqrt(list(value_dict.values()))))\n\nif kernel is not None:\n    lds_kernel_window = get_kernel_window(kernel, ks, sigma)\n    smoothed_values = convolve1d(\n        list(value_dict.values()), weights=lds_kernel_window, mode=\"constant\"\n    )\n    weigths = sum(smoothed_values) / (len(smoothed_values) * smoothed_values)\nelse:\n    values = list(value_dict.values())\n    weigths = sum(values) / (len(values) * values)  # type: ignore[operator]\nvalue_dict = dict(zip(value_dict.keys(), weigths))\n\nleft_bin_edges = find_bin(bin_edges, Y)\nweights = np.array([value_dict[edge] for edge in left_bin_edges], dtype=\"float32\")\n\n\nax2.set_title(\"Bar plot with inverse-balanced weights for each bin from histogram\")\nax2.bar(\n    value_dict.keys(),\n    value_dict.values(),\n    width=list(value_dict.keys())[1] - list(value_dict.keys())[0],\n)\nfig.tight_layout()\n</pre> lds = True kernel = \"gaussian\" ks = 5 sigma = 2 reweight = \"sqrt\" Y = df[\"MedHouseVal\"].values lds_y_max = None lds_y_min = None granularity = 100  fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1)  y_max = max(Y) if lds_y_max is None else lds_y_max y_min = min(Y) if lds_y_min is None else lds_y_min bin_edges = np.linspace(y_min, y_max, num=granularity, endpoint=True) value_dict = dict(zip(bin_edges[:-1], np.histogram(Y, bin_edges)[0]))  ax1.set_title(\"Histogram of values in the Y\") ax1.bar(     value_dict.keys(),     value_dict.values(),     width=list(value_dict.keys())[1] - list(value_dict.keys())[0], )  if reweight:     value_dict = dict(zip(value_dict.keys(), np.sqrt(list(value_dict.values()))))  if kernel is not None:     lds_kernel_window = get_kernel_window(kernel, ks, sigma)     smoothed_values = convolve1d(         list(value_dict.values()), weights=lds_kernel_window, mode=\"constant\"     )     weigths = sum(smoothed_values) / (len(smoothed_values) * smoothed_values) else:     values = list(value_dict.values())     weigths = sum(values) / (len(values) * values)  # type: ignore[operator] value_dict = dict(zip(value_dict.keys(), weigths))  left_bin_edges = find_bin(bin_edges, Y) weights = np.array([value_dict[edge] for edge in left_bin_edges], dtype=\"float32\")   ax2.set_title(\"Bar plot with inverse-balanced weights for each bin from histogram\") ax2.bar(     value_dict.keys(),     value_dict.values(),     width=list(value_dict.keys())[1] - list(value_dict.keys())[0], ) fig.tight_layout() In\u00a0[5]: Copied! <pre>features = torch.tensor(df.drop(columns=[\"MedHouseVal\"]).values)\nlabels = torch.tensor(np.vstack(df[\"MedHouseVal\"].values))\nFDS = fds_layer.FDSLayer(feature_dim=features.size(1))\n</pre> features = torch.tensor(df.drop(columns=[\"MedHouseVal\"]).values) labels = torch.tensor(np.vstack(df[\"MedHouseVal\"].values)) FDS = fds_layer.FDSLayer(feature_dim=features.size(1)) In\u00a0[6]: Copied! <pre>for epoch in range(3):\n    FDS.update_last_epoch_stats(epoch)\n    FDS.update_running_stats(torch.clone(features).detach(), labels, epoch)\n</pre> for epoch in range(3):     FDS.update_last_epoch_stats(epoch)     FDS.update_running_stats(torch.clone(features).detach(), labels, epoch) In\u00a0[7]: Copied! <pre>pd.DataFrame(FDS.running_mean_last_epoch.numpy()).iloc[:, 7].plot(\n    title=\"Running mean bina values for 'Longitude' feature\"\n);\n</pre> pd.DataFrame(FDS.running_mean_last_epoch.numpy()).iloc[:, 7].plot(     title=\"Running mean bina values for 'Longitude' feature\" ); In\u00a0[8]: Copied! <pre>pd.DataFrame(FDS.smoothed_mean_last_epoch.numpy()).iloc[:, 7].plot(\n    title=\"Smoothed mean bina values for 'Longitude' feature\"\n);\n</pre> pd.DataFrame(FDS.smoothed_mean_last_epoch.numpy()).iloc[:, 7].plot(     title=\"Smoothed mean bina values for 'Longitude' feature\" ); In\u00a0[9]: Copied! <pre>smoothed_features = FDS._smooth(torch.clone(features).detach(), labels, epoch).numpy()\nleft_bin_edges_indices = find_bin(\n    FDS.bin_edges, labels.squeeze(), ret_value=False\n).numpy()\ncontinuous_cols = df.drop(columns=[\"MedHouseVal\"]).columns.values.tolist()\n\ndf_w_bins = df.copy()\ndf_w_bins[\"MedHouseVal_bins\"] = left_bin_edges_indices\ndf_smoothed_w_bins = df_w_bins.copy()\ndf_smoothed_w_bins[continuous_cols] = smoothed_features\n</pre> smoothed_features = FDS._smooth(torch.clone(features).detach(), labels, epoch).numpy() left_bin_edges_indices = find_bin(     FDS.bin_edges, labels.squeeze(), ret_value=False ).numpy() continuous_cols = df.drop(columns=[\"MedHouseVal\"]).columns.values.tolist()  df_w_bins = df.copy() df_w_bins[\"MedHouseVal_bins\"] = left_bin_edges_indices df_smoothed_w_bins = df_w_bins.copy() df_smoothed_w_bins[continuous_cols] = smoothed_features In\u00a0[10]: Copied! <pre>df_w_bins[df_w_bins[\"MedHouseVal_bins\"] == 5][\"Longitude\"].plot(\n    title=\"Longitude feature values before calibration\"\n);\n</pre> df_w_bins[df_w_bins[\"MedHouseVal_bins\"] == 5][\"Longitude\"].plot(     title=\"Longitude feature values before calibration\" ); In\u00a0[11]: Copied! <pre>df_smoothed_w_bins[df_w_bins[\"MedHouseVal_bins\"] == 5][\"Longitude\"].plot(\n    title=\"Longitude feature values after calibration\\n(only slight change in values)\"\n);\n</pre> df_smoothed_w_bins[df_w_bins[\"MedHouseVal_bins\"] == 5][\"Longitude\"].plot(     title=\"Longitude feature values after calibration\\n(only slight change in values)\" ); In\u00a0[12]: Copied! <pre>df_train, df_valid = train_test_split(df, test_size=0.2, random_state=1)\ndf_valid, df_test = train_test_split(df_valid, test_size=0.5, random_state=1)\n</pre> df_train, df_valid = train_test_split(df, test_size=0.2, random_state=1) df_valid, df_test = train_test_split(df_valid, test_size=0.5, random_state=1) In\u00a0[13]: Copied! <pre>continuous_cols = df.drop(columns=[\"MedHouseVal\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"MedHouseVal\"]).columns.values.tolist() In\u00a0[14]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"MedHouseVal\"].values\ny_valid = df_valid[\"MedHouseVal\"].values\ny_test = df_test[\"MedHouseVal\"].values\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"MedHouseVal\"].values y_valid = df_valid[\"MedHouseVal\"].values y_test = df_test[\"MedHouseVal\"].values  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n  self._check_inputs(cat_embed_cols)\n</pre> In\u00a0[15]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, with_fds=True)\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, with_fds=True) model Out[15]: <pre>WideDeep(\n  (deeptabular): TabMlp(\n    (cont_norm): Identity()\n    (encoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=8, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (fds_layer): FDSLayer(\n    (pred_layer): Linear(in_features=100, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[16]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"huber\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\n        \"deeptabular\": XavierNormal,\n        \"fds_layer\": XavierNormal,\n        # \"FDS_dropout\": XavierNormal,\n        # \"pred_layer\": XavierNormal,\n    },\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[],\n    with_lds=True,\n    lds_kernel=\"gaussian\",\n    lds_ks=5,\n    lds_sigma=2,\n    lds_granularity=100,\n    lds_reweight=False,\n    lds_y_max=None,\n    lds_y_min=None,\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n\nprint(\n    f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\"\n)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"huber\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={         \"deeptabular\": XavierNormal,         \"fds_layer\": XavierNormal,         # \"FDS_dropout\": XavierNormal,         # \"pred_layer\": XavierNormal,     },     optimizers={\"deeptabular\": deep_opt},     metrics=[],     with_lds=True,     lds_kernel=\"gaussian\",     lds_ks=5,     lds_sigma=2,     lds_granularity=100,     lds_reweight=False,     lds_y_max=None,     lds_y_min=None, )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)  print(     f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\" ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 165.40it/s, loss=0.591, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 218.64it/s, loss=0.479, metrics={}]\nFDS update: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:00&lt;00:00, 366.86it/s]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:01&lt;00:00, 182.78it/s, loss=0.497, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 214.30it/s, loss=0.47, metrics={}]\nFDS update: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:00&lt;00:00, 350.68it/s]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:04&lt;00:00, 81.28it/s, loss=0.52, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 146.50it/s, loss=0.452, metrics={}]\nFDS update: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 122.88it/s]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:03&lt;00:00, 99.77it/s, loss=0.508, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 157.48it/s, loss=0.45, metrics={}]\nFDS update: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 147.92it/s]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:03&lt;00:00, 93.21it/s, loss=0.591, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 154.60it/s, loss=0.45, metrics={}]\nFDS update: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 145.73it/s]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 243.46it/s]</pre> <pre>test RMSE: 0.7417540528440087\n</pre> <pre>\n</pre> In\u00a0[17]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, fds=False)\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"huber\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[],\n    with_lds=True,\n    lds_kernel=\"gaussian\",\n    lds_ks=5,\n    lds_sigma=2,\n    lds_granularity=100,\n    lds_reweight=False,\n    lds_y_max=None,\n    lds_y_min=None,\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n\nprint(\n    f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\"\n)\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, fds=False)  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"huber\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[],     with_lds=True,     lds_kernel=\"gaussian\",     lds_ks=5,     lds_sigma=2,     lds_granularity=100,     lds_reweight=False,     lds_y_max=None,     lds_y_min=None, )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)  print(     f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\" ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 135.40it/s, loss=0.449, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 158.01it/s, loss=0.386, metrics={}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 137.79it/s, loss=0.377, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 156.84it/s, loss=0.399, metrics={}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 138.69it/s, loss=0.358, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 150.62it/s, loss=0.41, metrics={}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 131.96it/s, loss=0.339, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 146.01it/s, loss=0.321, metrics={}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 136.04it/s, loss=0.331, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 173.22it/s, loss=0.32, metrics={}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 296.77it/s]\n</pre> <pre>test RMSE: 0.6000006967500053\n</pre> In\u00a0[18]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, fds=False)\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"huber\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[],\n    with_lds=False,\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n\nprint(\n    f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\"\n)\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, fds=False)  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"huber\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[],     with_lds=False, )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)  print(     f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\" ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 129.54it/s, loss=0.445, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 137.64it/s, loss=0.427, metrics={}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 135.98it/s, loss=0.374, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 148.50it/s, loss=0.389, metrics={}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 127.72it/s, loss=0.359, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 147.63it/s, loss=0.383, metrics={}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 136.54it/s, loss=0.339, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 146.55it/s, loss=0.323, metrics={}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 331/331 [00:02&lt;00:00, 131.18it/s, loss=0.331, metrics={}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 174.87it/s, loss=0.318, metrics={}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 42/42 [00:00&lt;00:00, 287.25it/s]\n</pre> <pre>test RMSE: 0.6014019159826868\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_DIR-LDS_and_FDS.html#label-and-feature-distribution-smoothing-for-deep-imbalanced-regression","title":"Label and Feature Distribution Smoothing for Deep Imbalanced Regression\u00b6","text":""},{"location":"examples/15_DIR-LDS_and_FDS.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/15_DIR-LDS_and_FDS.html#load-dataset","title":"Load dataset\u00b6","text":""},{"location":"examples/15_DIR-LDS_and_FDS.html#effects-of-ks-and-sigma-paramaters-on-kernel-function","title":"Effects of ks and sigma paramaters on kernel function\u00b6","text":""},{"location":"examples/15_DIR-LDS_and_FDS.html#label-distribution-smoothing-visualization","title":"Label Distribution Smoothing - visualization\u00b6","text":"<ul> <li>visualization of  pytorch_widedeep.training._wd_dataset.WideDeepDataset._prepare_weights(...)</li> </ul> <p>Assign weight to each sample by following procedure:</p> <ol> <li>creating histogram from label values with nuber of bins = granularity 2.[OPTIONAL] reweighting label frequencies by sqrt 3.[OPTIONAL] smoothing label frequencies by convolution of kernel function window with frequencies list</li> <li>inverting values by n_samples / (n_classes * np.bincount(y)), see</li> <li>assigning weight to each sample from closest bin value</li> </ol>"},{"location":"examples/15_DIR-LDS_and_FDS.html#feature-distribution-smoothing","title":"Feature Distribution Smoothing\u00b6","text":"<p>We use dataset feature values in this example, but during the training process features tensors are the ouput of last layer before FDS layer.</p> <ul> <li>labels are np.vstack-ed to reflect normal training scenario</li> </ul>"},{"location":"examples/15_DIR-LDS_and_FDS.html#data-preparation","title":"Data preparation\u00b6","text":""},{"location":"examples/15_DIR-LDS_and_FDS.html#model-with-lds-fds","title":"Model with LDS &amp; FDS\u00b6","text":""},{"location":"examples/15_DIR-LDS_and_FDS.html#model-with-lds-only","title":"Model with LDS only\u00b6","text":""},{"location":"examples/15_DIR-LDS_and_FDS.html#model-without-fds-or-lds","title":"Model without FDS or LDS\u00b6","text":""},{"location":"examples/16_Self_Supervised_Pretraning_pt1.html","title":"16_Self-Supervised Pre-Training pt 1","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) In\u00a0[3]: Copied! <pre># one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>df_tr.head(2)\n</pre> df_tr.head(2) Out[4]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 9042 26 Local-gov 250551 HS-grad 9 Married-civ-spouse Craft-repair Own-child Black Male 0 0 40 United-States 0 25322 50 Private 34832 Bachelors 13 Married-civ-spouse Tech-support Husband White Male 15024 0 40 United-States 1 In\u00a0[5]: Copied! <pre># As always, we need to define which cols will be represented as embeddings\n# and which one will be continuous features\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n</pre> # As always, we need to define which cols will be represented as embeddings # and which one will be continuous features cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" In\u00a0[6]: Copied! <pre># We prepare the data to be passed to the model\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> # We prepare the data to be passed to the model tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>X_tab[:5]\n</pre> X_tab[:5] Out[7]: <pre>array([[ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1, 26, 40],\n       [ 2,  2,  1,  2,  2,  2,  1,  2,  1,  1, 50, 40],\n       [ 2,  1,  1,  3,  2,  2,  1,  1,  2,  1, 39, 46],\n       [ 2,  3,  2,  4,  1,  2,  2,  1,  1,  1, 17, 10],\n       [ 3,  4,  2,  1,  1,  2,  1,  1,  1,  1, 32, 20]])</pre> In\u00a0[8]: Copied! <pre># We define a model that will act as the encoder in the encoder/decoder\n# architecture. This could be any of: TabMlp, TabResnet or TabNet\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n</pre> # We define a model that will act as the encoder in the encoder/decoder # architecture. This could be any of: TabMlp, TabResnet or TabNet tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, ) In\u00a0[9]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[9]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[10]: Copied! <pre># If we do not pass a custom decoder, which is perfectly possible via the\n# decoder param,  the EncoderDecoderTrainer will automatically build a\n# decoder which will be the 'mirror' image of the encoder\nencoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp)\n</pre> # If we do not pass a custom decoder, which is perfectly possible via the # decoder param,  the EncoderDecoderTrainer will automatically build a # decoder which will be the 'mirror' image of the encoder encoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp) In\u00a0[11]: Copied! <pre># let's have a look to the encoder_decoder_model (aka ed_model)\nencoder_decoder_trainer.ed_model\n</pre> # let's have a look to the encoder_decoder_model (aka ed_model) encoder_decoder_trainer.ed_model Out[11]: <pre>EncoderDecoderModel(\n  (encoder): TabMlp(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n        (emb_layer_education): Embedding(17, 8, padding_idx=0)\n        (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n        (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n        (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n        (emb_layer_race): Embedding(6, 4, padding_idx=0)\n        (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n        (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n        (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n        (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.0, inplace=False)\n    )\n    (cont_norm): Identity()\n    (encoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=95, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (decoder): TabMlpDecoder(\n    (decoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=95, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (masker): RandomObfuscator()\n)</pre> <p>Ignoring the <code>masker</code>, which just...well...masks, the <code>ed_model</code> consists of:</p> <ol> <li>An encoder model that is a <code>TabMlp</code> model that is in itself comprised by an Embedding layer (or rather a collection of them, referred as <code>cat_and_cont_embed</code>) and an encoder (a simple MLP, referred as <code>encoder</code>)</li> <li>A decoder which is just an \"inverted\" MLP (referred as <code>decoder</code>)</li> </ol> In\u00a0[12]: Copied! <pre># And we just...pretrain\nencoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # And we just...pretrain encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.90it/s, loss=4.07]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 89.87it/s, loss=3.09]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.86it/s, loss=2.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.24it/s, loss=2.09]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.38it/s, loss=1.78]\n</pre> <p>At this point we have two options, we could either save the model for later use or we could continue to supervised training. The latter is rather simple, after running:</p> <pre>encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> <p>you just have to</p> <pre>model = WideDeep(deeptabular=tab_mlp)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\n</pre> <p>Let's say that in any case, we are 'decent' scientists/people and we want to save the model:</p> In\u00a0[13]: Copied! <pre>encoder_decoder_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\"\n)\n</pre> encoder_decoder_trainer.save(     path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\" ) <p>some time has passed...</p> In\u00a0[14]: Copied! <pre>encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\")\n</pre> encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\") <p>Now, AND THIS IS IMPORTANT We have loaded the encoder AND the decoder. To proceed to the supervised training we ONLY need the encoder</p> In\u00a0[15]: Copied! <pre>pretrained_encoder = encoder_decoder_model.encoder\n</pre> pretrained_encoder = encoder_decoder_model.encoder In\u00a0[16]: Copied! <pre>pretrained_encoder\n</pre> pretrained_encoder Out[16]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[17]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.04it/s, loss=0.374, metrics={'acc': 0.8253}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 85.63it/s, loss=0.324, metrics={'acc': 0.8491}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 87.56it/s, loss=0.301, metrics={'acc': 0.8608}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 73.38it/s, loss=0.29, metrics={'acc': 0.8655}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 78.68it/s, loss=0.284, metrics={'acc': 0.8686}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 173.02it/s]\n</pre> <pre>0.8730678677449074\n</pre> <p>As we mentioned before, we can also use a <code>TabResNet</code> or <code>TabNet</code> model and a custom decoder. Let's have a look:</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder\n</pre> from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder In\u00a0[19]: Copied! <pre>resnet_encoder = TabResnetEncoder(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 100, 100],\n)\n</pre> resnet_encoder = TabResnetEncoder(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     blocks_dims=[200, 100, 100], ) <p>let's have a look to the model</p> In\u00a0[20]: Copied! <pre>resnet_encoder\n</pre> resnet_encoder Out[20]: <pre>TabResnet(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n      (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      (block_0): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=False)\n          (1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=200, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n)</pre> <p>As we can see, the tensor we are trying to reconstruct, the embeddings, is of size <code>94</code> (this number is stored in the attribute: <code>esnet_encoder.cat_and_cont_embed.output_dim</code>), with that information we could build or own decoder as:</p> In\u00a0[21]: Copied! <pre># for all possible params see the docs\nresnet_decoder = TabResnetDecoder(\n    embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,\n    blocks_dims=[100, 100, 200],\n)\n</pre> # for all possible params see the docs resnet_decoder = TabResnetDecoder(     embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,     blocks_dims=[100, 100, 200], ) In\u00a0[22]: Copied! <pre>resnet_decoder\n</pre> resnet_decoder Out[22]: <pre>TabResnetDecoder(\n  (decoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (block_0): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=False)\n          (1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=100, out_features=200, bias=False)\n        (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=200, out_features=200, bias=False)\n        (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n  (reconstruction_layer): Linear(in_features=200, out_features=95, bias=False)\n)</pre> <p>and now:</p> In\u00a0[23]: Copied! <pre>ec_trainer = EncoderDecoderTrainer(\n    encoder=resnet_encoder,\n    decoder=resnet_decoder,\n    masked_prob=0.2,\n)\nec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> ec_trainer = EncoderDecoderTrainer(     encoder=resnet_encoder,     decoder=resnet_decoder,     masked_prob=0.2, ) ec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.89it/s, loss=1.52]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.78it/s, loss=0.81]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 39.82it/s, loss=0.56]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.73it/s, loss=0.417]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.24it/s, loss=0.329]\n</pre> In\u00a0[24]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=resnet_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=resnet_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.63it/s, loss=0.335, metrics={'acc': 0.8442}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.02it/s, loss=0.296, metrics={'acc': 0.864}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.91it/s, loss=0.283, metrics={'acc': 0.8687}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.00it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 51.95it/s, loss=0.272, metrics={'acc': 0.8732}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 120.15it/s]\n</pre> <pre>0.8725560446309756\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/16_Self_Supervised_Pretraning_pt1.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 1st of the two approaches (the 'TabNet approach'). For details on the second approach please see <code>16_Self_Supervised_Pretraning_pt2</code>.</p>"},{"location":"examples/16_Self_Supervised_Pretraning_pt1.html#self-supervision-for-non-transformer-based-models","title":"Self Supervision for non-transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns can have all different dimensions. In this library, these are: <code>TabMlp</code>, <code>TabResNet</code> and <code>TabNet</code></p> <p>As shown in Figure, this is an encoder-encoder approach where we learn to predict values in the incoming data that have been masked. However, as I mentioned before, our implementation is a bit different, and the masking occurs in th embedding space.</p> <p>Nonetheless, the code below illustrates how to use this encoder-decoder approach with <code>pytorch-widedeep</code></p>"},{"location":"examples/16_Self_Supervised_Pretraning_pt2.html","title":"16_Self-Supervised Pre-Training pt 2","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import WideDeep, FTTransformer\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import (\n    ContrastiveDenoisingTrainer,\n)\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import WideDeep, FTTransformer from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import (     ContrastiveDenoisingTrainer, ) In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n\n# one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    with_attention=True,\n    with_cls_token=True,  # this is optional\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True)  # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\"  tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     with_attention=True,     with_cls_token=True,  # this is optional ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[3]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    input_dim=32,\n    kv_compression_factor=0.5,\n    n_blocks=3,\n    n_heads=4,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     input_dim=32,     kv_compression_factor=0.5,     n_blocks=3,     n_heads=4, ) In\u00a0[4]: Copied! <pre># for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs.\n# Note that using these params involves some knowledge of the routine and the architecture of the model used\ncontrastive_denoising_trainer = ContrastiveDenoisingTrainer(\n    model=ft_transformer,\n    preprocessor=tab_preprocessor,\n)\ncontrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs. # Note that using these params involves some knowledge of the routine and the architecture of the model used contrastive_denoising_trainer = ContrastiveDenoisingTrainer(     model=ft_transformer,     preprocessor=tab_preprocessor, ) contrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:13&lt;00:00, 11.73it/s, loss=579]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.56it/s, loss=143]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.49it/s, loss=141]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 12.77it/s, loss=138]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 13.29it/s, loss=137]\n</pre> In\u00a0[5]: Copied! <pre>contrastive_denoising_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\"\n)\n</pre> contrastive_denoising_trainer.save(     path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\" ) <p>some time has passed</p> In\u00a0[6]: Copied! <pre># some time has passed, we load the model with torch as usual:\ncontrastive_denoising_model = torch.load(\n    \"pretrained_weights/contrastive_denoising_model.pt\"\n)\n</pre> # some time has passed, we load the model with torch as usual: contrastive_denoising_model = torch.load(     \"pretrained_weights/contrastive_denoising_model.pt\" ) <p>NOW, AND THIS IS IMPORTANT! We have loaded the entire contrastive, denoising model. To proceed to the supervised training we ONLY need the attention-based model, which is the 'model' attribute of the trainer, let's have a look</p> In\u00a0[7]: Copied! <pre>contrastive_denoising_model.model\n</pre> contrastive_denoising_model.model Out[7]: <pre>FTTransformer(\n  (cat_embed): SameSizeCatEmbeddings(\n    (embed): Embedding(323, 32, padding_idx=0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): Sequential(\n    (fttransformer_block0): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block1): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block2): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n  )\n)</pre> In\u00a0[8]: Copied! <pre>pretrained_model = contrastive_denoising_model.model\n</pre> pretrained_model = contrastive_denoising_model.model In\u00a0[9]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_model)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_model) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  # And, you know...we get a test metric X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.19it/s, loss=0.383, metrics={'acc': 0.8176}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.08it/s, loss=0.325, metrics={'acc': 0.8502}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.56it/s, loss=0.306, metrics={'acc': 0.8601}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.41it/s, loss=0.295, metrics={'acc': 0.8641}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:06&lt;00:00, 24.70it/s, loss=0.289, metrics={'acc': 0.8656}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 97.26it/s]</pre> <pre>0.8695874705701709\n</pre> <pre>\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/16_Self_Supervised_Pretraning_pt2.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 2nd of the two approaches (the 'SAINT approach'). For details on the 1st approach (the 'TabNet' approach) please see <code>16_Self_Supervised_Pretraning_pt1</code>.</p>"},{"location":"examples/16_Self_Supervised_Pretraning_pt2.html#self-supervision-transformer-based-models","title":"Self Supervision transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns have all the same dimensions. In this library, these are:</p> <ul> <li>TabTransformer</li> <li>FTTransformer</li> <li>SAINT</li> <li>TabFastFormer</li> </ul> <p>Note that there is one additional Transformer-based model, the <code>TabPerceiver</code>, however this is a \"particular\" model and at the moment we do not support self supervision for it, but it will come.</p> <p>Let see at one example using the <code>FTTransformer</code>.</p>"},{"location":"examples/17_Usign_a_custom_hugging_face_model.html","title":"17_Usign-a-custom-hugging-face-model","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport lightgbm as lgb\nfrom lightgbm import Dataset as lgbDataset\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.metrics import (\n    f1_score,\n    recall_score,\n    accuracy_score,\n    precision_score,\n    confusion_matrix,\n)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom torch import Tensor, nn\nfrom transformers import DistilBertModel, DistilBertTokenizer\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.utils import Tokenizer, LabelEncoder\nfrom pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.utils.fastai_transforms import (\n    fix_html,\n    spec_add_spaces,\n    rm_useless_spaces,\n)\n</pre> import numpy as np import torch import lightgbm as lgb from lightgbm import Dataset as lgbDataset from scipy.sparse import hstack, csr_matrix from sklearn.metrics import (     f1_score,     recall_score,     accuracy_score,     precision_score,     confusion_matrix, ) from sklearn.model_selection import train_test_split from sklearn.feature_extraction.text import TfidfVectorizer  from torch import Tensor, nn from transformers import DistilBertModel, DistilBertTokenizer from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.utils import Tokenizer, LabelEncoder from pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.utils.fastai_transforms import (     fix_html,     spec_add_spaces,     rm_useless_spaces, ) <p>Let's load the data and have a look:</p> In\u00a0[2]: Copied! <pre>df = load_womens_ecommerce(as_frame=True)\n\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> df = load_womens_ecommerce(as_frame=True)  df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 0 767 33 None Absolutely wonderful - silky and sexy and comf... 2 1 0 Initmates Intimate Intimates 1 1080 34 None Love this dress!  it's sooo pretty.  i happene... 3 1 4 General Dresses Dresses 2 1077 60 Some major design flaws I had such high hopes for this dress and reall... 1 0 0 General Dresses Dresses 3 1049 50 My favorite buy! I love, love, love this jumpsuit. it's fun, fl... 3 1 0 General Petite Bottoms Pants 4 847 47 Flattering shirt This shirt is very flattering to all due to th... 3 1 6 General Tops Blouses <p>So, we will use the <code>review_text</code> column to predict the <code>rating</code>. Later on, we will try to combine it with some other columns (like <code>division_name</code> and <code>age</code>) see if these help.</p> <p>Let's first have a look to the distribution of ratings</p> In\u00a0[4]: Copied! <pre>df.rating.value_counts()\n</pre> df.rating.value_counts() Out[4]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> <p>This shows that we could have perhaps grouped rating scores of 1, 2 and 3 into 1...but anyway, let's just move on with those 4 classes.</p> <p>We are not going to carry any hyperparameter optimization here, so, we will only need a train and a test set (i.e.  no need of a validation set for the example in this notebook)</p> In\u00a0[5]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating) <p>Let's see what we have to beat. What metrics would we obtain if we always predict the most common rating (3)?</p> In\u00a0[6]: Copied! <pre>most_common_pred = [train.rating.value_counts().index[0]] * len(test)\n\nmost_common_acc = accuracy_score(test.rating, most_common_pred)\nmost_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\")\n</pre> most_common_pred = [train.rating.value_counts().index[0]] * len(test)  most_common_acc = accuracy_score(test.rating, most_common_pred) most_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\") In\u00a0[7]: Copied! <pre>print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\")\n</pre> print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\") <pre>Accuracy: 0.553516143299425. F1 Score: 0.3944344218301668\n</pre> <p>ok, these are our \"baseline\" metrics.</p> <p>Let's start by using simply tf-idf + lightGBM</p> In\u00a0[8]: Copied! <pre># ?Tokenizer\n</pre> # ?Tokenizer In\u00a0[9]: Copied! <pre># this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.\n\n# When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm)\n# that can only be solved by using only one CPU\ntok = Tokenizer(n_cpus=1)\ntok_reviews_tr = tok.process_all(train.review_text.tolist())\ntok_reviews_te = tok.process_all(test.review_text.tolist())\n</pre> # this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.  # When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm) # that can only be solved by using only one CPU tok = Tokenizer(n_cpus=1) tok_reviews_tr = tok.process_all(train.review_text.tolist()) tok_reviews_te = tok.process_all(test.review_text.tolist()) In\u00a0[10]: Copied! <pre>vectorizer = TfidfVectorizer(\n    max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5\n)\n\nX_text_tr = vectorizer.fit_transform(tok_reviews_tr)\nX_text_te = vectorizer.transform(tok_reviews_te)\n</pre> vectorizer = TfidfVectorizer(     max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5 )  X_text_tr = vectorizer.fit_transform(tok_reviews_tr) X_text_te = vectorizer.transform(tok_reviews_te) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n  warnings.warn(\n</pre> In\u00a0[11]: Copied! <pre>X_text_tr\n</pre> X_text_tr Out[11]: <pre>&lt;18086x4566 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 884074 stored elements in Compressed Sparse Row format&gt;</pre> <p>We now move our matrices to lightGBM <code>Dataset</code> format</p> In\u00a0[12]: Copied! <pre>lgbtrain_text = lgbDataset(\n    X_text_tr,\n    train.rating.values,\n    free_raw_data=False,\n)\n\nlgbtest_text = lgbDataset(\n    X_text_te,\n    test.rating.values,\n    reference=lgbtrain_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_text = lgbDataset(     X_text_tr,     train.rating.values,     free_raw_data=False, )  lgbtest_text = lgbDataset(     X_text_te,     test.rating.values,     reference=lgbtrain_text,     free_raw_data=False, ) <p>and off we go. By the way, I think as we run the next cell, we should appreciate how fast lightGBM runs. Yes, the input is a sparse matrix, but still, trains on 18086x4566 in a matter of secs</p> In\u00a0[\u00a0]: Copied! <pre>lgb_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_text,\n    valid_sets=[lgbtest_text, lgbtrain_text],\n    valid_names=[\"test\", \"train\"],\n)\n</pre> lgb_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_text,     valid_sets=[lgbtest_text, lgbtrain_text],     valid_names=[\"test\", \"train\"], ) In\u00a0[14]: Copied! <pre>preds_text = lgb_text_model.predict(X_text_te)\npred_text_class = np.argmax(preds_text, 1)\n</pre> preds_text = lgb_text_model.predict(X_text_te) pred_text_class = np.argmax(preds_text, 1) In\u00a0[15]: Copied! <pre>acc_text = accuracy_score(lgbtest_text.label, pred_text_class)\nf1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\")\ncm_text = confusion_matrix(lgbtest_text.label, pred_text_class)\n</pre> acc_text = accuracy_score(lgbtest_text.label, pred_text_class) f1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\") cm_text = confusion_matrix(lgbtest_text.label, pred_text_class) In\u00a0[16]: Copied! <pre>print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\")\n</pre> print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\") <pre>LightGBM Accuracy: 0.6444051304732419. LightGBM F1 Score: 0.617154488246181\n</pre> In\u00a0[17]: Copied! <pre>print(f\"LightGBM Confusion Matrix: \\n {cm_text}\")\n</pre> print(f\"LightGBM Confusion Matrix: \\n {cm_text}\") <pre>LightGBM Confusion Matrix: \n [[ 199  135   61   79]\n [ 123  169  149  123]\n [  30   94  279  578]\n [  16   30  190 2267]]\n</pre> <p>Ok, so, with no hyperparameter optimization lightGBM gets an accuracy of 0.64 and a F1 score of 0.62. This is significantly better than predicting always the most popular.</p> <p>Let's see if in this implementation, some additional features, like <code>age</code> or <code>class_name</code> are of any help</p> In\u00a0[18]: Copied! <pre>tab_cols = [\n    \"age\",\n    \"division_name\",\n    \"department_name\",\n    \"class_name\",\n]\n\nfor tab_df in [train, test]:\n    for c in [\"division_name\", \"department_name\", \"class_name\"]:\n        tab_df[c] = tab_df[c].str.lower()\n        tab_df[c].fillna(\"missing\", inplace=True)\n</pre> tab_cols = [     \"age\",     \"division_name\",     \"department_name\",     \"class_name\", ]  for tab_df in [train, test]:     for c in [\"division_name\", \"department_name\", \"class_name\"]:         tab_df[c] = tab_df[c].str.lower()         tab_df[c].fillna(\"missing\", inplace=True) In\u00a0[19]: Copied! <pre># This is our LabelEncoder. A class that is designed to work with the models in this library but\n# can be used for general purposes\nle = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"])\ntrain_tab_le = le.fit_transform(train)\ntest_tab_le = le.transform(test)\n</pre> # This is our LabelEncoder. A class that is designed to work with the models in this library but # can be used for general purposes le = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"]) train_tab_le = le.fit_transform(train) test_tab_le = le.transform(test) In\u00a0[20]: Copied! <pre>train_tab_le.head()\n</pre> train_tab_le.head() Out[20]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 4541 836 35 None Bought this on sale in my reg size- 10. im 5'9... 2 1 2 1 1 1 18573 1022 25 Look like \"mom jeans\" Maybe i just have the wrong body type for thes... 1 0 0 2 2 2 1058 815 39 Ig brought me here Love the way this top layers under my jackets ... 2 1 0 1 1 1 12132 984 47 Runs small especially the arms I love this jacket. it's the prettiest and mos... 3 1 0 1 3 3 20756 1051 42 True red, true beauty. These pants are gorgeous--the fabric has a sat... 3 1 0 2 2 4 <p>let's for example have a look to the encodings for the categorical feature <code>class_name</code></p> In\u00a0[21]: Copied! <pre>le.encoding_dict[\"class_name\"]\n</pre> le.encoding_dict[\"class_name\"] Out[21]: <pre>{'blouses': 1,\n 'jeans': 2,\n 'jackets': 3,\n 'pants': 4,\n 'knits': 5,\n 'dresses': 6,\n 'skirts': 7,\n 'sweaters': 8,\n 'fine gauge': 9,\n 'legwear': 10,\n 'lounge': 11,\n 'shorts': 12,\n 'outerwear': 13,\n 'intimates': 14,\n 'swim': 15,\n 'trend': 16,\n 'sleep': 17,\n 'layering': 18,\n 'missing': 19,\n 'casual bottoms': 20,\n 'chemises': 21}</pre> In\u00a0[22]: Copied! <pre># tabular training and test sets\nX_tab_tr = csr_matrix(train_tab_le[tab_cols].values)\nX_tab_te = csr_matrix(test_tab_le[tab_cols].values)\n\n# text + tabular training and test sets\nX_tab_text_tr = hstack((X_tab_tr, X_text_tr))\nX_tab_text_te = hstack((X_tab_te, X_text_te))\n</pre> # tabular training and test sets X_tab_tr = csr_matrix(train_tab_le[tab_cols].values) X_tab_te = csr_matrix(test_tab_le[tab_cols].values)  # text + tabular training and test sets X_tab_text_tr = hstack((X_tab_tr, X_text_tr)) X_tab_text_te = hstack((X_tab_te, X_text_te)) In\u00a0[23]: Copied! <pre>X_tab_tr\n</pre> X_tab_tr Out[23]: <pre>&lt;18086x4 sparse matrix of type '&lt;class 'numpy.int64'&gt;'\n\twith 72344 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[24]: Copied! <pre>X_tab_text_tr\n</pre> X_tab_text_tr Out[24]: <pre>&lt;18086x4570 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 956418 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[25]: Copied! <pre>lgbtrain_tab_text = lgbDataset(\n    X_tab_text_tr,\n    train.rating.values,\n    categorical_feature=[0, 1, 2, 3],\n    free_raw_data=False,\n)\n\nlgbtest_tab_text = lgbDataset(\n    X_tab_text_te,\n    test.rating.values,\n    reference=lgbtrain_tab_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_tab_text = lgbDataset(     X_tab_text_tr,     train.rating.values,     categorical_feature=[0, 1, 2, 3],     free_raw_data=False, )  lgbtest_tab_text = lgbDataset(     X_tab_text_te,     test.rating.values,     reference=lgbtrain_tab_text,     free_raw_data=False, ) In\u00a0[26]: Copied! <pre>lgb_tab_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_tab_text,\n    valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],\n    valid_names=[\"test\", \"train\"],\n    verbose_eval=False,\n)\n</pre> lgb_tab_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_tab_text,     valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],     valid_names=[\"test\", \"train\"],     verbose_eval=False, ) <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2065: UserWarning: Using categorical_feature in Dataset.\n  _log_warning('Using categorical_feature in Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2068: UserWarning: categorical_feature in Dataset is overridden.\nNew categorical_feature is [0, 1, 2, 3]\n  _log_warning('categorical_feature in Dataset is overridden.\\n'\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/engine.py:239: UserWarning: 'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n  _log_warning(\"'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. \"\n</pre> <pre>[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.138280 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 143432\n[LightGBM] [Info] Number of data points in the train set: 18086, number of used features: 2289\n[LightGBM] [Info] Start training from score -2.255919\n[LightGBM] [Info] Start training from score -2.081545\n[LightGBM] [Info] Start training from score -1.528281\n[LightGBM] [Info] Start training from score -0.591354\n</pre> <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1780: UserWarning: Overriding the parameters from Reference Dataset.\n  _log_warning('Overriding the parameters from Reference Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1513: UserWarning: categorical_column in param dict is overridden.\n  _log_warning(f'{cat_alias} in param dict is overridden.')\n</pre> In\u00a0[27]: Copied! <pre>preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te)\npreds_tab_text_class = np.argmax(preds_tab_text, 1)\n\nacc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class)\nf1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\")\ncm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class)\n</pre> preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te) preds_tab_text_class = np.argmax(preds_tab_text, 1)  acc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class) f1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\") cm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class) In\u00a0[28]: Copied! <pre>print(\n    f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\"\n)\n</pre> print(     f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\" ) <pre>LightGBM text + tabular Accuracy: 0.6382131800088456. LightGBM text + tabular F1 Score: 0.6080251307242649\n</pre> In\u00a0[29]: Copied! <pre>print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\")\n</pre> print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\") <pre>LightGBM text + tabular Confusion Matrix:\n [[ 193  123   68   90]\n [ 123  146  157  138]\n [  37   90  272  582]\n [  16   37  175 2275]]\n</pre> <p>So, in this set up, the addition tabular columns do not help performance.</p> In\u00a0[30]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1\n)\n\nwd_X_text_tr = text_preprocessor.fit_transform(train)\nwd_X_text_te = text_preprocessor.transform(test)\n</pre> text_preprocessor = TextPreprocessor(     text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1 )  wd_X_text_tr = text_preprocessor.fit_transform(train) wd_X_text_te = text_preprocessor.transform(test) <pre>The vocabulary contains 4328 tokens\n</pre> In\u00a0[31]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n\n\nwd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4)\n</pre> basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], )   wd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4) In\u00a0[32]: Copied! <pre>wd_text_model\n</pre> wd_text_model Out[32]: <pre>WideDeep(\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[33]: Copied! <pre>text_trainer = Trainer(\n    wd_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> text_trainer = Trainer(     wd_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[34]: Copied! <pre>text_trainer.fit(\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> text_trainer.fit(     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.39it/s, loss=1.16, metrics={'acc': 0.5349, 'f1': 0.2011}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.35it/s, loss=0.964, metrics={'acc': 0.5827, 'f1': 0.3005}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.33it/s, loss=0.845, metrics={'acc': 0.6252, 'f1': 0.4133}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.99it/s, loss=0.765, metrics={'acc': 0.6575, 'f1': 0.4875}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.55it/s, loss=0.709, metrics={'acc': 0.6879, 'f1': 0.5423}]\n</pre> In\u00a0[35]: Copied! <pre>wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te)\nwd_pred_text_class = np.argmax(wd_pred_text, 1)\n</pre> wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te) wd_pred_text_class = np.argmax(wd_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 211.51it/s]\n</pre> In\u00a0[36]: Copied! <pre>wd_acc_text = accuracy_score(test.rating, wd_pred_text_class)\nwd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\")\nwd_cm_text = confusion_matrix(test.rating, wd_pred_text_class)\n</pre> wd_acc_text = accuracy_score(test.rating, wd_pred_text_class) wd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\") wd_cm_text = confusion_matrix(test.rating, wd_pred_text_class) In\u00a0[37]: Copied! <pre>print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\")\n</pre> print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\") <pre>Basic RNN Accuracy: 0.6076957098628926. Basic RNN F1 Score: 0.6017335854471788\n</pre> In\u00a0[38]: Copied! <pre>print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\")\n</pre> print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\") <pre>Basic RNN Confusion Matrix:\n [[ 327   76   62    9]\n [ 285  115  117   47]\n [ 131  122  315  413]\n [  42   69  401 1991]]\n</pre> <p>The performance is very similar to that of using simply tf-idf and lightgbm. Let see if adding tabular features helps when using <code>pytorch-widedeep</code></p> In\u00a0[39]: Copied! <pre># ?TabPreprocessor\n</pre> # ?TabPreprocessor In\u00a0[40]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)\n\nwd_X_tab_tr = tab_preprocessor.fit_transform(train)\nwd_X_tab_te = tab_preprocessor.transform(test)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)  wd_X_tab_tr = tab_preprocessor.fit_transform(train) wd_X_tab_te = tab_preprocessor.transform(test) In\u00a0[41]: Copied! <pre># ?TabMlp\n</pre> # ?TabMlp In\u00a0[42]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[43]: Copied! <pre>tab_model\n</pre> tab_model Out[43]: <pre>TabMlp(\n  (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_age): Embedding(78, 18, padding_idx=0)\n        (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n        (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n        (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.1, inplace=False)\n    )\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=35, out_features=100, bias=True)\n        (2): ReLU(inplace=True)\n      )\n      (dense_layer_1): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=100, out_features=50, bias=True)\n        (2): ReLU(inplace=True)\n      )\n    )\n  )\n)</pre> In\u00a0[44]: Copied! <pre>text_model = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n</pre> text_model = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], ) In\u00a0[45]: Copied! <pre>wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4)\n</pre> wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4) In\u00a0[46]: Copied! <pre>wd_tab_and_text_model\n</pre> wd_tab_and_text_model Out[46]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n        (cat_embed): DiffSizeCatEmbeddings(\n          (embed_layers): ModuleDict(\n            (emb_layer_age): Embedding(78, 18, padding_idx=0)\n            (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n            (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n            (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n          )\n          (embedding_dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=35, out_features=100, bias=True)\n            (2): ReLU(inplace=True)\n          )\n          (dense_layer_1): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=100, out_features=50, bias=True)\n            (2): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=50, out_features=4, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[47]: Copied! <pre>tab_and_text_trainer = Trainer(\n    wd_tab_and_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> tab_and_text_trainer = Trainer(     wd_tab_and_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[48]: Copied! <pre>tab_and_text_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> tab_and_text_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.04it/s, loss=1.13, metrics={'acc': 0.538, 'f1': 0.1911}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.28it/s, loss=0.936, metrics={'acc': 0.5887, 'f1': 0.3507}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.26it/s, loss=0.825, metrics={'acc': 0.6394, 'f1': 0.4545}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 51.33it/s, loss=0.757, metrics={'acc': 0.6696, 'f1': 0.5214}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 50.39it/s, loss=0.702, metrics={'acc': 0.6963, 'f1': 0.5654}]\n</pre> In\u00a0[49]: Copied! <pre>wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=wd_X_text_te\n)\nwd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1)\n</pre> wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=wd_X_text_te ) wd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 136.94it/s]\n</pre> In\u00a0[50]: Copied! <pre>wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class)\nwd_f1_tab_and_text = f1_score(\n    test.rating, wd_pred_tab_and_text_class, average=\"weighted\"\n)\nwd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class)\n</pre> wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class) wd_f1_tab_and_text = f1_score(     test.rating, wd_pred_tab_and_text_class, average=\"weighted\" ) wd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class) In\u00a0[51]: Copied! <pre>print(\n    f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\"\n)\nprint(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\")\n</pre> print(     f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\" ) print(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\") <pre>Basic RNN + Tabular  Accuracy: 0.6333480760725343. Basic RNN + TabularF1 Score: 0.6332310089593208\nBasic RNN + Tabular  Confusion Matrix:\n [[ 267  132   65   10]\n [ 198  168  159   39]\n [  57  113  410  401]\n [  12   58  414 2019]]\n</pre> <p>We are going to \"manually\" code the Tokenizer and the model and see how they can be used as part of the process along with the <code>pytorch-widedeep</code> library.</p> <p>Tokenizer:</p> In\u00a0[52]: Copied! <pre>class BertTokenizer(object):\n    def __init__(\n        self,\n        pretrained_tokenizer=\"distilbert-base-uncased\",\n        do_lower_case=True,\n        max_length=90,\n    ):\n        super(BertTokenizer, self).__init__()\n        self.pretrained_tokenizer = pretrained_tokenizer\n        self.do_lower_case = do_lower_case\n        self.max_length = max_length\n\n    def fit(self, texts):\n        self.tokenizer = DistilBertTokenizer.from_pretrained(\n            self.pretrained_tokenizer, do_lower_case=self.do_lower_case\n        )\n\n        return self\n\n    def transform(self, texts):\n        input_ids = []\n        for text in texts:\n            encoded_sent = self.tokenizer.encode_plus(\n                text=self._pre_rules(text),\n                add_special_tokens=True,\n                max_length=self.max_length,\n                padding=\"max_length\",\n                truncation=True,\n            )\n\n            input_ids.append(encoded_sent.get(\"input_ids\"))\n        return np.stack(input_ids)\n\n    def fit_transform(self, texts):\n        return self.fit(texts).transform(texts)\n\n    @staticmethod\n    def _pre_rules(text):\n        return fix_html(rm_useless_spaces(spec_add_spaces(text)))\n</pre> class BertTokenizer(object):     def __init__(         self,         pretrained_tokenizer=\"distilbert-base-uncased\",         do_lower_case=True,         max_length=90,     ):         super(BertTokenizer, self).__init__()         self.pretrained_tokenizer = pretrained_tokenizer         self.do_lower_case = do_lower_case         self.max_length = max_length      def fit(self, texts):         self.tokenizer = DistilBertTokenizer.from_pretrained(             self.pretrained_tokenizer, do_lower_case=self.do_lower_case         )          return self      def transform(self, texts):         input_ids = []         for text in texts:             encoded_sent = self.tokenizer.encode_plus(                 text=self._pre_rules(text),                 add_special_tokens=True,                 max_length=self.max_length,                 padding=\"max_length\",                 truncation=True,             )              input_ids.append(encoded_sent.get(\"input_ids\"))         return np.stack(input_ids)      def fit_transform(self, texts):         return self.fit(texts).transform(texts)      @staticmethod     def _pre_rules(text):         return fix_html(rm_useless_spaces(spec_add_spaces(text))) <p>Model:</p> In\u00a0[53]: Copied! <pre>class BertModel(nn.Module):\n    def __init__(\n        self,\n        model_name: str = \"distilbert-base-uncased\",\n        freeze_bert: bool = False,\n    ):\n        super(BertModel, self).__init__()\n\n        self.bert = DistilBertModel.from_pretrained(\n            model_name,\n        )\n\n        if freeze_bert:\n            for param in self.bert.parameters():\n                param.requires_grad = False\n\n    def forward(self, X_inp: Tensor) -&gt; Tensor:\n        attn_mask = (X_inp != 0).type(torch.int8)\n        outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)\n        return outputs[0][:, 0, :]\n\n    @property\n    def output_dim(self) -&gt; int:\n        # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must\n        # have a 'output_dim' property so the WideDeep class knows the incoming dims\n        # from the custom model. in this case, I hardcoded it\n        return 768\n</pre> class BertModel(nn.Module):     def __init__(         self,         model_name: str = \"distilbert-base-uncased\",         freeze_bert: bool = False,     ):         super(BertModel, self).__init__()          self.bert = DistilBertModel.from_pretrained(             model_name,         )          if freeze_bert:             for param in self.bert.parameters():                 param.requires_grad = False      def forward(self, X_inp: Tensor) -&gt; Tensor:         attn_mask = (X_inp != 0).type(torch.int8)         outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)         return outputs[0][:, 0, :]      @property     def output_dim(self) -&gt; int:         # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must         # have a 'output_dim' property so the WideDeep class knows the incoming dims         # from the custom model. in this case, I hardcoded it         return 768 In\u00a0[54]: Copied! <pre>bert_tokenizer = BertTokenizer()\nX_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist())\nX_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist())\n</pre> bert_tokenizer = BertTokenizer() X_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist()) X_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist()) <p>As I mentioned a number of times in the documentation and examples, <code>pytorch-widedeep</code> is designed for flexibility. For any of the data modes (tabular, text and images) there are available components/models in the library. However, the user can choose to use any model they want with the only requirement that such model must have a <code>output_dim</code> property.</p> <p>With that in mind, the <code>BertModel</code> class defined above can be used by <code>pytorch-widedeep</code> as any other of the internal components. In other words, simply...pass it to the <code>WideDeep</code> class. In this case we are going to add a FC-head as part of the classifier.</p> In\u00a0[55]: Copied! <pre>bert_model = BertModel(freeze_bert=True)\nwd_bert_model = WideDeep(\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> bert_model = BertModel(freeze_bert=True) wd_bert_model = WideDeep(     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) <pre>Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n</pre> In\u00a0[56]: Copied! <pre>wd_bert_model\n</pre> wd_bert_model Out[56]: <pre>WideDeep(\n  (deeptext): BertModel(\n    (bert): DistilBertModel(\n      (embeddings): Embeddings(\n        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n        (position_embeddings): Embedding(512, 768)\n        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (transformer): Transformer(\n        (layer): ModuleList(\n          (0): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (1): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (2): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (3): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (4): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (5): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n        )\n      )\n    )\n  )\n  (deephead): Sequential(\n    (0): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=768, out_features=256, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=256, out_features=128, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_2): Sequential(\n          (0): Linear(in_features=128, out_features=64, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n    (1): Linear(in_features=64, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[57]: Copied! <pre>wd_bert_trainer = Trainer(\n    wd_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n\nwd_bert_trainer.fit(\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_bert_trainer = Trainer(     wd_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work )  wd_bert_trainer.fit(     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.68it/s, loss=0.968, metrics={'acc': 0.5879, 'f1': 0.3591}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.63it/s, loss=0.884, metrics={'acc': 0.6178, 'f1': 0.4399}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.55it/s, loss=0.87, metrics={'acc': 0.6234, 'f1': 0.4527}]\n</pre> In\u00a0[58]: Copied! <pre>wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te)\nwd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1)\n</pre> wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te) wd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.97it/s]\n</pre> In\u00a0[59]: Copied! <pre>wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class)\nwd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\")\nwd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class)\n</pre> wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class) wd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\") wd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class) In\u00a0[60]: Copied! <pre>print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\")\nprint(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\")\n</pre> print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\") print(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\") <pre>Distilbert Accuracy: 0.6326846528084918. Distilbert F1 Score: 0.5796652991272998\nDistilbert Confusion Matrix:\n [[ 287   75   22   90]\n [ 197  136   62  169]\n [  68  119  123  671]\n [  40   64   84 2315]]\n</pre> <p>Now, adding a tabular model follows the exact same process as the one described in section 2.</p> In\u00a0[61]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[62]: Copied! <pre>wd_tab_bert_model = WideDeep(\n    deeptabular=tab_model,\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> wd_tab_bert_model = WideDeep(     deeptabular=tab_model,     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) In\u00a0[63]: Copied! <pre>wd_tab_bert_trainer = Trainer(\n    wd_tab_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> wd_tab_bert_trainer = Trainer(     wd_tab_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[64]: Copied! <pre>wd_tab_bert_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_tab_bert_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.15it/s, loss=0.974, metrics={'acc': 0.5838, 'f1': 0.3404}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.38it/s, loss=0.885, metrics={'acc': 0.618, 'f1': 0.4378}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.40it/s, loss=0.868, metrics={'acc': 0.6252, 'f1': 0.4575}]\n</pre> In\u00a0[65]: Copied! <pre>wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=X_bert_te\n)\nwd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1)\n</pre> wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=X_bert_te ) wd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.32it/s]\n</pre> In\u00a0[66]: Copied! <pre>wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class)\nwd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\")\nwd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class)\n</pre> wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class) wd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\") wd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class) In\u00a0[67]: Copied! <pre>print(\n    f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\"\n)\nprint(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\")\n</pre> print(     f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\" ) print(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\") <pre>Distilbert + Tabular Accuracy: 0.6242812914639541. Distilbert+ Tabular F1 Score: 0.5508351761564895\nDistilbert + Tabular Confusion Matrix:\n [[ 297   56   11  110]\n [ 229   91   38  206]\n [  86   90   71  734]\n [  49   48   42 2364]]\n</pre>"},{"location":"examples/17_Usign_a_custom_hugging_face_model.html#using-a-hugginface-model","title":"Using a Hugginface model\u00b6","text":"<p>In this notebook we will show how to use an \"external\" Hugginface model along with any other model in the libray. In particular we will show how to combine it with a tabular DL model.</p> <p>Since we are here, we will also compare the performance of a few models on a text classification problem.</p> <p>The notebook will go as follows:</p> <ol> <li>Text classification using tf-idf + LightGBM</li> <li>Text classification using a basic RNN</li> <li>Text classification using Distilbert</li> </ol> <p>In all 3 cases we will add some tabular features to see if these help.</p> <p>In general, I would not pay much attention to the results since I have placed no effort in getting the best possible results (i.e. no hyperparameter optimization or trying different architectures, for example).</p> <p>Let's go</p>"},{"location":"examples/17_Usign_a_custom_hugging_face_model.html#1-text-classification-using-tf-idf-lightgbm","title":"1. Text classification using tf-idf + LightGBM\u00b6","text":""},{"location":"examples/17_Usign_a_custom_hugging_face_model.html#2-text-classification-using-pytorch-widedeeps-built-in-models-a-basic-rnn","title":"2. Text classification using pytorch-widedeep's built-in models (a basic RNN)\u00b6","text":"<p>Moving on now to fully using <code>pytorch-widedeep</code> in this dataset, let's have a look on how one could use a simple RNN to predict the ratings with the library.</p>"},{"location":"examples/17_Usign_a_custom_hugging_face_model.html#3-text-classification-using-a-hugginface-model-as-a-custom-model-in-pytorch-widedeeps","title":"3. Text classification using a Hugginface model as a custom model in pytorch-widedeep's\u00b6","text":""},{"location":"examples/18_feature_importance_via_attention_weights.html","title":"18_feature_importance_via_attention_weights","text":"In\u00a0[1]: Copied! <pre>import torch\n\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import torch  import numpy as np from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score   from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep from pytorch_widedeep.callbacks import EarlyStopping from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre># use_cuda = torch.cuda.is_available()\ndf = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\ntarget_colname = \"income_label\"\n</pre> # use_cuda = torch.cuda.is_available() df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True) target_colname = \"income_label\" In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_embed_cols = []\nfor col in df.columns:\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:\n        cat_embed_cols.append(col)\n</pre> cat_embed_cols = [] for col in df.columns:     if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:         cat_embed_cols.append(col) In\u00a0[5]: Copied! <pre># all cols will be categorical\nassert len(cat_embed_cols) == df.shape[1] - 1\n</pre> # all cols will be categorical assert len(cat_embed_cols) == df.shape[1] - 1 In\u00a0[6]: Copied! <pre>train, test = train_test_split(\n    df, test_size=0.1, random_state=1, stratify=df[[target_colname]]\n)\n</pre> train, test = train_test_split(     df, test_size=0.1, random_state=1, stratify=df[[target_colname]] ) In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True) In\u00a0[8]: Copied! <pre>X_tab_train = tab_preprocessor.fit_transform(train)\nX_tab_test = tab_preprocessor.transform(test)\ntarget = train[target_colname].values\n</pre> X_tab_train = tab_preprocessor.fit_transform(train) X_tab_test = tab_preprocessor.transform(test) target = train[target_colname].values In\u00a0[9]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    embed_continuous_method=\"standard\",\n    input_dim=8,\n    n_heads=2,\n    n_blocks=1,\n    attn_dropout=0.1,\n    transformer_activation=\"relu\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     embed_continuous_method=\"standard\",     input_dim=8,     n_heads=2,     n_blocks=1,     attn_dropout=0.1,     transformer_activation=\"relu\", ) In\u00a0[10]: Copied! <pre>model = WideDeep(deeptabular=tab_transformer)\n</pre> model = WideDeep(deeptabular=tab_transformer) In\u00a0[11]: Copied! <pre>optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[12]: Copied! <pre>lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[13]: Copied! <pre>early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[14]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=optimizer,\n    lr_schedulers=lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[early_stopping],\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=optimizer,     lr_schedulers=lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[early_stopping],     metrics=[Accuracy], ) <p>The feature importances will be computed after training, using a sample of the training dataset of size <code>feature_importance_sample_size</code></p> In\u00a0[15]: Copied! <pre>trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.80it/s, loss=0.334, metrics={'acc': 0.847}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.34it/s, loss=0.294, metrics={'acc': 0.8669}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.02it/s, loss=0.293, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.03it/s, loss=0.283, metrics={'acc': 0.8678}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 87.69it/s, loss=0.282, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.22it/s, loss=0.279, metrics={'acc': 0.8717}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.24it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.29it/s, loss=0.277, metrics={'acc': 0.8731}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.76it/s, loss=0.275, metrics={'acc': 0.8727}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.80it/s, loss=0.276, metrics={'acc': 0.8727}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.78it/s, loss=0.273, metrics={'acc': 0.873}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.43it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.271, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.14it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.29it/s, loss=0.271, metrics={'acc': 0.875}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.46it/s, loss=0.276, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.10it/s, loss=0.27, metrics={'acc': 0.8761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.49it/s, loss=0.275, metrics={'acc': 0.8728}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 70.40it/s, loss=0.269, metrics={'acc': 0.8747}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.47it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.83it/s, loss=0.268, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.03it/s, loss=0.274, metrics={'acc': 0.873}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.86it/s, loss=0.267, metrics={'acc': 0.8743}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.61it/s, loss=0.274, metrics={'acc': 0.8734}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.39it/s, loss=0.267, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.05it/s, loss=0.275, metrics={'acc': 0.8717}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.36it/s, loss=0.265, metrics={'acc': 0.8767}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 109.60it/s, loss=0.276, metrics={'acc': 0.8747}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.34it/s, loss=0.264, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.55it/s, loss=0.276, metrics={'acc': 0.8706}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.264, metrics={'acc': 0.8777}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.08it/s, loss=0.275, metrics={'acc': 0.8753}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.96it/s, loss=0.263, metrics={'acc': 0.877}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.83it/s, loss=0.277, metrics={'acc': 0.8739}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.38it/s, loss=0.263, metrics={'acc': 0.8779}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.18it/s, loss=0.278, metrics={'acc': 0.8714}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.61it/s, loss=0.261, metrics={'acc': 0.8784}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.19it/s, loss=0.278, metrics={'acc': 0.8712}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.43it/s, loss=0.261, metrics={'acc': 0.8791}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.87it/s, loss=0.28, metrics={'acc': 0.873}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.97it/s, loss=0.26, metrics={'acc': 0.8787}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 107.50it/s, loss=0.279, metrics={'acc': 0.8732}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.76it/s, loss=0.253, metrics={'acc': 0.8816}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.11it/s, loss=0.279, metrics={'acc': 0.8707}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.92it/s, loss=0.252, metrics={'acc': 0.8828}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.14it/s, loss=0.28, metrics={'acc': 0.8711}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.04it/s, loss=0.252, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 98.36it/s, loss=0.28, metrics={'acc': 0.8708}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.31it/s, loss=0.251, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.97it/s, loss=0.281, metrics={'acc': 0.8709}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.63it/s, loss=0.25, metrics={'acc': 0.8834}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.07it/s, loss=0.281, metrics={'acc': 0.8698}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.37it/s, loss=0.251, metrics={'acc': 0.884}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.75it/s, loss=0.281, metrics={'acc': 0.87}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.19it/s, loss=0.25, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.282, metrics={'acc': 0.8699}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.19it/s, loss=0.25, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.11it/s, loss=0.282, metrics={'acc': 0.8695}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.57it/s, loss=0.249, metrics={'acc': 0.8839}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.64it/s, loss=0.283, metrics={'acc': 0.8689}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.55it/s, loss=0.249, metrics={'acc': 0.8846}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.10it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.56it/s, loss=0.248, metrics={'acc': 0.8841}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.45it/s, loss=0.284, metrics={'acc': 0.8687}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.06it/s, loss=0.248, metrics={'acc': 0.8848}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 129.26it/s, loss=0.284, metrics={'acc': 0.8689}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.53it/s, loss=0.248, metrics={'acc': 0.8854}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.61it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.78it/s, loss=0.248, metrics={'acc': 0.8853}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.31it/s, loss=0.283, metrics={'acc': 0.8694}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.51it/s, loss=0.248, metrics={'acc': 0.8863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.94it/s, loss=0.283, metrics={'acc': 0.8693}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.35it/s, loss=0.247, metrics={'acc': 0.8844}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.77it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.62it/s, loss=0.248, metrics={'acc': 0.8837}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.62it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.82it/s, loss=0.248, metrics={'acc': 0.8842}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.64it/s, loss=0.283, metrics={'acc': 0.8695}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.86it/s, loss=0.247, metrics={'acc': 0.8855}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.34it/s, loss=0.283, metrics={'acc': 0.8692}]\n</pre> <pre>Best Epoch: 10. Best val_loss: 0.27451\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[16]: Copied! <pre>trainer.feature_importance\n</pre> trainer.feature_importance Out[16]: <pre>{'age': 0.09718182,\n 'workclass': 0.090637445,\n 'education': 0.08910798,\n 'marital_status': 0.08971319,\n 'occupation': 0.12546304,\n 'relationship': 0.086381145,\n 'race': 0.050686445,\n 'gender': 0.05116429,\n 'capital_gain': 0.08165918,\n 'capital_loss': 0.07702667,\n 'hours_per_week': 0.08205996,\n 'native_country': 0.07891885}</pre> In\u00a0[17]: Copied! <pre>preds = trainer.predict(X_tab=X_tab_test)\n</pre> preds = trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 199.63it/s]\n</pre> In\u00a0[18]: Copied! <pre>accuracy_score(preds, test.income_label)\n</pre> accuracy_score(preds, test.income_label) Out[18]: <pre>0.8685772773797339</pre> In\u00a0[19]: Copied! <pre>test.reset_index(drop=True, inplace=True)\n</pre> test.reset_index(drop=True, inplace=True) In\u00a0[20]: Copied! <pre>test[test.income_label == 0].head(1)\n</pre> test[test.income_label == 0].head(1) Out[20]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 26 Private Some-college Never-married Exec-managerial Not-in-family White Male 0 0 60 United-States 0 In\u00a0[21]: Copied! <pre>test[test.income_label == 1].head(1)\n</pre> test[test.income_label == 1].head(1) Out[21]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 3 36 Local-gov Doctorate Married-civ-spouse Prof-specialty Husband White Male 0 1887 50 United-States 1 <p>To get the feature importance of a test dataset, simply use the <code>explain</code> method</p> In\u00a0[22]: Copied! <pre>feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False)\n</pre> feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False) In\u00a0[23]: Copied! <pre>list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])])\n</pre> list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])]) Out[23]: <pre>['hours_per_week',\n 'education',\n 'relationship',\n 'occupation',\n 'workclass',\n 'capital_gain',\n 'native_country',\n 'marital_status',\n 'capital_loss',\n 'age',\n 'race',\n 'gender']</pre> In\u00a0[24]: Copied! <pre>list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])])\n</pre> list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])]) Out[24]: <pre>['age',\n 'capital_loss',\n 'hours_per_week',\n 'marital_status',\n 'native_country',\n 'relationship',\n 'race',\n 'education',\n 'occupation',\n 'capital_gain',\n 'gender',\n 'workclass']</pre> <p>We could do the same with the <code>ContextAttentionMLP</code></p> In\u00a0[25]: Copied! <pre>context_attn_mlp = ContextAttentionMLP(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.0,\n    input_dim=16,\n    attn_dropout=0.1,\n    attn_activation=\"relu\",\n)\n</pre> context_attn_mlp = ContextAttentionMLP(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.0,     input_dim=16,     attn_dropout=0.1,     attn_activation=\"relu\", ) In\u00a0[26]: Copied! <pre>mlp_model = WideDeep(deeptabular=context_attn_mlp)\n</pre> mlp_model = WideDeep(deeptabular=context_attn_mlp) In\u00a0[27]: Copied! <pre>mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[28]: Copied! <pre>mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    mlp_optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     mlp_optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[29]: Copied! <pre>mlp_early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> mlp_early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[30]: Copied! <pre>mlp_trainer = Trainer(\n    mlp_model,\n    objective=\"binary\",\n    optimizers=mlp_optimizer,\n    lr_schedulers=mlp_lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[mlp_early_stopping],\n    metrics=[Accuracy],\n)\n</pre> mlp_trainer = Trainer(     mlp_model,     objective=\"binary\",     optimizers=mlp_optimizer,     lr_schedulers=mlp_lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[mlp_early_stopping],     metrics=[Accuracy], ) In\u00a0[31]: Copied! <pre>mlp_trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> mlp_trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.405, metrics={'acc': 0.8094}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.26it/s, loss=0.309, metrics={'acc': 0.8583}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.70it/s, loss=0.332, metrics={'acc': 0.8447}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.36it/s, loss=0.293, metrics={'acc': 0.8646}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.42it/s, loss=0.319, metrics={'acc': 0.8505}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.05it/s, loss=0.293, metrics={'acc': 0.8654}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.00it/s, loss=0.312, metrics={'acc': 0.8554}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.49it/s, loss=0.291, metrics={'acc': 0.8661}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.50it/s, loss=0.308, metrics={'acc': 0.8583}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.48it/s, loss=0.287, metrics={'acc': 0.8669}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.84it/s, loss=0.303, metrics={'acc': 0.8605}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 128.70it/s, loss=0.288, metrics={'acc': 0.8673}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.93it/s, loss=0.301, metrics={'acc': 0.8597}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 121.47it/s, loss=0.298, metrics={'acc': 0.8628}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.56it/s, loss=0.3, metrics={'acc': 0.8592}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.84it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.17it/s, loss=0.298, metrics={'acc': 0.8619}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.32it/s, loss=0.28, metrics={'acc': 0.8716}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.13it/s, loss=0.297, metrics={'acc': 0.8615}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.50it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.54it/s, loss=0.293, metrics={'acc': 0.8641}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.57it/s, loss=0.284, metrics={'acc': 0.867}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.92it/s, loss=0.293, metrics={'acc': 0.863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.42it/s, loss=0.282, metrics={'acc': 0.8701}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.61it/s, loss=0.293, metrics={'acc': 0.8635}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.56it/s, loss=0.276, metrics={'acc': 0.8719}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.92it/s, loss=0.29, metrics={'acc': 0.8633}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.06it/s, loss=0.286, metrics={'acc': 0.8669}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.83it/s, loss=0.291, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.88it/s, loss=0.282, metrics={'acc': 0.8677}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.74it/s, loss=0.29, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.85it/s, loss=0.285, metrics={'acc': 0.8672}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.99it/s, loss=0.29, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.53it/s, loss=0.282, metrics={'acc': 0.8681}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.22it/s, loss=0.288, metrics={'acc': 0.8651}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.89it/s, loss=0.288, metrics={'acc': 0.8676}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.85it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.26it/s, loss=0.284, metrics={'acc': 0.8662}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.80it/s, loss=0.289, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.44it/s, loss=0.281, metrics={'acc': 0.8703}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.72it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.04it/s, loss=0.285, metrics={'acc': 0.8648}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.86it/s, loss=0.289, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.75it/s, loss=0.282, metrics={'acc': 0.8666}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.289, metrics={'acc': 0.8668}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.86it/s, loss=0.282, metrics={'acc': 0.8724}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.93it/s, loss=0.288, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.69it/s, loss=0.285, metrics={'acc': 0.8656}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.99it/s, loss=0.284, metrics={'acc': 0.8671}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.27it/s, loss=0.277, metrics={'acc': 0.8707}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.86it/s, loss=0.282, metrics={'acc': 0.8686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.94it/s, loss=0.276, metrics={'acc': 0.8712}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.86it/s, loss=0.283, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.11it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.43it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.64it/s, loss=0.277, metrics={'acc': 0.8712}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.39it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.83it/s, loss=0.277, metrics={'acc': 0.872}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.93it/s, loss=0.28, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.07it/s, loss=0.275, metrics={'acc': 0.8714}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.35it/s, loss=0.281, metrics={'acc': 0.8697}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.68it/s, loss=0.276, metrics={'acc': 0.872}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.72it/s, loss=0.28, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.50it/s, loss=0.276, metrics={'acc': 0.8709}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.28, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.07it/s, loss=0.277, metrics={'acc': 0.8709}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.58it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.07it/s, loss=0.274, metrics={'acc': 0.8719}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.03it/s, loss=0.28, metrics={'acc': 0.8687}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.15it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.279, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.10it/s, loss=0.278, metrics={'acc': 0.8705}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.30it/s, loss=0.279, metrics={'acc': 0.869}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.75it/s, loss=0.279, metrics={'acc': 0.8702}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.34it/s, loss=0.28, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.79it/s, loss=0.277, metrics={'acc': 0.8698}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.49it/s, loss=0.279, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.25it/s, loss=0.279, metrics={'acc': 0.87}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.51it/s, loss=0.28, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.90it/s, loss=0.277, metrics={'acc': 0.8694}]\nepoch 41: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.278, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.74it/s, loss=0.28, metrics={'acc': 0.8675}]\nepoch 42: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.279, metrics={'acc': 0.8695}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.76it/s, loss=0.277, metrics={'acc': 0.8699}]\nepoch 43: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:04&lt;00:00, 66.14it/s, loss=0.279, metrics={'acc': 0.8681}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.20it/s, loss=0.277, metrics={'acc': 0.8714}]\nepoch 44: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.37it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.18it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 45: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.23it/s, loss=0.278, metrics={'acc': 0.8702}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.83it/s, loss=0.278, metrics={'acc': 0.8707}]\nepoch 46: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.12it/s, loss=0.278, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.62it/s, loss=0.279, metrics={'acc': 0.8693}]\nepoch 47: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.276, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 48: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.25it/s, loss=0.278, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.52it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 49: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.35it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.82it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 50: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.15it/s, loss=0.277, metrics={'acc': 0.8717}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.62it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 51: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.277, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.63it/s, loss=0.278, metrics={'acc': 0.87}]\nepoch 52: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.04it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.39it/s, loss=0.278, metrics={'acc': 0.8697}]\nepoch 53: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.15it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.56it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 54: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.41it/s, loss=0.277, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.95it/s, loss=0.278, metrics={'acc': 0.8698}]\nepoch 55: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.90it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 56: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.83it/s, loss=0.277, metrics={'acc': 0.8707}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.13it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 57: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.20it/s, loss=0.277, metrics={'acc': 0.8722}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.16it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 58: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.33it/s, loss=0.276, metrics={'acc': 0.871}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.37it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 59: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.41it/s, loss=0.277, metrics={'acc': 0.8714}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.17it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 60: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.26it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.60it/s, loss=0.278, metrics={'acc': 0.869}]\nepoch 61: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.88it/s, loss=0.278, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.47it/s, loss=0.278, metrics={'acc': 0.8692}]\nepoch 62: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.13it/s, loss=0.276, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.80it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 63: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.20it/s, loss=0.277, metrics={'acc': 0.8715}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.50it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 64: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.11it/s, loss=0.276, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.52it/s, loss=0.278, metrics={'acc': 0.869}]\n</pre> <pre>Best Epoch: 34. Best val_loss: 0.27449\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[32]: Copied! <pre>mlp_trainer.feature_importance\n</pre> mlp_trainer.feature_importance Out[32]: <pre>{'age': 0.116632804,\n 'workclass': 0.050255153,\n 'education': 0.094621316,\n 'marital_status': 0.12328919,\n 'occupation': 0.107893184,\n 'relationship': 0.11747801,\n 'race': 0.054717205,\n 'gender': 0.07514235,\n 'capital_gain': 0.059732802,\n 'capital_loss': 0.06738944,\n 'hours_per_week': 0.0610674,\n 'native_country': 0.07178114}</pre> In\u00a0[33]: Copied! <pre>mlp_preds = mlp_trainer.predict(X_tab=X_tab_test)\n</pre> mlp_preds = mlp_trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 212.38it/s]\n</pre> In\u00a0[34]: Copied! <pre>accuracy_score(mlp_preds, test.income_label)\n</pre> accuracy_score(mlp_preds, test.income_label) Out[34]: <pre>0.8726714431934494</pre>"},{"location":"examples/18_feature_importance_via_attention_weights.html#feature-importance-via-the-attention-weights","title":"Feature Importance via the attention weights\u00b6","text":"<p>I will start by saying that I consider this feature of the library purely experimental. First of all I think there are multiple ways one could address finding the features importances for these models. However, and more importantly, one has to bear in mind that even tree-based algorithms on the same dataset produce different feature importances. This is more \"dramatic\" if one uses different techniques, such as shap or feature permutation (see for example this and references therein). All this to say that, sometimes, feature importance is just a measure contained within the experiment run, and for the model used.</p> <p>With that in mind, each instantiation of a deep tabular model, that has millions of trainable parameters, will potentially produce a different set of feature importances, even if the model has the same architecture. Moreover, this effect will become more apparent if the dataset is relatively easy and there are dependent/related columns so that one could get to the same success metric with different parameters.</p> <p>In summary, feature importances are implemented in this librray for all attention-based models for tabular data, with the exception of the <code>TabPerceiver</code>. However this functionality has to be used and interpreted with care and consider of value within the 'universe' (or context) of the model with which these features were produced.</p> <p>Nonetheless, let's have a look to how one would access to the feature importances when using this library.</p>"},{"location":"examples/19_wide_and_deep_for_recsys_pt1.html","title":"19_wide_and_deep_for_recsys_pt1","text":"<p>The goal of this, and the companion (part 2) notebooks is to illustrate how one could use this library in the context of recommendation systems. In particular, this notebook and the scripts at the <code>wide_deep_for_recsys</code> dir are a response to this issue. Therefore, we will use the Kaggle notebook referred in that issue here.</p> <p>In order to keep the length of the notebook tractable, we will split this exercise in 2. In this first notebook we will prepare the data in almost the exact same way as it is done in the Kaggle notebook and also show how one could use <code>pytorch-widedeep</code> to build a model almost identical to the one in that notebook.</p> <p>In a second notebook, we will show how one could use this library to implement other models, still following the same problem formulation.</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\nimport warnings\n\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep.datasets import load_movielens100k\n</pre> from pathlib import Path import warnings  import pandas as pd from sklearn.model_selection import train_test_split  from pytorch_widedeep.datasets import load_movielens100k In\u00a0[2]: Copied! <pre>warnings.filterwarnings(\"ignore\")\n</pre> warnings.filterwarnings(\"ignore\") In\u00a0[3]: Copied! <pre>save_path = Path(\"prepared_data\")\nif not save_path.exists():\n    save_path.mkdir(parents=True, exist_ok=True)\n</pre> save_path = Path(\"prepared_data\") if not save_path.exists():     save_path.mkdir(parents=True, exist_ok=True) In\u00a0[4]: Copied! <pre>data, users, items = load_movielens100k(as_frame=True)\n</pre> data, users, items = load_movielens100k(as_frame=True) In\u00a0[5]: Copied! <pre># Alternatively, as specified in the docs: 'The last 19 fields are the genres' so:\n# list_of_genres = items.columns.tolist()[-19:]\nlist_of_genres = [\n    \"unknown\",\n    \"Action\",\n    \"Adventure\",\n    \"Animation\",\n    \"Children's\",\n    \"Comedy\",\n    \"Crime\",\n    \"Documentary\",\n    \"Drama\",\n    \"Fantasy\",\n    \"Film-Noir\",\n    \"Horror\",\n    \"Musical\",\n    \"Mystery\",\n    \"Romance\",\n    \"Sci-Fi\",\n    \"Thriller\",\n    \"War\",\n    \"Western\",\n]\n</pre> # Alternatively, as specified in the docs: 'The last 19 fields are the genres' so: # list_of_genres = items.columns.tolist()[-19:] list_of_genres = [     \"unknown\",     \"Action\",     \"Adventure\",     \"Animation\",     \"Children's\",     \"Comedy\",     \"Crime\",     \"Documentary\",     \"Drama\",     \"Fantasy\",     \"Film-Noir\",     \"Horror\",     \"Musical\",     \"Mystery\",     \"Romance\",     \"Sci-Fi\",     \"Thriller\",     \"War\",     \"Western\", ] <p>Let's first start by loading the interactions, user and item data</p> In\u00a0[6]: Copied! <pre>data.head()\n</pre> data.head() Out[6]: user_id movie_id rating timestamp 0 196 242 3 881250949 1 186 302 3 891717742 2 22 377 1 878887116 3 244 51 2 880606923 4 166 346 1 886397596 In\u00a0[7]: Copied! <pre>users.head()\n</pre> users.head() Out[7]: user_id age gender occupation zip_code 0 1 24 M technician 85711 1 2 53 F other 94043 2 3 23 M writer 32067 3 4 24 M technician 43537 4 5 33 F other 15213 In\u00a0[8]: Copied! <pre>items.head()\n</pre> items.head() Out[8]: movie_id movie_title release_date video_release_date IMDb_URL unknown Action Adventure Animation Children's ... Fantasy Film-Noir Horror Musical Mystery Romance Sci-Fi Thriller War Western 0 1 Toy Story (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Toy%20Story%2... 0 0 0 1 1 ... 0 0 0 0 0 0 0 0 0 0 1 2 GoldenEye (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?GoldenEye%20(... 0 1 1 0 0 ... 0 0 0 0 0 0 0 1 0 0 2 3 Four Rooms (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Four%20Rooms%... 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 3 4 Get Shorty (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Get%20Shorty%... 0 1 0 0 0 ... 0 0 0 0 0 0 0 0 0 0 4 5 Copycat (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Copycat%20(1995) 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 <p>5 rows \u00d7 24 columns</p> In\u00a0[9]: Copied! <pre># adding a column with the number of movies watched per user\ndataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True)\ndataset[\"one\"] = 1\ndataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum()\ndataset.drop(\"one\", axis=1, inplace=True)\ndataset.head()\n</pre> # adding a column with the number of movies watched per user dataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True) dataset[\"one\"] = 1 dataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum() dataset.drop(\"one\", axis=1, inplace=True) dataset.head() Out[9]: user_id movie_id rating timestamp num_watched 0 1 168 5 874965478 1 1 1 172 5 874965478 2 2 1 165 5 874965518 3 3 1 156 4 874965556 4 4 1 196 5 874965677 5 In\u00a0[10]: Copied! <pre># adding a column with the mean rating at a point in time per user\ndataset[\"mean_rate\"] = (\n    dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> # adding a column with the mean rating at a point in time per user dataset[\"mean_rate\"] = (     dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"] ) dataset.head() Out[10]: user_id movie_id rating timestamp num_watched mean_rate 0 1 168 5 874965478 1 5.00 1 1 172 5 874965478 2 5.00 2 1 165 5 874965518 3 5.00 3 1 156 4 874965556 4 4.75 4 1 196 5 874965677 5 4.80 In\u00a0[11]: Copied! <pre>dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1)\n</pre> dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1) <p>Following the same processing used by the author in the before-mentioned Kaggle notebook, we build sequences of previous movies watched</p> In\u00a0[12]: Copied! <pre># Here the author builds the sequences\ndataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x))\ndataset[\"prev_movies\"] = (\n    dataset.groupby(\"user_id\")[\"prev_movies\"]\n    .apply(lambda x: (x + \" \").cumsum().str.strip())\n    .reset_index(drop=True)\n)\ndataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split())\ndataset.head()\n</pre> # Here the author builds the sequences dataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x)) dataset[\"prev_movies\"] = (     dataset.groupby(\"user_id\")[\"prev_movies\"]     .apply(lambda x: (x + \" \").cumsum().str.strip())     .reset_index(drop=True) ) dataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split()) dataset.head() Out[12]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies 0 1 168 5 874965478 1 5.00 172.0 [168] 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] <p>And now we add a <code>genre_rate</code> as the mean of all movies rated for a given genre per user</p> In\u00a0[13]: Copied! <pre>dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\")\nfor genre in list_of_genres:\n    dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]\n    dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()\n    dataset[f\"{genre}_rate\"] = (\n        dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]\n    )\ndataset[list_of_genres] = dataset[list_of_genres].apply(\n    lambda x: x / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\") for genre in list_of_genres:     dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]     dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()     dataset[f\"{genre}_rate\"] = (         dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]     ) dataset[list_of_genres] = dataset[list_of_genres].apply(     lambda x: x / dataset[\"num_watched\"] ) dataset.head() Out[13]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Fantasy_rate Film-Noir_rate Horror_rate Musical_rate Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN <p>5 rows \u00d7 46 columns</p> <p>Adding user features</p> In\u00a0[14]: Copied! <pre>dataset = dataset.merge(users, on=\"user_id\", how=\"left\")\ndataset.head()\n</pre> dataset = dataset.merge(users, on=\"user_id\", how=\"left\") dataset.head() Out[14]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN 24 M technician 85711 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 <p>5 rows \u00d7 50 columns</p> <p>Again, we use the same settings as those in the Kaggle notebook, but <code>COLD_START_TRESH</code> is pretty aggressive</p> In\u00a0[15]: Copied! <pre>COLD_START_TRESH = 5\n\nfiltred_data = dataset[\n    (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna())\n].sort_values(\"timestamp\")\ntrain_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False)\nvalid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False)\n</pre> COLD_START_TRESH = 5  filtred_data = dataset[     (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna()) ].sort_values(\"timestamp\") train_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False) valid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False) In\u00a0[16]: Copied! <pre>cols_to_drop = [\n    # \"rating\",\n    \"timestamp\",\n    \"num_watched\",\n]\n\ndf_train = train_data.drop(cols_to_drop, axis=1)\ndf_valid = valid_data.drop(cols_to_drop, axis=1)\ndf_test = test_data.drop(cols_to_drop, axis=1)\n\ndf_train.to_pickle(save_path / \"df_train.pkl\")\ndf_valid.to_pickle(save_path / \"df_valid.pkl\")\ndf_test.to_pickle(save_path / \"df_test.pkl\")\n</pre> cols_to_drop = [     # \"rating\",     \"timestamp\",     \"num_watched\", ]  df_train = train_data.drop(cols_to_drop, axis=1) df_valid = valid_data.drop(cols_to_drop, axis=1) df_test = test_data.drop(cols_to_drop, axis=1)  df_train.to_pickle(save_path / \"df_train.pkl\") df_valid.to_pickle(save_path / \"df_valid.pkl\") df_test.to_pickle(save_path / \"df_test.pkl\") <p>Let's now build a model that is nearly identical to the one use in the Kaggle notebook</p> In\u00a0[17]: Copied! <pre>import numpy as np\nimport torch\nfrom torch import nn\nfrom scipy.sparse import coo_matrix\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import numpy as np import torch from torch import nn from scipy.sparse import coo_matrix  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[18]: Copied! <pre>device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nsave_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n</pre> device = \"cuda\" if torch.cuda.is_available() else \"cpu\"  save_path = Path(\"prepared_data\")  PAD_IDX = 0 <p>Let's use some of the functions the author of the kaggle's notebook uses to prepare the data</p> In\u00a0[19]: Copied! <pre>def get_coo_indexes(lil):\n    rows = []\n    cols = []\n    for i, el in enumerate(lil):\n        if type(el) != list:\n            el = [el]\n        for j in el:\n            rows.append(i)\n            cols.append(j)\n    return rows, cols\n\n\ndef get_sparse_features(series, shape):\n    coo_indexes = get_coo_indexes(series.tolist())\n    sparse_df = coo_matrix(\n        (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape\n    )\n    return sparse_df\n\n\ndef sparse_to_idx(data, pad_idx=-1):\n    indexes = data.nonzero()\n    indexes_df = pd.DataFrame()\n    indexes_df[\"rows\"] = indexes[0]\n    indexes_df[\"cols\"] = indexes[1]\n    mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())\n    max_len = mdf.apply(lambda x: len(x)).max()\n    return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values\n</pre> def get_coo_indexes(lil):     rows = []     cols = []     for i, el in enumerate(lil):         if type(el) != list:             el = [el]         for j in el:             rows.append(i)             cols.append(j)     return rows, cols   def get_sparse_features(series, shape):     coo_indexes = get_coo_indexes(series.tolist())     sparse_df = coo_matrix(         (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape     )     return sparse_df   def sparse_to_idx(data, pad_idx=-1):     indexes = data.nonzero()     indexes_df = pd.DataFrame()     indexes_df[\"rows\"] = indexes[0]     indexes_df[\"cols\"] = indexes[1]     mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())     max_len = mdf.apply(lambda x: len(x)).max()     return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values <p>For the time being, we will not use a validation set for hyperparameter optimization, and we will simply concatenate the validation and the test set in one test set. I simply splitted the data into train/valid/test in case the reader wants to actually do hyperparameter optimization (and because I know in the future I will).</p> <p>There is also another caveat worth mentioning, related to the indexing of the movies. To build the matrices of movies watched, we use the entire dataset. A more realistic (and correct) approach would be to use ONLY the movies that appear in the training set and consider <code>unknown</code> or <code>unseen</code> those in the testing set that have not been seen during training. Nonetheless, this will not affect the purposes of this notebook, which is to illustrate how one could use <code>pytorch-widedeep</code> to build a recommendation algorithm. However, if one wanted to explore the performance of different algorithms in a \"proper\" way, these \"details\" need to be accounted for.</p> In\u00a0[20]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) In\u00a0[21]: Copied! <pre>id_cols = [\"user_id\", \"movie_id\"]\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> id_cols = [\"user_id\", \"movie_id\"] max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) In\u00a0[22]: Copied! <pre>X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_train = np.array(df_train.target.values, dtype=\"int64\")\ntrain_movies_watched = get_sparse_features(\n    df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1)\n)\n\nX_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_test = np.array(df_test.target.values, dtype=\"int64\")\ntest_movies_watched = get_sparse_features(\n    df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1)\n)\n</pre> X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_train = np.array(df_train.target.values, dtype=\"int64\") train_movies_watched = get_sparse_features(     df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1) )  X_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_test = np.array(df_test.target.values, dtype=\"int64\") test_movies_watched = get_sparse_features(     df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1) ) <p>let's have a look to the information in each dataset</p> In\u00a0[23]: Copied! <pre>X_train.head()\n</pre> X_train.head() Out[23]: mean_rate unknown Action Adventure Animation Children's Comedy Crime Documentary Drama ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 25423 4.000000 0.0 0.400000 0.200000 0.0 0.0 0.400000 0.0 0.0 0.200000 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25425 4.000000 0.0 0.285714 0.142857 0.0 0.0 0.428571 0.0 0.0 0.285714 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25424 4.000000 0.0 0.333333 0.166667 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25426 3.875000 0.0 0.250000 0.125000 0.0 0.0 0.375000 0.0 0.0 0.250000 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 25427 3.888889 0.0 0.222222 0.111111 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 <p>5 rows \u00d7 43 columns</p> In\u00a0[24]: Copied! <pre>y_train\n</pre> y_train Out[24]: <pre>array([772, 288, 108, ..., 183, 432, 509])</pre> In\u00a0[25]: Copied! <pre>train_movies_watched\n</pre> train_movies_watched Out[25]: <pre>&lt;76228x1683 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 7957390 stored elements in COOrdinate format&gt;</pre> In\u00a0[26]: Copied! <pre>sorted(df_train.prev_movies.tolist()[0])\n</pre> sorted(df_train.prev_movies.tolist()[0]) Out[26]: <pre>['173', '185', '255', '286', '298']</pre> In\u00a0[27]: Copied! <pre>np.where(train_movies_watched.todense()[0])\n</pre> np.where(train_movies_watched.todense()[0]) Out[27]: <pre>(array([0, 0, 0, 0, 0]), array([173, 185, 255, 286, 298]))</pre> <p>And from now on is when the specifics related to this library start to appear. The only component that is going to be a bit different is the so-called tabular component, referred as <code>continuous</code> in the notebook.</p> <p>In the case of <code>pytorch-widedeep</code> we have the <code>TabPreprocessor</code> that allows for a lot of flexibility as to how we would like to process the tabular component of this Wide and Deep model. In other words, here our tabular component is a bit more elaborated than that in the notebook, just a bit...</p> In\u00a0[28]: Copied! <pre>cat_cols = [\"gender\", \"occupation\", \"zip_code\"]\ncont_cols = [c for c in X_train if c not in cat_cols]\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_cols,\n    continuous_cols=cont_cols,\n)\n</pre> cat_cols = [\"gender\", \"occupation\", \"zip_code\"] cont_cols = [c for c in X_train if c not in cat_cols] tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_cols,     continuous_cols=cont_cols, ) In\u00a0[29]: Copied! <pre>X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0))\nX_test_tab = tab_preprocessor.transform(X_test.fillna(0))\n</pre> X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0)) X_test_tab = tab_preprocessor.transform(X_test.fillna(0)) <p>Now, in the notebook, the author moves the sparse matrices to sparse tensors and then turns them into dense tensors. In reality, this is not neccessary, one could feed sparse tensors to <code>nn.Linear</code> layers in pytorch. Nonetheless, this is not the most efficient implementation and is the reason why in our library the wide, linear component is implemented as an embedding layer.</p> <p>Nonetheless, to reproduce the notebook the best we can and because currently the <code>Wide</code> model in <code>pytorch-widedeep</code> is not designed to receive sparse tensors (we might consider implementing this functionality), we will turn the sparse COO matrices into dense arrays. We will then code a fairly simple, custom <code>Wide</code> component.</p> In\u00a0[30]: Copied! <pre>X_train_wide = np.array(train_movies_watched.todense())\nX_test_wide = np.array(test_movies_watched.todense())\n</pre> X_train_wide = np.array(train_movies_watched.todense()) X_test_wide = np.array(test_movies_watched.todense()) <p>Finally, the author of the notebook uses a simple <code>Embedding</code> layer to encode the sequences of movies watched, the <code>prev_movies</code> columns. In my opinion, there is an element of information redundancy here. This is because the wide and text components have implicitely the same information, but in different form. Moreover, both of the models used for these two components ignore the sequential element in the data. Nonetheless, we want to reproduce the Kaggle notebook as close as possible, AND as one can explore later (by simply performing simple ablation studies), the wide component seems to carry most of the predictive power.</p> In\u00a0[31]: Copied! <pre>X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX)\nX_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX)\n</pre> X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX) X_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX) <p>Let's now build the models</p> In\u00a0[32]: Copied! <pre>class Wide(nn.Module):\n    def __init__(self, input_dim: int, pred_dim: int):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # When I coded the library I never though that someone would want to code\n        # their own wide component. However, if you do, the wide component must have\n        # a 'wide_linear' attribute. In other words, the linear layer must be\n        # called 'wide_linear'\n        self.wide_linear = nn.Linear(input_dim, pred_dim)\n\n    def forward(self, X):\n        out = self.wide_linear(X.type(torch.float32))\n        return out\n\n\nwide = Wide(X_train_wide.shape[1], max_movie_index + 1)\n</pre> class Wide(nn.Module):     def __init__(self, input_dim: int, pred_dim: int):         super().__init__()          self.input_dim = input_dim         self.pred_dim = pred_dim          # When I coded the library I never though that someone would want to code         # their own wide component. However, if you do, the wide component must have         # a 'wide_linear' attribute. In other words, the linear layer must be         # called 'wide_linear'         self.wide_linear = nn.Linear(input_dim, pred_dim)      def forward(self, X):         out = self.wide_linear(X.type(torch.float32))         return out   wide = Wide(X_train_wide.shape[1], max_movie_index + 1) In\u00a0[33]: Copied! <pre>wide\n</pre> wide Out[33]: <pre>Wide(\n  (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n)</pre> In\u00a0[34]: Copied! <pre>class SimpleEmbed(nn.Module):\n    def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):\n        super().__init__()\n\n        self.vocab_size = vocab_size\n        self.embed_dim = embed_dim\n        self.pad_idx = pad_idx\n\n        # The sequences of movies watched are simply embedded in the Kaggle\n        # notebook. No RNN, Transformer or any model is used\n        self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)\n\n    def forward(self, X):\n        embed = self.embed(X)\n        embed_mean = torch.mean(embed, dim=1)\n        return embed_mean\n\n    @property\n    def output_dim(self) -&gt; int:\n        # All deep components in a custom 'pytorch-widedeep' model must have\n        # an output_dim property\n        return self.embed_dim\n\n\n#  In the notebook the author uses simply embeddings\nsimple_embed = SimpleEmbed(max_movie_index + 1, 16, 0)\n</pre> class SimpleEmbed(nn.Module):     def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):         super().__init__()          self.vocab_size = vocab_size         self.embed_dim = embed_dim         self.pad_idx = pad_idx          # The sequences of movies watched are simply embedded in the Kaggle         # notebook. No RNN, Transformer or any model is used         self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)      def forward(self, X):         embed = self.embed(X)         embed_mean = torch.mean(embed, dim=1)         return embed_mean      @property     def output_dim(self) -&gt; int:         # All deep components in a custom 'pytorch-widedeep' model must have         # an output_dim property         return self.embed_dim   #  In the notebook the author uses simply embeddings simple_embed = SimpleEmbed(max_movie_index + 1, 16, 0) In\u00a0[35]: Copied! <pre>simple_embed\n</pre> simple_embed Out[35]: <pre>SimpleEmbed(\n  (embed): Embedding(1683, 16, padding_idx=0)\n)</pre> <p>Maybe one would like to use an RNN to account for the sequence nature of the problem. If that was the case it would be as easy as:</p> In\u00a0[36]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=max_movie_index + 1,\n    embed_dim=16,\n    hidden_dim=32,\n    n_layers=2,\n    rnn_type=\"gru\",\n)\n</pre> basic_rnn = BasicRNN(     vocab_size=max_movie_index + 1,     embed_dim=16,     hidden_dim=32,     n_layers=2,     rnn_type=\"gru\", ) <p>And finally, the tabular component, which is the notebook is simply a stak of linear + Rely layers. In our case we have an embedding layer before the linear layers to encode categorial and numerical cols</p> In\u00a0[37]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     cont_norm_layer=None,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", ) In\u00a0[38]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[38]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n      (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=111, out_features=1024, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=1024, out_features=512, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_2): Sequential(\n        (0): Linear(in_features=512, out_features=256, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Finally, we simply wrap up all models with the <code>WideDeep</code> 'collector' class and we are ready to train.</p> In\u00a0[39]: Copied! <pre>wide_deep_model = WideDeep(\n    wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1\n)\n</pre> wide_deep_model = WideDeep(     wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1 ) In\u00a0[40]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[40]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n  )\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n          (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=111, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): SimpleEmbed(\n      (embed): Embedding(1683, 16, padding_idx=0)\n    )\n    (1): Linear(in_features=16, out_features=1683, bias=True)\n  )\n)</pre> <p>Note that the main difference between this wide and deep model and the Wide and Deep model in the Kaggle notebook is that in that notebook, the author concatenates the embedings and the tabular features, then passes this concatenation through a stack of linear + Relu layers with a final output dim of 256. Then concatenates this output with the binary features and connects this concatenation with the final linear layer (so the final weights are of dim (batch_size, 256 + 1683)). Our implementation follows the notation of the original paper and instead of concatenating the tabular, text and wide components and then connect them to the output neurons, we first compute their output, and then add it (see here: https://arxiv.org/pdf/1606.07792.pdf, their Eq 3). Note that this is effectively the same, with the caveat that while in one case one initialises a big weight matrix \"at once\", in our implementation we initialise different matrices for different components. Anyway, let's give it a go.</p> In\u00a0[41]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), ) In\u00a0[42]: Copied! <pre>trainer.fit(\n    X_train={\n        \"X_wide\": X_train_wide,\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_wide\": X_test_wide,\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=5,\n    batch_size=512,\n    shuffle=False,\n)\n</pre> trainer.fit(     X_train={         \"X_wide\": X_train_wide,         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_wide\": X_test_wide,         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=5,     batch_size=512,     shuffle=False, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.66it/s, loss=6.66]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.75it/s, loss=6.6]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:21&lt;00:00,  6.95it/s, loss=5.97]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 21.03it/s, loss=6.52]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.51it/s, loss=5.65]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 20.16it/s, loss=6.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:23&lt;00:00,  6.29it/s, loss=5.41]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 13.97it/s, loss=6.57]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.58it/s, loss=5.2]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.82it/s, loss=6.63]\n</pre> <p>Now one could continue to the 'compare' metrics section of the Kaggle notebook. However, for the purposes of illustrating how one could use <code>pytorch-widedeep</code> to build recommendation algorithms we consider this notebook completed and move onto part 2</p>"},{"location":"examples/19_wide_and_deep_for_recsys_pt1.html#problem-formulation","title":"Problem formulation\u00b6","text":"<p>In this particular exercise the problem is formulated as predicting the next movie that will be watched (in consequence the last interactions will be discarded)</p>"},{"location":"examples/19_wide_and_deep_for_recsys_pt2.html","title":"19_wide_and_deep_for_recsys_pt2","text":"<p>This is the second of the two notebooks where we aim to illustrate how one could use this library to build recommendation algorithms using the example in this Kaggle notebook as guidance. In the previous notebook we used <code>pytorch-widedeep</code> to build a model that replicated almost exactly that in the notebook. In this, shorter notebook we will show how one could use the library to explore other models, following the same problem formulation, this is: given a state of a user at a certain point in time having watched a series of movies, our goal is to predict which movie the user will watch next.</p> <p>Assuming that one has read (and run) the previous notebook, the required data will be stored in a local dir called <code>prepared_data</code>, so let's read it:</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\n\nimport numpy as np\nimport torch\nimport pandas as pd\nfrom torch import nn\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.utils import pad_sequences\nfrom pytorch_widedeep.models import TabMlp, WideDeep, Transformer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pathlib import Path  import numpy as np import torch import pandas as pd from torch import nn  from pytorch_widedeep import Trainer from pytorch_widedeep.utils import pad_sequences from pytorch_widedeep.models import TabMlp, WideDeep, Transformer from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre>save_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n\nid_cols = [\"user_id\", \"movie_id\"]\n\ndf_train = pd.read_pickle(save_path / \"df_train.pkl\")\ndf_valid = pd.read_pickle(save_path / \"df_valid.pkl\")\ndf_test = pd.read_pickle(save_path / \"df_test.pkl\")\n</pre> save_path = Path(\"prepared_data\")  PAD_IDX = 0  id_cols = [\"user_id\", \"movie_id\"]  df_train = pd.read_pickle(save_path / \"df_train.pkl\") df_valid = pd.read_pickle(save_path / \"df_valid.pkl\") df_test = pd.read_pickle(save_path / \"df_test.pkl\") <p>...remember that in the previous notebook we explained that we are not  going to use a validation set here (in a real-world example, or simply a more realistic example, one should always use it).</p> In\u00a0[3]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) <p>Also remember that, in the previous notebook we discussed that the <code>'maxlen'</code> and <code>'max_movie_index'</code> parameters should be computed using only the train set. In particular, to properly do the tokenization, one would have to use ONLY train tokens and add a token for new 'unknown'/'unseen' movies in the test set. This can also be done with this library or manually, so I will leave it to the reader to implement that tokenzation appraoch.</p> In\u00a0[4]: Copied! <pre>maxlen = max(\n    df_train.prev_movies.apply(lambda x: len(x)).max(),\n    df_test.prev_movies.apply(lambda x: len(x)).max(),\n)\n\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> maxlen = max(     df_train.prev_movies.apply(lambda x: len(x)).max(),     df_test.prev_movies.apply(lambda x: len(x)).max(), )  max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) <p>From now one things are pretty simple, moreover bearing in mind that in this example we are not going to use a wide component since, in pple, one would believe that the information in that component is also 'carried' by the movie sequences (However in the previous notebook, if one performs ablation studies, these suggest that most of the prediction power comes from the linear, wide model).</p> <p>In the example here we are going to explore one (of many) possibilities. We are simply going to encode the triplet <code>(user, item, rating)</code> and use it as a <code>deeptabular</code> component and the sequences of previously watched movies as the <code>deeptext</code> component. For the <code>deeptext</code> component we are going to use a basic encoder-only transformer model.</p> <p>Let's start with the tabular data preparation</p> In\u00a0[5]: Copied! <pre>df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntrain_movies_sequences = df_train.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_train = df_train.target.values.astype(int)\n\ndf_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntest_movies_sequences = df_test.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_test = df_test.target.values.astype(int)\n\ntab_preprocessor = tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"],\n)\nX_train_tab = tab_preprocessor.fit_transform(df_train_user_item)\nX_test_tab = tab_preprocessor.transform(df_test_user_item)\n</pre> df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] train_movies_sequences = df_train.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_train = df_train.target.values.astype(int)  df_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] test_movies_sequences = df_test.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_test = df_test.target.values.astype(int)  tab_preprocessor = tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"], ) X_train_tab = tab_preprocessor.fit_transform(df_train_user_item) X_test_tab = tab_preprocessor.transform(df_test_user_item) <p>And not the text component, simply padding the sequences:</p> In\u00a0[6]: Copied! <pre>X_train_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=PAD_IDX,\n        )\n        for s in train_movies_sequences\n    ]\n)\nX_test_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=0,\n        )\n        for s in test_movies_sequences\n    ]\n)\n</pre> X_train_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=PAD_IDX,         )         for s in train_movies_sequences     ] ) X_test_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=0,         )         for s in test_movies_sequences     ] ) <p>We now define the model components and the wide and deep model.</p> In\u00a0[7]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n\n# plenty of options here, see the docs\ntransformer = Transformer(\n    vocab_size=max_movie_index + 1,\n    embed_dim=32,\n    n_heads=2,\n    n_blocks=2,\n    seq_length=maxlen,\n)\n\nwide_deep_model = WideDeep(\n    deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", )  # plenty of options here, see the docs transformer = Transformer(     vocab_size=max_movie_index + 1,     embed_dim=32,     n_heads=2,     n_blocks=2,     seq_length=maxlen, )  wide_deep_model = WideDeep(     deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1 ) In\u00a0[8]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[8]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_user_id): Embedding(749, 65, padding_idx=0)\n          (emb_layer_movie_id): Embedding(1612, 100, padding_idx=0)\n          (emb_layer_rating): Embedding(6, 4, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=169, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): Transformer(\n      (embedding): Embedding(1683, 32, padding_idx=0)\n      (pos_encoder): PositionalEncoding(\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=23552, out_features=1683, bias=True)\n  )\n)</pre> <p>And as in the previous notebook, let's train (you will need a GPU for this)</p> In\u00a0[\u00a0]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n\ntrainer.fit(\n    X_train={\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=10,\n    batch_size=521,\n    shuffle=False,\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), )  trainer.fit(     X_train={         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=10,     batch_size=521,     shuffle=False, ) <pre>epoch 1:   0%|                                                                                                         | 0/147 [00:34&lt;?, ?it/s]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/20_load_from_folder_functionality.html","title":"20_load_from_folder_functionality","text":"<p>In this notebook I want to illustrate how one can use our <code>[...]FromFolder</code> functionalities along with the <code>[...]ChunkPreProcessors</code> in those cases where the dataset is too bit to fit in memory.</p> <p>These functionalities in the library have been designed for the following scenarop</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\nfrom torch.utils.data import DataLoader\n\nfrom pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import TrainerFromFolder\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import (\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n    ChunkTabPreprocessor,\n    ChunkTextPreprocessor,\n)\nfrom pytorch_widedeep.load_from_folder import (\n    TabFromFolder,\n    TextFromFolder,\n    ImageFromFolder,\n    WideDeepDatasetFromFolder,\n)\n</pre> import numpy as np import torch import pandas as pd from torch.utils.data import DataLoader  from pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep from pytorch_widedeep.training import TrainerFromFolder from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import (     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor,     ChunkTabPreprocessor,     ChunkTextPreprocessor, ) from pytorch_widedeep.load_from_folder import (     TabFromFolder,     TextFromFolder,     ImageFromFolder,     WideDeepDatasetFromFolder, ) In\u00a0[2]: Copied! <pre># in my case, I place the data in a folder I call tmp_data, let's see how it looks\nairbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\nairbnb_data.sample(5)\n</pre> # in my case, I place the data in a folder I call tmp_data, let's see how it looks airbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") airbnb_data.sample(5) Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 39 53242.jpg 247650 A lovely big bright bedroom in a 2 bedroom fla... 2.0 t Lambeth 51.47075 -0.12913 t apartment ... 0 0 0 0 1 0 0 250.0 5.0 9.75 214 236716.jpg 1241070 We offer a warm welcome in our quiet double ro... 1.0 t Hackney 51.56593 -0.07482 t other ... 0 0 0 0 1 0 0 200.0 10.0 76.50 400 346523.jpg 1756532 Available for you to rent is a cozy studio in ... 2.0 t Kensington and Chelsea 51.48311 -0.18428 t other ... 0 0 0 0 1 0 0 0.0 50.0 180.90 512 389627.jpg 1949299 This gorgeous studio flat is situated in the v... 1.0 t Westminster 51.51838 -0.14238 f apartment ... 0 0 0 0 1 0 0 250.0 25.0 276.90 504 388767.jpg 1945165 If you want to experience London at it's best ... 2.0 f Camden 51.54293 -0.14073 t apartment ... 0 0 0 0 1 0 0 150.0 10.0 591.10 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># for example\nfrom IPython.display import Image\n\npil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\")\ndisplay(pil_img)\n</pre> # for example from IPython.display import Image  pil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\") display(pil_img) In\u00a0[4]: Copied! <pre># And the description for the property that that picture belongs to is:\nairbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist()\n</pre> # And the description for the property that that picture belongs to is: airbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist() Out[4]: <pre>[\"Bright, sunny beautiful room that will give you the perfect base to explore all of London. Come and explore one of London's best neighbourhoods - Herne Hill! As mentioned in (Website hidden by Airbnb)   (Website hidden by Airbnb)  WiFi availability with a fully stocked and clean uplifting home. Lovely sunny, airy and big double bedroom on a leafy south-London street.    Note: This room comes with a reserved Off-Street parking spot! The room is on the first floor and boasts an enormous Super King bed, gorgeous wooden floors, tall ceilings and large windows which let in the sunshine almost all day. (Yoga May or meditation cushion available on request) The flat is bright and airy and big! So lots of space for all.  Location wise you are only 10 minutes walk to either Herne Hill or West Dulwich stations, both of which will take you to Victoria and the city within minutes. You can also hop on a bus right outside the flat that will take you to Brixton tube station within 8 minutes where you \"]</pre> <p>Ok, so we have tabular data where one column is <code>description</code> and another <code>id</code>, points towards the images stored in disk. Now, remember the following, because this will appear a few times in the notebook: our \"reference dataset\" is the tabular data.</p> <p>Therefore, since I want to illustrate a \"semi-realistic\" case, if we need to split the data into training, validation and test datasets, these datasets needs to be separetely stored in disk. In my case I have done this and in the <code>tmp_data/airbnb</code> dir I have the following:</p> <pre>../tmp_data/airbnb\n\u251c\u2500\u2500 airbnb_sample.csv\n\u251c\u2500\u2500 airbnb_sample_eval.csv\n\u251c\u2500\u2500 airbnb_sample_test.csv\n\u251c\u2500\u2500 airbnb_sample_train.csv\n\u2514\u2500\u2500 property_picture\n</pre> <p>Where <code>airbnb_sample.csv</code> is the full sample (1001 rows) and the <code>train</code>, <code>eval</code> and <code>test</code> set is the corresponding splits. In a realistic example, the full sample would be the 'gigantic' dataset and the rest the corresponding splits. One has to do this 'offline', prior to start the coding.</p> <p>Also, one thing that one needs to know is the number of total observations/rows, as well as the splits. In our case the train size is 800, and the eval and test sizes are 100 and 101 respectively.</p> <p>With all that info, let's star</p> In\u00a0[5]: Copied! <pre># path to the tabular data and the splits\ndata_path = \"../tmp_data/airbnb/\"\ntrain_fname = \"airbnb_sample_train.csv\"\neval_fname = \"airbnb_sample_eval.csv\"\ntest_fname = \"airbnb_sample_test.csv\"\n\n# split sizes\ntrain_size = 800\neval_size = 100\ntest_size = 101\n\n# number of chunks for the Chunk Preprocessors\nchunksize = 100\nn_chunks = int(np.ceil(train_size / chunksize))\n\n# path to the image dataset and name of the image col\nimg_path = \"../tmp_data/airbnb/property_picture/\"\nimg_col = \"id\"\n\n# name of the text col\ntext_col = \"description\"\n\n# mane of the target\ntarget_col = \"yield\"\n\n# definition of the categorical and continuous cols for the TabPreprocessor\ncat_embed_cols = [\n    \"host_listings_count\",\n    \"neighbourhood_cleansed\",\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"accommodates\",\n    \"bathrooms\",\n    \"bedrooms\",\n    \"beds\",\n    \"guests_included\",\n    \"minimum_nights\",\n    \"instant_bookable\",\n    \"cancellation_policy\",\n    \"has_house_rules\",\n    \"host_gender\",\n    \"accommodates_catg\",\n    \"guests_included_catg\",\n    \"minimum_nights_catg\",\n    \"host_listings_count_catg\",\n    \"bathrooms_catg\",\n    \"bedrooms_catg\",\n    \"beds_catg\",\n    \"security_deposit\",\n    \"extra_people\",\n]\ncont_cols = [\"latitude\", \"longitude\"]\n</pre> # path to the tabular data and the splits data_path = \"../tmp_data/airbnb/\" train_fname = \"airbnb_sample_train.csv\" eval_fname = \"airbnb_sample_eval.csv\" test_fname = \"airbnb_sample_test.csv\"  # split sizes train_size = 800 eval_size = 100 test_size = 101  # number of chunks for the Chunk Preprocessors chunksize = 100 n_chunks = int(np.ceil(train_size / chunksize))  # path to the image dataset and name of the image col img_path = \"../tmp_data/airbnb/property_picture/\" img_col = \"id\"  # name of the text col text_col = \"description\"  # mane of the target target_col = \"yield\"  # definition of the categorical and continuous cols for the TabPreprocessor cat_embed_cols = [     \"host_listings_count\",     \"neighbourhood_cleansed\",     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"accommodates\",     \"bathrooms\",     \"bedrooms\",     \"beds\",     \"guests_included\",     \"minimum_nights\",     \"instant_bookable\",     \"cancellation_policy\",     \"has_house_rules\",     \"host_gender\",     \"accommodates_catg\",     \"guests_included_catg\",     \"minimum_nights_catg\",     \"host_listings_count_catg\",     \"bathrooms_catg\",     \"bedrooms_catg\",     \"beds_catg\",     \"security_deposit\",     \"extra_people\", ] cont_cols = [\"latitude\", \"longitude\"] In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    default_embed_dim=8,\n    verbose=0,\n)\n\ntext_preprocessor = TextPreprocessor(\n    text_col=text_col,\n    n_cpus=1,\n)\n\nimg_preprocessor = ImagePreprocessor(\n    img_col=img_col,\n    img_path=img_path,\n)\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     default_embed_dim=8,     verbose=0, )  text_preprocessor = TextPreprocessor(     text_col=text_col,     n_cpus=1, )  img_preprocessor = ImagePreprocessor(     img_col=img_col,     img_path=img_path, ) In\u00a0[7]: Copied! <pre>tab_preprocessor.fit(airbnb_data)\ntext_preprocessor.fit(airbnb_data)\nimg_preprocessor.fit(airbnb_data)\n</pre> tab_preprocessor.fit(airbnb_data) text_preprocessor.fit(airbnb_data) img_preprocessor.fit(airbnb_data) <pre>The vocabulary contains 2192 tokens\n</pre> Out[7]: <pre>ImagePreprocessor(img_col=id, img_path=../tmp_data/airbnb/property_picture/, width=224, height=224, verbose=1)</pre> In\u00a0[8]: Copied! <pre>chunk_tab_preprocessor = ChunkTabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    n_chunks=n_chunks,\n    default_embed_dim=8,\n    verbose=0,\n)\n\nchunk_text_preprocessor = ChunkTextPreprocessor(\n    n_chunks=n_chunks,\n    text_col=text_col,\n    n_cpus=1,\n    verbose=0,\n)\n\nfor i, chunk in enumerate(\n    pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize)\n):\n    print(f\"chunk in loop: {i + 1}\")\n    chunk_tab_preprocessor.fit(chunk)\n    chunk_text_preprocessor.fit(chunk)\n</pre> chunk_tab_preprocessor = ChunkTabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     n_chunks=n_chunks,     default_embed_dim=8,     verbose=0, )  chunk_text_preprocessor = ChunkTextPreprocessor(     n_chunks=n_chunks,     text_col=text_col,     n_cpus=1,     verbose=0, )  for i, chunk in enumerate(     pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize) ):     print(f\"chunk in loop: {i + 1}\")     chunk_tab_preprocessor.fit(chunk)     chunk_text_preprocessor.fit(chunk) <pre>chunk in loop: 1\nchunk in loop: 2\nchunk in loop: 3\nchunk in loop: 4\nchunk in loop: 5\nchunk in loop: 6\nchunk in loop: 7\nchunk in loop: 8\n</pre> In\u00a0[9]: Copied! <pre>train_tab_folder = TabFromFolder(\n    fname=train_fname,\n    directory=data_path,\n    target_col=target_col,\n    preprocessor=tab_preprocessor,\n    text_col=text_col,\n    img_col=img_col,\n)\n\n# Note how we can use the `train_tab_folder` as reference so we don't have to\n# define all parameters again\neval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)\n\n# Note that for the test set we can ignore the target as no metric will be\n# computed by the `predict` method\ntest_tab_folder = TabFromFolder(\n    fname=test_fname, reference=train_tab_folder, ignore_target=True\n)\n</pre> train_tab_folder = TabFromFolder(     fname=train_fname,     directory=data_path,     target_col=target_col,     preprocessor=tab_preprocessor,     text_col=text_col,     img_col=img_col, )  # Note how we can use the `train_tab_folder` as reference so we don't have to # define all parameters again eval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)  # Note that for the test set we can ignore the target as no metric will be # computed by the `predict` method test_tab_folder = TabFromFolder(     fname=test_fname, reference=train_tab_folder, ignore_target=True ) In\u00a0[10]: Copied! <pre># for the text and image datasets we do not need to specify eval or test loaders\ntext_folder = TextFromFolder(preprocessor=text_preprocessor)\nimg_folder = ImageFromFolder(preprocessor=img_preprocessor)\n</pre> # for the text and image datasets we do not need to specify eval or test loaders text_folder = TextFromFolder(preprocessor=text_preprocessor) img_folder = ImageFromFolder(preprocessor=img_preprocessor) In\u00a0[11]: Copied! <pre>train_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=train_size,\n    tab_from_folder=train_tab_folder,\n    text_from_folder=text_folder,\n    img_from_folder=img_folder,\n)\n\n# Note that the eval and test loaders only need their corresponding\n# `TabFromFolder` classes. The rest of the parameters can be defined\n# via a `reference` `TabFromFolder` class\neval_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=eval_size,\n    tab_from_folder=eval_tab_folder,\n    reference=train_dataset_folder,\n)\n\ntest_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=test_size,\n    tab_from_folder=test_tab_folder,\n    reference=train_dataset_folder,\n)\n</pre> train_dataset_folder = WideDeepDatasetFromFolder(     n_samples=train_size,     tab_from_folder=train_tab_folder,     text_from_folder=text_folder,     img_from_folder=img_folder, )  # Note that the eval and test loaders only need their corresponding # `TabFromFolder` classes. The rest of the parameters can be defined # via a `reference` `TabFromFolder` class eval_dataset_folder = WideDeepDatasetFromFolder(     n_samples=eval_size,     tab_from_folder=eval_tab_folder,     reference=train_dataset_folder, )  test_dataset_folder = WideDeepDatasetFromFolder(     n_samples=test_size,     tab_from_folder=test_tab_folder,     reference=train_dataset_folder, ) In\u00a0[12]: Copied! <pre>train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1)\neval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1)\ntest_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1)\n</pre> train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1) eval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1) test_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1) <p>And from here on is business as usual:</p> In\u00a0[13]: Copied! <pre># for example\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=32,\n    hidden_dim=64,\n    n_layers=2,\n)\n\ndeepimage = Vision()\n\ndeepdense = TabMlp(\n    mlp_hidden_dims=[32, 16],\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=cont_cols,\n)\n\nmodel = WideDeep(\n    deeptabular=deepdense,\n    deeptext=basic_rnn,\n    deepimage=deepimage,\n)\n\nmodel\n</pre> # for example basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=32,     hidden_dim=64,     n_layers=2, )  deepimage = Vision()  deepdense = TabMlp(     mlp_hidden_dims=[32, 16],     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=cont_cols, )  model = WideDeep(     deeptabular=deepdense,     deeptext=basic_rnn,     deepimage=deepimage, )  model Out[13]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_host_listings_count): Embedding(28, 10, padding_idx=0)\n          (emb_layer_neighbourhood_cleansed): Embedding(33, 11, padding_idx=0)\n          (emb_layer_is_location_exact): Embedding(3, 2, padding_idx=0)\n          (emb_layer_property_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_room_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates): Embedding(14, 7, padding_idx=0)\n          (emb_layer_bathrooms): Embedding(11, 6, padding_idx=0)\n          (emb_layer_bedrooms): Embedding(7, 4, padding_idx=0)\n          (emb_layer_beds): Embedding(11, 6, padding_idx=0)\n          (emb_layer_guests_included): Embedding(11, 6, padding_idx=0)\n          (emb_layer_minimum_nights): Embedding(25, 9, padding_idx=0)\n          (emb_layer_instant_bookable): Embedding(3, 2, padding_idx=0)\n          (emb_layer_cancellation_policy): Embedding(6, 4, padding_idx=0)\n          (emb_layer_has_house_rules): Embedding(3, 2, padding_idx=0)\n          (emb_layer_host_gender): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_guests_included_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_minimum_nights_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_host_listings_count_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_bathrooms_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_bedrooms_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_beds_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_security_deposit): Embedding(53, 15, padding_idx=0)\n          (emb_layer_extra_people): Embedding(39, 12, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=128, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=32, out_features=16, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=16, out_features=1, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(2192, 32, padding_idx=1)\n      (rnn): LSTM(32, 64, num_layers=2, batch_first=True, dropout=0.1)\n      (rnn_mlp): Identity()\n    )\n    (1): Linear(in_features=64, out_features=1, bias=True)\n  )\n  (deepimage): Sequential(\n    (0): Vision(\n      (features): Sequential(\n        (conv_layer_0): Sequential(\n          (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n          (1): BatchNorm2d(64, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (maxpool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n        )\n        (conv_layer_1): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_2): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_3): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (adaptiveavgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n        )\n      )\n    )\n    (1): Linear(in_features=512, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[14]: Copied! <pre>trainer = TrainerFromFolder(\n    model,\n    objective=\"regression\",\n)\n\ntrainer.fit(\n    train_loader=train_loader,\n    eval_loader=eval_loader,\n)\n</pre> trainer = TrainerFromFolder(     model,     objective=\"regression\", )  trainer.fit(     train_loader=train_loader,     eval_loader=eval_loader, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 50/50 [03:41&lt;00:00,  4.42s/it, loss=1.64e+4]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:23&lt;00:00,  3.30s/it, loss=6.27e+3]\n</pre> In\u00a0[15]: Copied! <pre>preds = trainer.predict(test_loader=test_loader)\n</pre> preds = trainer.predict(test_loader=test_loader) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:22&lt;00:00,  3.26s/it]\n</pre> <p>Note that in the case of predict you could also choose to do this</p> In\u00a0[16]: Copied! <pre>df_test = pd.read_csv(\"/\".join([data_path, test_fname]))\n</pre> df_test = pd.read_csv(\"/\".join([data_path, test_fname])) In\u00a0[17]: Copied! <pre># if the images for the test set fit in memory\nX_tab_test = chunk_tab_preprocessor.transform(df_test)\nX_text_test = chunk_text_preprocessor.transform(df_test)\nX_img_test = img_preprocessor.transform(df_test)\n</pre> # if the images for the test set fit in memory X_tab_test = chunk_tab_preprocessor.transform(df_test) X_text_test = chunk_text_preprocessor.transform(df_test) X_img_test = img_preprocessor.transform(df_test) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 101/101 [00:00&lt;00:00, 708.23it/s]</pre> <pre>Computing normalisation metrics\n</pre> <pre>\n</pre> In\u00a0[18]: Copied! <pre>preds = trainer.predict(\n    X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32\n)\n</pre> preds = trainer.predict(     X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32 ) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:03&lt;00:00,  1.14it/s]\n</pre>"},{"location":"examples/20_load_from_folder_functionality.html#scenario","title":"Scenario\u00b6","text":"<p>We have a tabular dataset combined with images and text and either some, or all these datasets do not fit in memory. Note that the tabular dataset MUST ALWAYS be present as it is considered the rerefence. This is, if we have an image dataset, the tabular dataset must contain a column that points to the image file names as stored in disk. Similarly, if we have a text dataset, then the tabular dataset must contain a column with the texts themselves or a the file names of the text files as stored in disk.</p> <p>If you only have text and/or images and not a tabular component, I would suggest using other libraries (such as hugginface probably).</p> <p>Within this scenario, they are two possible scenarios that we will cover here:</p> <ol> <li><p>The tabular data itsel fits in memory and is only the images that do not: in this case you could use the 'standard' <code>Preprocessors</code> in the library and off you go, move directly to the <code>[...]FromFolder</code> functionalities</p> </li> <li><p>The tabular data is also very large and does not fit in memory, so we have to process it in chuncks. For this second case I have created the so called <code>Chunk[...]Preprocessor</code> (Wide, Tab and Text).</p> </li> </ol> <p>Note that at the moment ONLY csv format is allowed for the tabular file. More formats will be supported in the future.</p> <p>Let's see a complete example to illustrate how each of these cases would be addressed with the new functionalities in the library. For this example we will use a sample of the airbnb dataset</p> <p>The airbnb dataset, which you could get from here, is too big to be included in our datasets module (when including images). Therefore, what I did was, go there, download it, and use the download_images.py script to get the images and the airbnb_data_processing.py to process the data. I did this ages ago and I believe the format of the dataset might be different now. Nonetheless, I will show samples of the dataset as we go through so you can extrapolate the content of this notebook to your particular problem.</p> <p>In the future we will find better datasets\ud83d\ude42. Finally, note that here we are only using a small sample to illustrate the use, so PLEASE ignore the results, just focus on usage.</p>"},{"location":"examples/20_load_from_folder_functionality.html#setting-variables-and-constants","title":"Setting variables and constants\u00b6","text":""},{"location":"examples/20_load_from_folder_functionality.html#step-1-the-preprocessors","title":"Step 1: the preprocessors\u00b6","text":""},{"location":"examples/20_load_from_folder_functionality.html#scenario-1-only-the-images-do-not-fit-in-disk","title":"Scenario 1: only the images do not fit in disk\u00b6","text":"<p>In this case we can prepare the data in the 'standard' way</p>"},{"location":"examples/20_load_from_folder_functionality.html#scenario-2-the-tabular-data-is-also-huge","title":"Scenario 2: the tabular data is also huge\u00b6","text":"<p>Then we need to prepare it in chunks. Note that, unfortunately, the tabular and text preprocessors need to see the whole dataset once. This is because to process tabular or text data we need to encode values. For those encodings to be consistent they need to have seen the whole dataset. Alternatively, one could code a solution with some streaming encoder for both datasets. However, such implementation is not trivial for this library (and in general). I also don't think that having to see the whole data once is such a big limitation. Let's see how is done.</p> <p>Note that I have not mentioned the image dataset. This is because the processing of the image dataset does not require any form of encoding and in consequence can be done 'on the fly'. Therefore, no <code>ChunkImgPreprocessor</code> processor is needed.</p>"},{"location":"examples/20_load_from_folder_functionality.html#step-2-the-fromfolder-classes","title":"Step 2: the <code>[...]FromFolder</code> classes\u00b6","text":"<p>Once we have the preprocessors, we need to instantiate the classes that will enable us to load the data from their respective folders. From now on I am going to proceed with the <code>chunk_tab_preprocessor</code>, <code>chunk_text_preprocessor</code> and <code>img_preprocessor</code>, but the code would be identical if instead of the first two preprocessors we decided to use the <code>tab_preprocessor</code> and <code>text_preprocessor</code>.</p> <p>Once more, our reference datasets are the tabular datasets, which we have splitted in train, eval and test prior to start the coding. Therefore, we will eventually need a loader for each split</p>"},{"location":"examples/20_load_from_folder_functionality.html#step-3-pytorch-datasets-and-dataloaders","title":"Step 3: pytorch datasets and dataloaders\u00b6","text":"<p>From here in advance, is all very 'standard' if you are familiar with pytorch. One needs to define a class that inherits from the <code>Dataset</code> class in pytorch. Then this will be passed to a <code>DataLoader</code> class and we are ready to train. Our <code>Dataset</code> child class is <code>WideDeepDatasetFromFolder</code>. This class will use the tabular dataset and the corresponding text and image columns to load the adequate data in the batches</p> <p>Let's do it</p>"},{"location":"examples/20_load_from_folder_functionality.html#step-4-define-the-model","title":"Step 4: define the model\u00b6","text":""},{"location":"examples/20_load_from_folder_functionality.html#step-5-fit-and-predict","title":"Step 5: fit and predict\u00b6","text":""},{"location":"examples/21_Using_huggingface_within_widedeep.html","title":"21-Using-huggingface-within-widedeep","text":"<p>In this notebook we will show how to use Hugginface's tokenizers and models as they are integrated within the library. In notebook number 17 you can find examples on how to code your own, custom, Hugginface (hereafter HF) model and use it in combination of any other model in the library</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import HFModel, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.preprocessing import HFPreprocessor\n</pre> import numpy as np import pandas as pd from sklearn.metrics import f1_score, accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import HFModel, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.preprocessing import HFPreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore\n</pre> df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore In\u00a0[3]: Copied! <pre>df.shape\n</pre> df.shape Out[3]: <pre>(23486, 10)</pre> In\u00a0[4]: Copied! <pre>df.sample(3)\n</pre> df.sample(3) Out[4]: Clothing ID Age Title Review Text Rating Recommended IND Positive Feedback Count Division Name Department Name Class Name 7004 862 43 Cute and feminine Loved this sweater wrap and bought it in both ... 5 1 2 General Tops Knits 12508 975 66 Love it The linen fabric is elegantly thin feels and l... 5 1 3 General Jackets Jackets 10288 950 41 Perfect for fall This sweater is just as pictured. the fit is t... 5 1 0 General Tops Sweaters In\u00a0[5]: Copied! <pre># Let's do some mild preprocessing\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n</pre> # Let's do some mild preprocessing df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\") In\u00a0[6]: Copied! <pre># drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[7]: Copied! <pre>df.shape\n</pre> df.shape Out[7]: <pre>(22608, 10)</pre> In\u00a0[8]: Copied! <pre># if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random\n# If you run this on a GPU you can comment out the following two cells\ndf.rating.value_counts()\n</pre> # if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random # If you run this on a GPU you can comment out the following two cells df.rating.value_counts() Out[8]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> In\u00a0[9]: Copied! <pre>df = (\n    df.groupby(\"rating\", group_keys=False)\n    .apply(lambda x: x.sample(min(len(x), 2369)))\n    .sample(1000)\n)\n</pre> df = (     df.groupby(\"rating\", group_keys=False)     .apply(lambda x: x.sample(min(len(x), 2369)))     .sample(1000) ) <pre>/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/ipykernel_5886/895673206.py:3: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n  .apply(lambda x: x.sample(min(len(x), 2369)))\n</pre> In\u00a0[10]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n\n# possible model names currently supported in the library\nmodel_names = [\n    \"distilbert-base-uncased\",\n    \"bert-base-uncased\",\n    \"FacebookAI/roberta-base\",\n    \"albert-base-v2\",\n    \"google/electra-base-discriminator\",\n]\n\n# Let's choose one. The syntax is the same for all the models\nmodel_name = \"distilbert-base-uncased\"\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)  # possible model names currently supported in the library model_names = [     \"distilbert-base-uncased\",     \"bert-base-uncased\",     \"FacebookAI/roberta-base\",     \"albert-base-v2\",     \"google/electra-base-discriminator\", ]  # Let's choose one. The syntax is the same for all the models model_name = \"distilbert-base-uncased\" <p>Now we can use the <code>HFPreprocessor</code> class. As most things in this library, the integration with HF has been coded aiming for a flexible use. With this in mind, there are two ways one can use a <code>HFPreprocessor</code> class.</p> <ol> <li>Passing a <code>text_col</code> and <code>encode_params</code> as the class is instantiated and then using the <code>fit</code> and <code>transform</code> as with any other preprocessor in the library</li> <li>Without passing  <code>text_col</code> and <code>encode_params</code> as the class is instantiated and using the <code>encode</code> method of the <code>HFPreprocessor</code> which is simply a wrapper around the encode method of HF's tokenizers</li> </ol> <p>Let's have a look</p> In\u00a0[11]: Copied! <pre>tokenizer1 = HFPreprocessor(\n    model_name=model_name,\n    text_col=\"review_text\",\n    num_workers=1,\n    encode_params={\n        \"max_length\": 90,\n        \"padding\": \"max_length\",\n        \"truncation\": True,\n        \"add_special_tokens\": True,\n    },\n)\n\nX_text_tr1 = tokenizer1.fit_transform(train)\nX_text_te1 = tokenizer1.transform(test)\n</pre> tokenizer1 = HFPreprocessor(     model_name=model_name,     text_col=\"review_text\",     num_workers=1,     encode_params={         \"max_length\": 90,         \"padding\": \"max_length\",         \"truncation\": True,         \"add_special_tokens\": True,     }, )  X_text_tr1 = tokenizer1.fit_transform(train) X_text_te1 = tokenizer1.transform(test) In\u00a0[12]: Copied! <pre>tokenizer2 = HFPreprocessor(\n    model_name=model_name,\n    num_workers=1,\n)\n\nX_text_tr2 = tokenizer2.encode(\n    train.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\nX_text_te2 = tokenizer2.encode(\n    test.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\n</pre> tokenizer2 = HFPreprocessor(     model_name=model_name,     num_workers=1, )  X_text_tr2 = tokenizer2.encode(     train.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) X_text_te2 = tokenizer2.encode(     test.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) In\u00a0[13]: Copied! <pre>all(X_text_tr1[0] == X_text_tr2[0])\n</pre> all(X_text_tr1[0] == X_text_tr2[0]) Out[13]: <pre>True</pre> In\u00a0[14]: Copied! <pre># Now we define a model which is as easy as:\n# Note that this will instantiation will lead to NO parameter trainable in the HF model.\n# If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument.\n# Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details)\nhf_model = HFModel(model_name=model_name)\n</pre> # Now we define a model which is as easy as: # Note that this will instantiation will lead to NO parameter trainable in the HF model. # If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument. # Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details) hf_model = HFModel(model_name=model_name) In\u00a0[15]: Copied! <pre># And from here on is the same as any other WideDeep model\nmodel = WideDeep(\n    deeptext=hf_model,\n    pred_dim=4,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"multiclass\",\n    metrics=[Accuracy(), F1Score(average=True)],\n)\n\ntrainer.fit(\n    X_text=X_text_tr2,\n    target=train.rating.values,\n    n_epochs=1,\n    batch_size=64,\n)\n# If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo\n</pre> # And from here on is the same as any other WideDeep model model = WideDeep(     deeptext=hf_model,     pred_dim=4, )  trainer = Trainer(     model,     objective=\"multiclass\",     metrics=[Accuracy(), F1Score(average=True)], )  trainer.fit(     X_text=X_text_tr2,     target=train.rating.values,     n_epochs=1,     batch_size=64, ) # If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 13/13 [02:06&lt;00:00,  9.75s/it, loss=3.2, metrics={'acc': 0.235, 'f1': 0.2336}]  \n</pre> In\u00a0[17]: Copied! <pre>preds_text = trainer.predict_proba(X_text=X_text_te2)\npred_text_class = np.argmax(preds_text, 1)\n\nacc_text = accuracy_score(test.rating, pred_text_class)\nf1_text = f1_score(test.rating, pred_text_class, average=\"weighted\")\nprint(f\"Accuracy: {acc_text:.4f}\")\nprint(f\"F1: {f1_text:.4f}\")\n</pre> preds_text = trainer.predict_proba(X_text=X_text_te2) pred_text_class = np.argmax(preds_text, 1)  acc_text = accuracy_score(test.rating, pred_text_class) f1_text = f1_score(test.rating, pred_text_class, average=\"weighted\") print(f\"Accuracy: {acc_text:.4f}\") print(f\"F1: {f1_text:.4f}\") <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:05&lt;00:00,  1.43s/it]</pre> <pre>Accuracy: 0.2500\nF1: 0.1000\n</pre> <pre>\n</pre>"},{"location":"pytorch-widedeep/bayesian_models.html","title":"The <code>bayesian models</code> module","text":"<p>This module contains the two Bayesian Models available in this library, namely the bayesian version of the <code>Wide</code> and <code>TabMlp</code> models, referred as <code>BayesianWide</code> and <code>BayesianTabMlp</code>. These models are very useful in scenarios where getting a measure of uncertainty is important.</p> <p>The models in this module are based on the publication: Weight Uncertainty in Neural Networks.</p>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide","title":"BayesianWide","text":"<pre><code>BayesianWide(\n    input_dim,\n    pred_dim=1,\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0.0,\n    posterior_rho_init=-7.0,\n)\n</code></pre> <p>             Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>Wide</code> model. This is a linear model where the non-linearlities are captured via crossed-columns</p> <p>Parameters:</p> <ul> <li> <code>input_dim</code>             (<code>int</code>)         \u2013          <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide component. For example, if the wide component receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> </li> <li> <code>pred_dim</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>size of the ouput tensor containing the predictions</p> </li> <li> <code>prior_sigma_1</code>             (<code>float</code>, default:                 <code>1.0</code> )         \u2013          <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> </li> <li> <code>prior_sigma_2</code>             (<code>float</code>, default:                 <code>0.002</code> )         \u2013          <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution</p> </li> <li> <code>prior_pi</code>             (<code>float</code>, default:                 <code>0.8</code> )         \u2013          <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution</p> </li> <li> <code>posterior_mu_init</code>             (<code>float</code>, default:                 <code>0.0</code> )         \u2013          <p>The posterior sample of the weights is defined as:</p> \\[    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} \\] <p>where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> </li> <li> <code>posterior_rho_init</code>             (<code>float</code>, default:                 <code>-7.0</code> )         \u2013          <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>bayesian_wide_linear</code>             (<code>Module</code>)         \u2013          <p>the linear layer that comprises the wide branch of the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n&gt;&gt;&gt; wide = BayesianWide(input_dim=X.unique().size(0), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code> <pre><code>def __init__(\n    self,\n    input_dim: int,\n    pred_dim: int = 1,\n    prior_sigma_1: float = 1.0,\n    prior_sigma_2: float = 0.002,\n    prior_pi: float = 0.8,\n    posterior_mu_init: float = 0.0,\n    posterior_rho_init: float = -7.0,\n):\n    super(BayesianWide, self).__init__()\n    #  Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n    self.bayesian_wide_linear = bnn.BayesianEmbedding(\n        n_embed=input_dim + 1,\n        embed_dim=pred_dim,\n        padding_idx=0,\n        prior_sigma_1=prior_sigma_1,\n        prior_sigma_2=prior_sigma_2,\n        prior_pi=prior_pi,\n        posterior_mu_init=posterior_mu_init,\n        posterior_rho_init=posterior_rho_init,\n    )\n    self.bias = nn.Parameter(torch.zeros(pred_dim))\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp","title":"BayesianTabMlp","text":"<pre><code>BayesianTabMlp(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    embed_continuous=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    use_cont_bias=None,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[200, 100],\n    mlp_activation=\"leaky_relu\",\n    prior_sigma_1=1,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0.0,\n    posterior_rho_init=-7.0,\n    pred_dim=1\n)\n</code></pre> <p>             Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>BayesianTabMlp</code> model.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of probabilistic dense layers (i.e. a MLP).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm', 'batchnorm' or None.</p> </li> <li> <code>embed_continuous</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the continuous columns will be embedded (i.e. passed each through a linear layer with or without activation)</p> </li> <li> <code>cont_embed_dim</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>Size of the continuous embeddings</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings</p> </li> <li> <code>use_cont_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the continuous embeddings</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>             (<code>List[int]</code>, default:                 <code>[200, 100]</code> )         \u2013          <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>             (<code>str</code>, default:                 <code>'leaky_relu'</code> )         \u2013          <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>prior_sigma_1</code>             (<code>float</code>, default:                 <code>1</code> )         \u2013          <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> </li> <li> <code>prior_sigma_2</code>             (<code>float</code>, default:                 <code>0.002</code> )         \u2013          <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution for each Bayesian linear and embedding layer</p> </li> <li> <code>prior_pi</code>             (<code>float</code>, default:                 <code>0.8</code> )         \u2013          <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution ffor each Bayesian linear and embedding layer</p> </li> <li> <code>posterior_mu_init</code>             (<code>float</code>, default:                 <code>0.0</code> )         \u2013          <p>The posterior sample of the weights is defined as:</p> <p>$$    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} $$ where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> </li> <li> <code>posterior_rho_init</code>             (<code>float</code>, default:                 <code>-7.0</code> )         \u2013          <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>bayesian_cat_and_cont_embed</code>             (<code>Module</code>)         \u2013          <p>This is the module that processes the categorical and continuous columns</p> </li> <li> <code>bayesian_tab_mlp</code>             (<code>Sequential</code>)         \u2013          <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    embed_continuous: Optional[bool] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    use_cont_bias: Optional[bool] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    mlp_hidden_dims: List[int] = [200, 100],\n    mlp_activation: str = \"leaky_relu\",\n    prior_sigma_1: float = 1,\n    prior_sigma_2: float = 0.002,\n    prior_pi: float = 0.8,\n    posterior_mu_init: float = 0.0,\n    posterior_rho_init: float = -7.0,\n    pred_dim=1,  # Bayesian models will require their own trainer and need the output layer\n):\n    super(BayesianTabMlp, self).__init__()\n\n    self.column_idx = column_idx\n    self.cat_embed_input = cat_embed_input\n    self.cat_embed_activation = cat_embed_activation\n\n    self.continuous_cols = continuous_cols\n    self.cont_norm_layer = cont_norm_layer\n    self.embed_continuous = embed_continuous\n    self.cont_embed_dim = cont_embed_dim\n    self.cont_embed_dropout = cont_embed_dropout\n    self.use_cont_bias = use_cont_bias\n    self.cont_embed_activation = cont_embed_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n\n    self.prior_sigma_1 = prior_sigma_1\n    self.prior_sigma_2 = prior_sigma_2\n    self.prior_pi = prior_pi\n    self.posterior_mu_init = posterior_mu_init\n    self.posterior_rho_init = posterior_rho_init\n\n    self.pred_dim = pred_dim\n\n    allowed_activations = [\"relu\", \"leaky_relu\", \"tanh\", \"gelu\"]\n    if self.mlp_activation not in allowed_activations:\n        raise ValueError(\n            \"Currently, only the following activation functions are supported \"\n            \"for the Bayesian MLP's dense layers: {}. Got '{}' instead\".format(\n                \", \".join(allowed_activations),\n                self.mlp_activation,\n            )\n        )\n\n    # Categorical\n    if self.cat_embed_input is not None:\n        self.cat_embed = BayesianDiffSizeCatEmbeddings(\n            column_idx=self.column_idx,\n            embed_input=self.cat_embed_input,\n            prior_sigma_1=self.prior_sigma_1,\n            prior_sigma_2=self.prior_sigma_2,\n            prior_pi=self.prior_pi,\n            posterior_mu_init=self.posterior_mu_init,\n            posterior_rho_init=self.posterior_rho_init,\n            activation_fn=self.cat_embed_activation,\n        )\n        self.cat_out_dim = int(np.sum([embed[2] for embed in self.cat_embed_input]))\n    else:\n        self.cat_out_dim = 0\n\n    # Continuous\n    if self.continuous_cols is not None:\n        self.cont_idx = [self.column_idx[col] for col in self.continuous_cols]\n        if cont_norm_layer == \"layernorm\":\n            self.cont_norm: NormLayers = nn.LayerNorm(len(self.continuous_cols))\n        elif cont_norm_layer == \"batchnorm\":\n            self.cont_norm = nn.BatchNorm1d(len(self.continuous_cols))\n        else:\n            self.cont_norm = nn.Identity()\n        if self.embed_continuous:\n            assert self.cont_embed_dim is not None, (\n                \"If 'embed_continuous' is True, 'cont_embed_dim' must be \"\n                \"provided\"\n            )\n            self.cont_embed = BayesianContEmbeddings(\n                n_cont_cols=len(self.continuous_cols),\n                embed_dim=self.cont_embed_dim,\n                prior_sigma_1=self.prior_sigma_1,\n                prior_sigma_2=self.prior_sigma_2,\n                prior_pi=self.prior_pi,\n                posterior_mu_init=self.posterior_mu_init,\n                posterior_rho_init=self.posterior_rho_init,\n                use_bias=(\n                    False if self.use_cont_bias is None else self.use_cont_bias\n                ),\n                activation_fn=self.cont_embed_activation,\n            )\n            self.cont_out_dim = len(self.continuous_cols) * self.cont_embed_dim\n        else:\n            self.cont_out_dim = len(self.continuous_cols)\n    else:\n        self.cont_out_dim = 0\n\n    self.output_dim = self.cat_out_dim + self.cont_out_dim\n\n    mlp_hidden_dims = [self.output_dim] + mlp_hidden_dims + [pred_dim]\n    self.bayesian_tab_mlp = BayesianMLP(\n        mlp_hidden_dims,\n        mlp_activation,\n        True,  # use_bias\n        prior_sigma_1,\n        prior_sigma_2,\n        prior_pi,\n        posterior_mu_init,\n        posterior_rho_init,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html","title":"Training Deep Learning Probabilistic Models","text":""},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer","title":"BayesianTrainer","text":"<pre><code>BayesianTrainer(\n    model,\n    objective,\n    custom_loss_function=None,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    metrics=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>BaseBayesianTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Both the Bayesian models and the Trainer in this repo are based on the paper: Weight Uncertainty in Neural Networks.</p> <p>Parameters:</p> <ul> <li> <code>model</code>             (<code>BaseBayesianModel</code>)         \u2013          <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code> section here in the docs.</p> </li> <li> <code>objective</code>             (<code>str</code>)         \u2013          <p>Defines the objective, loss or cost function. Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code> Possible values are: 'binary', 'multiclass', 'regression'</p> </li> <li> <code>custom_loss_function</code>             (<code>Optional[Module]</code>, default:                 <code>None</code> )         \u2013          <p>If none of the loss functions available suits the user, it is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples folder in the repo.</p> </li> <li> <code>optimizer</code>             (<code>Optional[Optimizer]</code>, default:                 <code>None</code> )         \u2013          <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>             (<code>Optional[LRScheduler]</code>, default:                 <code>None</code> )         \u2013          <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>             (<code>Optional[List[Callback]]</code>, default:                 <code>None</code> )         \u2013          <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>metrics</code>             (<code>Optional[Union[List[Metric], List[Metric]]]</code>, default:                 <code>None</code> )         \u2013          <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as   long as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the Examples folder in the repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library Examples   classification-metrics&gt;<code>_. It can also be a torchmetric custom metric as   long as it is an object of type</code>Metric<code>.   See</code>the instructions</li> </ul> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>         \u2013          <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cyclic_lr</code>             (<code>bool</code>)         \u2013          <p>Attribute that indicates if  the lr_scheduler is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See <code>Pytorch schedulers &lt;https://pytorch.org/docs/stable/optim.html&gt;</code>_.</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>@alias(  # noqa: C901\n    \"objective\",\n    [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n)\ndef __init__(\n    self,\n    model: BaseBayesianModel,\n    objective: str,\n    custom_loss_function: Optional[Module] = None,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        objective=objective,\n        custom_loss_function=custom_loss_function,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        metrics=metrics,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.fit","title":"fit","text":"<pre><code>fit(\n    X_tab,\n    target,\n    X_tab_val=None,\n    target_val=None,\n    val_split=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    n_train_samples=2,\n    n_val_samples=2,\n)\n</code></pre> <p>Fit method.</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>             (<code>ndarray</code>)         \u2013          <p>tabular dataset</p> </li> <li> <code>target</code>             (<code>ndarray</code>)         \u2013          <p>target values</p> </li> <li> <code>X_tab_val</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>validation data</p> </li> <li> <code>target_val</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>validation target values</p> </li> <li> <code>val_split</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>n_epochs</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>number of epochs</p> </li> <li> <code>validation_freq</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>epochs validation frequency</p> </li> <li> <code>batch_size</code>             (<code>int</code>, default:                 <code>32</code> )         \u2013          <p>batch size</p> </li> <li> <code>n_train_samples</code>             (<code>int</code>, default:                 <code>2</code> )         \u2013          <p>number of samples to average over during the training process. See Weight Uncertainty in Neural Networks for details.</p> </li> <li> <code>n_val_samples</code>             (<code>int</code>, default:                 <code>2</code> )         \u2013          <p>number of samples to average over during the validation process. See Weight Uncertainty in Neural Networks for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def fit(  # noqa: C901\n    self,\n    X_tab: np.ndarray,\n    target: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    target_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    n_train_samples: int = 2,\n    n_val_samples: int = 2,\n):\n    r\"\"\"Fit method.\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    target: np.ndarray\n        target values\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    target_val: np.ndarray, Optional, default = None\n        validation target values\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    n_train_samples: int, default=2\n        number of samples to average over during the training process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    n_val_samples: int, default=2\n        number of samples to average over during the validation process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = tabular_train_val_split(\n        self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n    )\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (X, y) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(\n                    X, y, n_train_samples, train_steps, batch_idx\n                )\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (X, y) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(\n                        X, y, n_val_samples, train_steps, i\n                    )\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict","title":"predict","text":"<pre><code>predict(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predictions</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>             (<code>ndarray</code>)         \u2013          <p>tabular dataset</p> </li> <li> <code>n_samples</code>             (<code>int</code>, default:                 <code>5</code> )         \u2013          <p>number of samples that will be either returned or averaged to produce an overal prediction</p> </li> <li> <code>return_samples</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the n samples will be averaged or directly returned</p> </li> <li> <code>batch_size</code>             (<code>int</code>, default:                 <code>256</code> )         \u2013          <p>batch size</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>         \u2013          <p>array with the predictions</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n    axis = 2 if return_samples else 1\n\n    if self.objective == \"regression\":\n        return preds.squeeze(axis)\n    if self.objective == \"binary\":\n        return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n    if self.objective == \"multiclass\":\n        return np.argmax(preds, axis)\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predicted probabilities</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>             (<code>ndarray</code>)         \u2013          <p>tabular dataset</p> </li> <li> <code>n_samples</code>             (<code>int</code>, default:                 <code>5</code> )         \u2013          <p>number of samples that will be either returned or averaged to produce an overal prediction</p> </li> <li> <code>return_samples</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the n samples will be averaged or directly returned</p> </li> <li> <code>batch_size</code>             (<code>int</code>, default:                 <code>256</code> )         \u2013          <p>batch size</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>array with the probabilities per class</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict_proba(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n    if self.objective == \"binary\":\n        if return_samples:\n            preds = preds.squeeze(2)\n            probs = np.zeros([n_samples, preds.shape[1], 2])\n            for i in range(n_samples):\n                probs[i, :, 0] = 1 - preds[i]\n                probs[i, :, 1] = preds[i]\n        else:\n            preds = preds.squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n        return probs\n    if self.objective == \"multiclass\":\n        return preds\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"bayesian_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers or learning rate schedulers) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>Parameters:</p> <ul> <li> <code>path</code>             (<code>str</code>)         \u2013          <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> </li> <li> <code>model_filename</code>             (<code>str</code>, default:                 <code>'bayesian_model.pt'</code> )         \u2013          <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"bayesian_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers or\n    learning rate schedulers) have to be defined externally and then\n    passed to the `Trainer`. As a result, the `Trainer` does not\n    generate any attribute or additional data products that need to be\n    saved other than the `model` object itself, which can be saved as\n    any other torch model (e.g. `torch.save(model, path)`).\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.model.state_dict(), model_path)\n    else:\n        torch.save(self.model, model_path)\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html","title":"Callbacks","text":"<p>Here are the 4 callbacks available to the user in <code>pytorch-widedepp</code>: <code>LRHistory</code>, <code>ModelCheckpoint</code>, <code>EarlyStopping</code> and <code>RayTuneReporter</code>.</p> <p> NOTE: other callbacks , like <code>History</code>, run always  by default. In particular, the <code>History</code> callback saves the metrics in the  <code>history</code> attribute of the <code>Trainer</code>.</p>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.LRHistory","title":"LRHistory","text":"<pre><code>LRHistory(n_epochs)\n</code></pre> <p>             Bases: <code>Callback</code></p> <p>Saves the learning rates during training in the <code>lr_history</code> attribute of the <code>Trainer</code>.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>n_epochs</code>             (<code>int</code>)         \u2013          <p>number of training epochs</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(self, n_epochs: int):\n    super(LRHistory, self).__init__()\n    self.n_epochs = n_epochs\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.ModelCheckpoint","title":"ModelCheckpoint","text":"<pre><code>ModelCheckpoint(\n    filepath=None,\n    monitor=\"val_loss\",\n    min_delta=0.0,\n    verbose=0,\n    save_best_only=False,\n    mode=\"auto\",\n    period=1,\n    max_save=-1,\n)\n</code></pre> <p>             Bases: <code>Callback</code></p> <p>Saves the model after every epoch.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>filepath</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Full path to save the output weights. It must contain only the root of the filenames. Epoch number and <code>.pt</code> extension (for pytorch) will be added. e.g. <code>filepath=\"path/to/output_weights/weights_out\"</code> And the saved files in that directory will be named: 'weights_out_1.pt', 'weights_out_2.pt', .... If set to <code>None</code> the class just report best metric and best_epoch.</p> </li> <li> <code>monitor</code>             (<code>str</code>, default:                 <code>'val_loss'</code> )         \u2013          <p>quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> </li> <li> <code>min_delta</code>             (<code>float</code>, default:                 <code>0.0</code> )         \u2013          <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>0</code> )         \u2013          <p>verbosity mode</p> </li> <li> <code>save_best_only</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>the latest best model according to the quantity monitored will not be overwritten.</p> </li> <li> <code>mode</code>             (<code>str</code>, default:                 <code>'auto'</code> )         \u2013          <p>If <code>save_best_only=True</code>, the decision to overwrite the current save file is made based on either the maximization or the minimization of the monitored quantity. For 'acc', this should be 'max', for 'loss' this should be 'min', etc. In 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> </li> <li> <code>period</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Interval (number of epochs) between checkpoints.</p> </li> <li> <code>max_save</code>             (<code>int</code>, default:                 <code>-1</code> )         \u2013          <p>Maximum number of outputs to save. If -1 will save all outputs</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>best</code>             (<code>float</code>)         \u2013          <p>best metric</p> </li> <li> <code>best_epoch</code>             (<code>int</code>)         \u2013          <p>best epoch</p> </li> <li> <code>best_state_dict</code>             (<code>dict</code>)         \u2013          <p>best model state dictionary. To restore model to its best state use <code>Trainer.model.load_state_dict (model_checkpoint.best_state_dict)</code> where <code>model_checkpoint</code> is an instance of the class <code>ModelCheckpoint</code>. See the Examples folder in the repo or the Examples section in this documentation for details</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(\n    self,\n    filepath: Optional[str] = None,\n    monitor: str = \"val_loss\",\n    min_delta: float = 0.0,\n    verbose: int = 0,\n    save_best_only: bool = False,\n    mode: str = \"auto\",\n    period: int = 1,\n    max_save: int = -1,\n):\n    super(ModelCheckpoint, self).__init__()\n\n    self.filepath = filepath\n    self.monitor = monitor\n    self.min_delta = min_delta\n    self.verbose = verbose\n    self.save_best_only = save_best_only\n    self.mode = mode\n    self.period = period\n    self.max_save = max_save\n\n    self.epochs_since_last_save = 0\n\n    if self.filepath:\n        if len(self.filepath.split(\"/\")[:-1]) == 0:\n            raise ValueError(\n                \"'filepath' must be the full path to save the output weights,\"\n                \" including the root of the filenames. e.g. 'checkpoints/weights_out'\"\n            )\n\n        root_dir = (\"/\").join(self.filepath.split(\"/\")[:-1])\n        if not os.path.exists(root_dir):\n            os.makedirs(root_dir)\n\n    if self.max_save &gt; 0:\n        self.old_files: List[str] = []\n\n    if self.mode not in [\"auto\", \"min\", \"max\"]:\n        warnings.warn(\n            \"ModelCheckpoint mode %s is unknown, \"\n            \"fallback to auto mode.\" % (self.mode),\n            RuntimeWarning,\n        )\n        self.mode = \"auto\"\n    if self.mode == \"min\":\n        self.monitor_op = np.less\n        self.best = np.Inf\n    elif self.mode == \"max\":\n        self.monitor_op = np.greater  # type: ignore[assignment]\n        self.best = -np.Inf\n    else:\n        if _is_metric(self.monitor):\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            self.monitor_op = np.less\n            self.best = np.Inf\n\n    if self.monitor_op == np.greater:\n        self.min_delta *= 1\n    else:\n        self.min_delta *= -1\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.EarlyStopping","title":"EarlyStopping","text":"<pre><code>EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=0.0,\n    patience=10,\n    verbose=0,\n    mode=\"auto\",\n    baseline=None,\n    restore_best_weights=False,\n)\n</code></pre> <p>             Bases: <code>Callback</code></p> <p>Stop training when a monitored quantity has stopped improving.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>monitor</code>             (<code>str</code>, default:                 <code>'val_loss'</code> )         \u2013          <p>Quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> </li> <li> <code>min_delta</code>             (<code>float</code>, default:                 <code>0.0</code> )         \u2013          <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> </li> <li> <code>patience</code>             (<code>int</code>, default:                 <code>10</code> )         \u2013          <p>Number of epochs that produced the monitored quantity with no improvement after which training will be stopped.</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>0</code> )         \u2013          <p>verbosity mode.</p> </li> <li> <code>mode</code>             (<code>str</code>, default:                 <code>'auto'</code> )         \u2013          <p>one of {'auto', 'min', 'max'}. In 'min' mode, training will stop when the quantity monitored has stopped decreasing; in 'max' mode it will stop when the quantity monitored has stopped increasing; in 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> </li> <li> <code>baseline</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Baseline value for the monitored quantity to reach. Training will stop if the model does not show improvement over the baseline.</p> </li> <li> <code>restore_best_weights</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Whether to restore model weights from the epoch with the best value of the monitored quantity. If <code>False</code>, the model weights obtained at the last step of training are used.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>best</code>             (<code>float</code>)         \u2013          <p>best metric</p> </li> <li> <code>stopped_epoch</code>             (<code>int</code>)         \u2013          <p>epoch when the training stopped</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(\n    self,\n    monitor: str = \"val_loss\",\n    min_delta: float = 0.0,\n    patience: int = 10,\n    verbose: int = 0,\n    mode: str = \"auto\",\n    baseline: Optional[float] = None,\n    restore_best_weights: bool = False,\n):\n    super(EarlyStopping, self).__init__()\n\n    self.monitor = monitor\n    self.min_delta = min_delta\n    self.patience = patience\n    self.verbose = verbose\n    self.mode = mode\n    self.baseline = baseline\n    self.restore_best_weights = restore_best_weights\n\n    self.wait = 0\n    self.stopped_epoch = 0\n    self.state_dict = None\n\n    if self.mode not in [\"auto\", \"min\", \"max\"]:\n        warnings.warn(\n            \"EarlyStopping mode %s is unknown, \"\n            \"fallback to auto mode.\" % self.mode,\n            RuntimeWarning,\n        )\n        self.mode = \"auto\"\n\n    if self.mode == \"min\":\n        self.monitor_op = np.less\n    elif self.mode == \"max\":\n        self.monitor_op = np.greater  # type: ignore[assignment]\n    else:\n        if _is_metric(self.monitor):\n            self.monitor_op = np.greater  # type: ignore[assignment]\n        else:\n            self.monitor_op = np.less\n\n    if self.monitor_op == np.greater:\n        self.min_delta *= 1\n    else:\n        self.min_delta *= -1\n</code></pre>"},{"location":"pytorch-widedeep/dataloaders.html","title":"Dataloaders","text":"<p> NOTE: This module should contain custom dataloaders  that the user might want to implement. At the moment <code>pytorch-widedeep</code>  offers one custom dataloader, <code>DataLoaderImbalanced</code>.</p>"},{"location":"pytorch-widedeep/dataloaders.html#pytorch_widedeep.dataloaders.DataLoaderImbalanced","title":"DataLoaderImbalanced","text":"<pre><code>DataLoaderImbalanced(\n    dataset, batch_size, num_workers, **kwargs\n)\n</code></pre> <p>             Bases: <code>DataLoader</code></p> <p>Class to load and shuffle batches with adjusted weights for imbalanced datasets. If the classes do not begin from 0 remapping is necessary. See here.</p> <p>Parameters:</p> <ul> <li> <code>dataset</code>             (<code>WideDeepDataset</code>)         \u2013          <p>see <code>pytorch_widedeep.training._wd_dataset</code></p> </li> <li> <code>batch_size</code>             (<code>int</code>)         \u2013          <p>size of batch</p> </li> <li> <code>num_workers</code>             (<code>int</code>)         \u2013          <p>number of workers</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>         \u2013          <p>This can include any parameter that can be passed to the 'standard' pytorch DataLoader and that is not already explicitely passed to the class. In addition, the dictionary can also include the extra parameter <code>oversample_mul</code> which will multiply the number of samples of the minority class to be sampled by the <code>WeightedRandomSampler</code>.</p> <p>In other words, the <code>num_samples</code> param in <code>WeightedRandomSampler</code> will be defined as:</p> \\[ minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul \\] </li> </ul> Source code in <code>pytorch_widedeep/dataloaders.py</code> <pre><code>def __init__(\n    self, dataset: WideDeepDataset, batch_size: int, num_workers: int, **kwargs\n):\n    assert dataset.Y is not None, (\n        \"The 'dataset' instance of WideDeepDataset must contain a \"\n        \"target array 'Y'\"\n    )\n\n    self.with_lds = dataset.with_lds\n    if \"oversample_mul\" in kwargs:\n        oversample_mul = kwargs[\"oversample_mul\"]\n        del kwargs[\"oversample_mul\"]\n    else:\n        oversample_mul = 1\n    weights, minor_cls_cnt, num_clss = get_class_weights(dataset)\n    num_samples = int(minor_cls_cnt * num_clss * oversample_mul)\n    samples_weight = list(np.array([weights[i] for i in dataset.Y]))\n    sampler = WeightedRandomSampler(samples_weight, num_samples, replacement=True)\n    super().__init__(\n        dataset, batch_size, num_workers=num_workers, sampler=sampler, **kwargs\n    )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html","title":"The <code>load_from_folder</code> module","text":"<p>The <code>load_from_folder</code> module contains the classes that are necessary to load data from disk and these are inspired by the <code>ImageFolder</code> class in the <code>torchvision</code> library. This module is designed with one specific case in mind. Such case is the following: given a multi-modal dataset with tabular data, images and text, the images do not fit in memory, and therefore, they have to be loaded from disk. However, as any other functionality in this library, there is some flexibility and some additional cases can also be addressed using this module.</p> <p>For this module to be used, the datasets must be prepared in a certain way:</p> <ol> <li> <p>the tabular data must contain a column with the images names as stored in disk, including the extension (<code>.jpg</code>, <code>.png</code>, etc...).</p> </li> <li> <p>Regarding to the text dataset, the tabular data can contain a column with the texts themselves or the names of the files containing the texts as stored in disk.</p> </li> </ol> <p>The tabular data might or might not fit in disk itself. If it does not, please see the <code>ChunkPreprocessor</code> utilities at the[<code>preprocessing</code>] (preprocessing.md) module and the examples folder in the repo, which illustrate such case. Finally note that only <code>csv</code> format is currently supported in that case(more formats coming soon).</p>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder","title":"TabFromFolder","text":"<pre><code>TabFromFolder(\n    fname,\n    directory=None,\n    target_col=None,\n    preprocessor=None,\n    text_col=None,\n    img_col=None,\n    ignore_target=False,\n    reference=None,\n    verbose=1,\n)\n</code></pre> <p>This class is used to load tabular data from disk. The current constrains are:</p> <ol> <li>The only file format supported right now is csv</li> <li>The csv file must contain headers</li> </ol> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>fname</code>             (<code>str</code>)         \u2013          <p>the name of the csv file</p> </li> <li> <code>directory</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>the path to the directory where the csv file is located. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>target_col</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>the name of the target column. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>preprocessor</code>             (<code>Optional[TabularPreprocessor]</code>, default:                 <code>None</code> )         \u2013          <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>text_col</code>             (<code>Optional[Union[str, List[str]]]</code>, default:                 <code>None</code> )         \u2013          <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>img_col</code>             (<code>Optional[Union[str, List[str]]]</code>, default:                 <code>None</code> )         \u2013          <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>ignore_target</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>whether to ignore the target column. This is normally set to True when this class is used for a test dataset.</p> </li> <li> <code>reference</code>             (<code>Optional[Any]</code>, default:                 <code>None</code> )         \u2013          <p>a reference <code>TabFromFolder</code> object. If provided, the <code>TabFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>TabFromFolder</code> object for evaluation or test purposes</p> </li> <li> <code>verbose</code>             (<code>Optional[int]</code>, default:                 <code>1</code> )         \u2013          <p>verbosity. If 0, no output will be printed during the process.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def __init__(\n    self,\n    fname: str,\n    directory: Optional[str] = None,\n    target_col: Optional[str] = None,\n    preprocessor: Optional[TabularPreprocessor] = None,\n    text_col: Optional[Union[str, List[str]]] = None,\n    img_col: Optional[Union[str, List[str]]] = None,\n    ignore_target: bool = False,\n    reference: Optional[Any] = None,  # is Type[\"TabFromFolder\"],\n    verbose: Optional[int] = 1,\n):\n    self.fname = fname\n    self.ignore_target = ignore_target\n    self.verbose = verbose\n\n    if reference is not None:\n        (\n            self.directory,\n            self.target_col,\n            self.preprocessor,\n            self.text_col,\n            self.img_col,\n        ) = self._set_from_reference(reference, preprocessor)\n    else:\n        assert (\n            directory is not None\n            and (target_col is not None and not ignore_target)\n            and preprocessor is not None\n        ), (\n            \"if no reference is provided, 'directory', 'target_col' and 'preprocessor' \"\n            \"must be provided\"\n        )\n\n        self.directory = directory\n        self.target_col = target_col\n        self.preprocessor = preprocessor\n        self.text_col = text_col\n        self.img_col = img_col\n\n    assert (\n        self.preprocessor.is_fitted\n    ), \"The preprocessor must be fitted before passing it to this class\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.WideFromFolder","title":"WideFromFolder","text":"<pre><code>WideFromFolder(\n    fname,\n    directory=None,\n    target_col=None,\n    preprocessor=None,\n    text_col=None,\n    img_col=None,\n    ignore_target=False,\n    reference=None,\n    verbose=1,\n)\n</code></pre> <p>             Bases: <code>TabFromFolder</code></p> <p>This class is mostly identical to <code>TabFromFolder</code> but exists because we want to separate the treatment of the wide and the deep tabular components</p> <p>Parameters:</p> <ul> <li> <code>fname</code>             (<code>str</code>)         \u2013          <p>the name of the csv file</p> </li> <li> <code>directory</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>the path to the directory where the csv file is located. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>target_col</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>the name of the target column. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>preprocessor</code>             (<code>Optional[TabularPreprocessor]</code>, default:                 <code>None</code> )         \u2013          <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>text_col</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>WideFromFolder</code> reference object must be provided=</p> </li> <li> <code>img_col</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>ignore_target</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>whether to ignore the target column. This is normally used when this class is used for a test dataset.</p> </li> <li> <code>reference</code>             (<code>Optional[Any]</code>, default:                 <code>None</code> )         \u2013          <p>a reference <code>WideFromFolder</code> object. If provided, the <code>WideFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>WideFromFolder</code> object for evaluation or test purposes</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>verbosity. If 0, no output will be printed during the process.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def __init__(\n    self,\n    fname: str,\n    directory: Optional[str] = None,\n    target_col: Optional[str] = None,\n    preprocessor: Optional[TabularPreprocessor] = None,\n    text_col: Optional[str] = None,\n    img_col: Optional[str] = None,\n    ignore_target: bool = False,\n    reference: Optional[Any] = None,  # is Type[\"WideFromFolder\"],\n    verbose: int = 1,\n):\n    super(WideFromFolder, self).__init__(\n        fname=fname,\n        directory=directory,\n        target_col=target_col,\n        preprocessor=preprocessor,\n        text_col=text_col,\n        img_col=img_col,\n        reference=reference,\n        ignore_target=ignore_target,\n        verbose=verbose,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder","title":"TextFromFolder","text":"<pre><code>TextFromFolder(preprocessor)\n</code></pre> <p>This class is used to load the text dataset (i.e. the text files) from a folder, or to retrieve the text given a texts column specified within the preprocessor object.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>preprocessor</code>             (<code>Union[TextPreprocessor, ChunkTextPreprocessor, HFPreprocessor, ChunkHFPreprocessor, List[TextPreprocessor], List[ChunkTextPreprocessor], List[HFPreprocessor], List[ChunkHFPreprocessor]]</code>)         \u2013          <p>The preprocessor used to process the text. It must be fitted before using this class</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>def __init__(\n    self,\n    preprocessor: Union[\n        TextPreprocessor,\n        ChunkTextPreprocessor,\n        HFPreprocessor,\n        ChunkHFPreprocessor,\n        List[TextPreprocessor],\n        List[ChunkTextPreprocessor],\n        List[HFPreprocessor],\n        List[ChunkHFPreprocessor],\n    ],\n):\n    if isinstance(preprocessor, list):\n        for p in preprocessor:\n            assert (\n                p.is_fitted\n            ), \"All preprocessors must be fitted before using this class\"\n    else:\n        assert (\n            preprocessor.is_fitted\n        ), \"The preprocessor must be fitted before using this class\"\n\n    self.preprocessor = preprocessor\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder","title":"ImageFromFolder","text":"<pre><code>ImageFromFolder(\n    directory=None,\n    preprocessor=None,\n    loader=default_loader,\n    extensions=None,\n    transforms=None,\n)\n</code></pre> <p>This class is used to load the image dataset from disk. It is inspired by the <code>ImageFolder</code> class at the <code>torchvision</code> library. Here, we have simply adapted to work within the context of a Wide and Deep multi-modal model.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>directory</code>             (<code>Optional[Union[str, List[str]]]</code>, default:                 <code>None</code> )         \u2013          <p>the path to the directory where the images are located. If None, a preprocessor must be provided.</p> </li> <li> <code>preprocessor</code>             (<code>Optional[Union[ImagePreprocessor, List[ImagePreprocessor]]]</code>, default:                 <code>None</code> )         \u2013          <p>a fitted <code>ImagePreprocessor</code> object.</p> </li> <li> <code>loader</code>             (<code>Callable[[str], Any]</code>, default:                 <code>default_loader</code> )         \u2013          <p>a function to load a sample given its path.</p> </li> <li> <code>extensions</code>             (<code>Optional[Tuple[str, ...]]</code>, default:                 <code>None</code> )         \u2013          <p>a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be used where IMG_EXTENSIONS =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"</p> </li> <li> <code>transforms</code>             (<code>Optional[Any]</code>, default:                 <code>None</code> )         \u2013          <p>a <code>torchvision.transforms</code> object. If None, this class will simply return an array representation of the PIL Image</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>def __init__(\n    self,\n    directory: Optional[Union[str, List[str]]] = None,\n    preprocessor: Optional[\n        Union[ImagePreprocessor, List[ImagePreprocessor]]\n    ] = None,\n    loader: Callable[[str], Any] = default_loader,\n    extensions: Optional[Tuple[str, ...]] = None,\n    transforms: Optional[Any] = None,\n) -&gt; None:\n    assert (\n        directory is not None or preprocessor is not None\n    ), \"Either a directory or an instance of ImagePreprocessor(s) must be provided\"\n\n    if directory is not None and preprocessor is not None:  # pragma: no cover\n        error_msg = (\n            \"If both 'directory' and 'preprocessor' are provided, the 'img_path' \"\n            \"attribute of the 'preprocessor' must be the same as the 'directory'\"\n        )\n        if isinstance(directory, list):\n            assert isinstance(preprocessor, list)\n            assert len(directory) == len(preprocessor)\n            for d, p in zip(directory, preprocessor):\n                assert d == p.img_path, error_msg\n        else:\n            assert isinstance(preprocessor, ImagePreprocessor)\n            assert directory == preprocessor.img_path, error_msg\n\n    if directory is not None:\n        self.directory = directory\n    else:\n        assert (\n            preprocessor is not None\n        ), \"Either a directory or an instance of ImagePreprocessor must be provided\"\n        if isinstance(preprocessor, list):\n            self.directory = [p.img_path for p in preprocessor]\n        else:\n            self.directory = preprocessor.img_path\n\n    self.preprocessor = preprocessor\n    self.loader = loader\n    self.extensions = extensions if extensions is not None else IMG_EXTENSIONS\n    self.transforms = transforms\n    if self.transforms:\n        self.transforms_names = [\n            tr.__class__.__name__ for tr in self.transforms.transforms\n        ]\n    else:\n        self.transforms_names = []\n\n        self.transpose = True\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.wd_dataset_from_folder.WideDeepDatasetFromFolder","title":"WideDeepDatasetFromFolder","text":"<pre><code>WideDeepDatasetFromFolder(\n    n_samples,\n    tab_from_folder=None,\n    wide_from_folder=None,\n    text_from_folder=None,\n    img_from_folder=None,\n    reference=None,\n)\n</code></pre> <p>             Bases: <code>Dataset</code></p> <p>This class is the Dataset counterpart of the <code>WideDeepDataset</code> class.</p> <p>Given a reference tabular dataset, with columns that indicate the path to the images and to the text files or the texts themselves, it will use the <code>[...]FromFolder</code> classes to load the data consistently from disk per batch.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>n_samples</code>             (<code>int</code>)         \u2013          <p>Number of samples in the dataset</p> </li> <li> <code>tab_from_folder</code>             (<code>Optional[TabFromFolder]</code>, default:                 <code>None</code> )         \u2013          <p>Instance of the <code>TabFromFolder</code> class</p> </li> <li> <code>wide_from_folder</code>             (<code>Optional[WideFromFolder]</code>, default:                 <code>None</code> )         \u2013          <p>Instance of the <code>WideFromFolder</code> class</p> </li> <li> <code>text_from_folder</code>             (<code>Optional[TextFromFolder]</code>, default:                 <code>None</code> )         \u2013          <p>Instance of the <code>TextFromFolder</code> class</p> </li> <li> <code>img_from_folder</code>             (<code>Optional[ImageFromFolder]</code>, default:                 <code>None</code> )         \u2013          <p>Instance of the <code>ImageFromFolder</code> class</p> </li> <li> <code>reference</code>             (<code>Optional[Any]</code>, default:                 <code>None</code> )         \u2013          <p>If not None, the 'text_from_folder' and 'img_from_folder' objects will be retrieved from the reference class. This is useful when we want to use a <code>WideDeepDatasetFromFolder</code> class used for a train dataset as a reference for the validation and test datasets. In this case, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be the same for all three datasets, so there is no need to create a new instance for each dataset.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py</code> <pre><code>def __init__(\n    self,\n    n_samples: int,\n    tab_from_folder: Optional[TabFromFolder] = None,\n    wide_from_folder: Optional[WideFromFolder] = None,\n    text_from_folder: Optional[TextFromFolder] = None,\n    img_from_folder: Optional[ImageFromFolder] = None,\n    reference: Optional[Any] = None,  # is Type[\"WideDeepDatasetFromFolder\"],\n):\n    super(WideDeepDatasetFromFolder, self).__init__()\n\n    if tab_from_folder is None and wide_from_folder is None:\n        raise ValueError(\n            \"Either 'tab_from_folder' or 'wide_from_folder' must be not None\"\n        )\n\n    if reference is not None:\n        assert (\n            img_from_folder is None and text_from_folder is None\n        ), \"If reference is not None, 'img_from_folder' and 'text_from_folder' left as None\"\n        self.text_from_folder, self.img_from_folder = self._get_from_reference(\n            reference\n        )\n    else:\n        assert (\n            text_from_folder is not None and img_from_folder is not None\n        ), \"If reference is None, 'img_from_folder' and 'text_from_folder' must be not None\"\n        self.text_from_folder = text_from_folder\n        self.img_from_folder = img_from_folder\n\n    self.n_samples = n_samples\n    self.tab_from_folder = tab_from_folder\n    self.wide_from_folder = wide_from_folder\n</code></pre>"},{"location":"pytorch-widedeep/losses.html","title":"Losses","text":"<p><code>pytorch-widedeep</code> accepts a number of losses and objectives that can be passed to the <code>Trainer</code> class via the parameter <code>objective</code> (see <code>pytorch-widedeep.training.Trainer</code>). For most cases the loss function that <code>pytorch-widedeep</code> will use internally is already implemented in Pytorch.</p> <p>In addition, <code>pytorch-widedeep</code> implements a series of  \"custom\" loss functions. These are described below for completion since, as mentioned before, they are used internally by the <code>Trainer</code>. Of course, onen could always use them on their own and can be imported as:</p> <p><code>from pytorch_widedeep.losses import FocalLoss</code></p> <p> NOTE:  Losses in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss","title":"MSELoss","text":"<pre><code>MSELoss()\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Mean square error loss with the option of using Label Smooth Distribution (LDS)</p> <p>LDS is based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target, lds_weight=None)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual values</p> </li> <li> <code>lds_weight</code>             (<code>Optional[Tensor]</code>, default:                 <code>None</code> )         \u2013          <p>Tensor of weights that will multiply the loss value.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)\n&gt;&gt;&gt; loss = MSELoss()(input, target, lds_weight)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self, input: Tensor, target: Tensor, lds_weight: Optional[Tensor] = None\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n    lds_weight: Tensor, Optional\n        Tensor of weights that will multiply the loss value.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSELoss()(input, target, lds_weight)\n    \"\"\"\n    loss = (input - target) ** 2\n    if lds_weight is not None:\n        loss *= lds_weight\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss","title":"MSLELoss","text":"<pre><code>MSLELoss()\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Mean square log error loss with the option of using Label Smooth Distribution (LDS)</p> <p>LDS is based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target, lds_weight=None)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual classes</p> </li> <li> <code>lds_weight</code>             (<code>Optional[Tensor]</code>, default:                 <code>None</code> )         \u2013          <p>Tensor of weights that will multiply the loss value.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)\n&gt;&gt;&gt; loss = MSLELoss()(input, target, lds_weight)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self, input: Tensor, target: Tensor, lds_weight: Optional[Tensor] = None\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n    lds_weight: Tensor, Optional\n        Tensor of weights that will multiply the loss value.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSLELoss()(input, target, lds_weight)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    if lds_weight is not None:\n        loss *= lds_weight\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss","title":"RMSELoss","text":"<pre><code>RMSELoss()\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Root mean square error loss adjusted for the possibility of using Label Smooth Distribution (LDS)</p> <p>LDS is based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target, lds_weight=None)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual classes</p> </li> <li> <code>lds_weight</code>             (<code>Optional[Tensor]</code>, default:                 <code>None</code> )         \u2013          <p>Tensor of weights that will multiply the loss value.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSELoss()(input, target, lds_weight)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self, input: Tensor, target: Tensor, lds_weight: Optional[Tensor] = None\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n    lds_weight: Tensor, Optional\n        Tensor of weights that will multiply the loss value.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSELoss()(input, target, lds_weight)\n    \"\"\"\n    loss = (input - target) ** 2\n    if lds_weight is not None:\n        loss *= lds_weight\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss","title":"RMSLELoss","text":"<pre><code>RMSLELoss()\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Root mean square log error loss adjusted for the possibility of using Label Smooth Distribution (LDS)</p> <p>LDS is based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target, lds_weight=None)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual classes</p> </li> <li> <code>lds_weight</code>             (<code>Optional[Tensor]</code>, default:                 <code>None</code> )         \u2013          <p>Tensor of weights that will multiply the loss value.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSLELoss()(input, target, lds_weight)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self, input: Tensor, target: Tensor, lds_weight: Optional[Tensor] = None\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n    lds_weight: Tensor, Optional\n        Tensor of weights that will multiply the loss value.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSLELoss()(input, target, lds_weight)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    if lds_weight is not None:\n        loss *= lds_weight\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss","title":"QuantileLoss","text":"<pre><code>QuantileLoss(\n    quantiles=[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Quantile loss defined as:</p> \\[ Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y)) \\] <p>All credits go to the implementation at pytorch-forecasting.</p> <p>Parameters:</p> <ul> <li> <code>quantiles</code>             (<code>List[float]</code>, default:                 <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code> )         \u2013          <p>List of quantiles</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    quantiles: List[float] = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98],\n):\n    super().__init__()\n    self.quantiles = quantiles\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # REGRESSION\n&gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n&gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n&gt;&gt;&gt; loss = qloss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # REGRESSION\n    &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n    &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n    &gt;&gt;&gt; loss = qloss(input, target)\n    \"\"\"\n\n    assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n        \"The input and target have inconsistent shape. The dimension of the prediction \"\n        \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n        f\"i.e. {len(self.quantiles)}.\"\n    )\n    target = target.view(-1, 1).float()\n    losses = []\n    for i, q in enumerate(self.quantiles):\n        errors = target - input[..., i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n    loss = torch.cat(losses, dim=2)\n\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss","title":"FocalLoss","text":"<pre><code>FocalLoss(alpha=0.25, gamma=1.0)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Implementation of the Focal loss for both binary and multiclass classification:</p> \\[ FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t) \\] <p>where, for a case of a binary classification problem</p> \\[ \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation} \\] <p>Parameters:</p> <ul> <li> <code>alpha</code>             (<code>float</code>, default:                 <code>0.25</code> )         \u2013          <p>Focal Loss <code>alpha</code> parameter</p> </li> <li> <code>gamma</code>             (<code>float</code>, default:                 <code>1.0</code> )         \u2013          <p>Focal Loss <code>gamma</code> parameter</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, alpha: float = 0.25, gamma: float = 1.0):\n    super().__init__()\n    self.alpha = alpha\n    self.gamma = gamma\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # BINARY\n&gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # MULTICLASS\n&gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # BINARY\n    &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # MULTICLASS\n    &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    \"\"\"\n    input_prob = torch.sigmoid(input)\n    if input.size(1) == 1:\n        input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n        num_class = 2\n    else:\n        num_class = input_prob.size(1)\n    binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n    if use_cuda:\n        binary_target = binary_target.cuda()\n    binary_target = binary_target.contiguous()\n    weight = self._get_weight(input_prob, binary_target)\n\n    return F.binary_cross_entropy(\n        input_prob, binary_target, weight, reduction=\"mean\"\n    )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss","title":"BayesianSELoss","text":"<pre><code>BayesianSELoss()\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Squared Loss (log Gaussian) for the case of a regression as specified in the original publication Weight Uncertainty in Neural Networks.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n    \"\"\"\n    return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss","title":"TweedieLoss","text":"<pre><code>TweedieLoss()\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Tweedie loss for extremely unbalanced zero-inflated data</p> <p>All credits go to Wenbo Shi. See this post and the original publication for details.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss.forward","title":"forward","text":"<pre><code>forward(input, target, lds_weight=None, p=1.5)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual values</p> </li> <li> <code>lds_weight</code>             (<code>Optional[Tensor]</code>, default:                 <code>None</code> )         \u2013          <p>If we choose to use LDS this is the tensor of weights that will multiply the loss value.</p> </li> <li> <code>p</code>             (<code>float</code>, default:                 <code>1.5</code> )         \u2013          <p>the power to be used to compute the loss. See the original publication for details</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)\n&gt;&gt;&gt; loss = TweedieLoss()(input, target, lds_weight)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    lds_weight: Optional[Tensor] = None,\n    p: float = 1.5,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n    lds_weight: Tensor, Optional\n        If we choose to use LDS this is the tensor of weights that will\n        multiply the loss value.\n    p: float, default = 1.5\n        the power to be used to compute the loss. See the original\n        publication for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)\n    &gt;&gt;&gt; loss = TweedieLoss()(input, target, lds_weight)\n    \"\"\"\n\n    assert (\n        input.min() &gt; 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n    loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n        2 - p\n    )\n    if lds_weight is not None:\n        loss *= lds_weight\n\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss","title":"ZILNLoss","text":"<pre><code>ZILNLoss()\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Adjusted implementation of the Zero Inflated LogNormal Loss</p> <p>See A Deep Probabilistic Model for Customer Lifetime Value Prediction and the corresponding code.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions with spape (N,3), where N is the batch size</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual target values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n&gt;&gt;&gt; loss = ZILNLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions with spape (N,3), where N is the batch size\n    target: Tensor\n        Target tensor with the actual target values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n    &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n    \"\"\"\n    positive = target &gt; 0\n    positive = positive.float()\n\n    assert input.shape == torch.Size([target.shape[0], 3]), (\n        \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n        \"model that is using ZILNLoss must be equal to 3.\"\n    )\n\n    positive_input = input[..., :1]\n\n    classification_loss = F.binary_cross_entropy_with_logits(\n        positive_input, positive, reduction=\"none\"\n    ).flatten()\n\n    loc = input[..., 1:2]\n\n    # when using max the two input tensors (input and other) have to be of\n    # the same type\n    max_input = F.softplus(input[..., 2:])\n    max_other = torch.sqrt(torch.Tensor([torch.finfo(torch.double).eps])).type(\n        max_input.type()\n    )\n    scale = torch.max(max_input, max_other)\n    safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n    regression_loss = -torch.mean(\n        positive\n        * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n            safe_labels\n        ),\n        dim=-1,\n    )\n\n    return torch.mean(classification_loss + regression_loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss","title":"L1Loss","text":"<pre><code>L1Loss()\n</code></pre> <p>             Bases: <code>Module</code></p> <p>L1 loss adjusted for the possibility of using Label Smooth Distribution (LDS)</p> <p>LDS is based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target, lds_weight=None)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual values</p> </li> <li> <code>lds_weight</code>             (<code>Optional[Tensor]</code>, default:                 <code>None</code> )         \u2013          <p>If we choose to use LDS this is the tensor of weights that will multiply the loss value.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self, input: Tensor, target: Tensor, lds_weight: Optional[Tensor] = None\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n    lds_weight: Tensor, Optional\n        If we choose to use LDS this is the tensor of weights that will\n        multiply the loss value.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    if lds_weight is not None:\n        loss *= lds_weight\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss","title":"FocalR_L1Loss","text":"<pre><code>FocalR_L1Loss(beta=0.2, gamma=1.0, activation_fn='sigmoid')\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Focal-R L1 loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>             (<code>float</code>, default:                 <code>0.2</code> )         \u2013          <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>             (<code>float</code>, default:                 <code>1.0</code> )         \u2013          <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>             (<code>Literal[sigmoid, tanh]</code>, default:                 <code>'sigmoid'</code> )         \u2013          <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target, lds_weight=None)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual classes</p> </li> <li> <code>lds_weight</code>             (<code>Optional[Tensor]</code>, default:                 <code>None</code> )         \u2013          <p>If we choose to use LDS this is the tensor of weights that will multiply the loss value.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    lds_weight: Optional[Tensor] = None,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n    lds_weight: Tensor, Optional\n        If we choose to use LDS this is the tensor of weights that will\n        multiply the loss value.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    if lds_weight is not None:\n        loss *= lds_weight\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss","title":"FocalR_MSELoss","text":"<pre><code>FocalR_MSELoss(\n    beta=0.2, gamma=1.0, activation_fn=\"sigmoid\"\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Focal-R MSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>             (<code>float</code>, default:                 <code>0.2</code> )         \u2013          <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>             (<code>float</code>, default:                 <code>1.0</code> )         \u2013          <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>             (<code>Literal[sigmoid, tanh]</code>, default:                 <code>'sigmoid'</code> )         \u2013          <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target, lds_weight=None)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual classes</p> </li> <li> <code>lds_weight</code>             (<code>Optional[Tensor]</code>, default:                 <code>None</code> )         \u2013          <p>If we choose to use LDS this is the tensor of weights that will multiply the loss value.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    lds_weight: Optional[Tensor] = None,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n    lds_weight: Tensor, Optional\n        If we choose to use LDS this is the tensor of weights that will\n        multiply the loss value.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    if lds_weight is not None:\n        loss *= lds_weight\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss","title":"FocalR_RMSELoss","text":"<pre><code>FocalR_RMSELoss(\n    beta=0.2, gamma=1.0, activation_fn=\"sigmoid\"\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Focal-R RMSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>             (<code>float</code>, default:                 <code>0.2</code> )         \u2013          <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>             (<code>float</code>, default:                 <code>1.0</code> )         \u2013          <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>             (<code>Literal[sigmoid, tanh]</code>, default:                 <code>'sigmoid'</code> )         \u2013          <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target, lds_weight=None)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual classes</p> </li> <li> <code>lds_weight</code>             (<code>Optional[Tensor]</code>, default:                 <code>None</code> )         \u2013          <p>If we choose to use LDS this is the tensor of weights that will multiply the loss value.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    lds_weight: Optional[Tensor] = None,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n    lds_weight: Tensor, Optional\n        If we choose to use LDS this is the tensor of weights that will\n        multiply the loss value.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    if lds_weight is not None:\n        loss *= lds_weight\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss","title":"HuberLoss","text":"<pre><code>HuberLoss(beta=0.2)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Hubbler Loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, beta: float = 0.2):\n    super().__init__()\n    self.beta = beta\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss.forward","title":"forward","text":"<pre><code>forward(input, target, lds_weight=None)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>             (<code>Tensor</code>)         \u2013          <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>             (<code>Tensor</code>)         \u2013          <p>Target tensor with the actual classes</p> </li> <li> <code>lds_weight</code>             (<code>Optional[Tensor]</code>, default:                 <code>None</code> )         \u2013          <p>If we choose to use LDS this is the tensor of weights that will multiply the loss value.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = HuberLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    lds_weight: Optional[Tensor] = None,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n    lds_weight: Tensor, Optional\n        If we choose to use LDS this is the tensor of weights that will\n        multiply the loss value.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = HuberLoss()(input, target)\n    \"\"\"\n    l1_loss = torch.abs(input - target)\n    cond = l1_loss &lt; self.beta\n    loss = torch.where(\n        cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n    )\n    if lds_weight is not None:\n        loss *= lds_weight\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss","title":"InfoNCELoss","text":"<pre><code>InfoNCELoss(temperature=0.1, reduction='mean')\n</code></pre> <p>             Bases: <code>Module</code></p> <p>InfoNCE Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Partially inspired by the code in this repo</p> <p>Parameters:</p> <ul> <li> <code>temperature</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>The logits are divided by the temperature before computing the loss value</p> </li> <li> <code>reduction</code>             (<code>str</code>, default:                 <code>'mean'</code> )         \u2013          <p>Loss reduction method</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, temperature: float = 0.1, reduction: str = \"mean\"):\n    super(InfoNCELoss, self).__init__()\n\n    self.temperature = temperature\n    self.reduction = reduction\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss.forward","title":"forward","text":"<pre><code>forward(g_projs)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>g_projs</code>             (<code>Tuple[Tensor, Tensor]</code>)         \u2013          <p>Tuple with the two tensors corresponding to the output of the two projection heads, as described 'SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training'.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n&gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n&gt;&gt;&gt; loss = InfoNCELoss()\n&gt;&gt;&gt; res = loss(g_projs)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    g_projs: Tuple\n        Tuple with the two tensors corresponding to the output of the two\n        projection heads, as described 'SAINT: Improved Neural Networks\n        for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n    &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n    &gt;&gt;&gt; loss = InfoNCELoss()\n    &gt;&gt;&gt; res = loss(g_projs)\n    \"\"\"\n    z, z_ = g_projs[0], g_projs[1]\n\n    norm_z = F.normalize(z, dim=-1).flatten(1)\n    norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n    logits = (norm_z @ norm_z_.t()) / self.temperature\n    logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n    # the target/labels are the entries on the diagonal\n    target = torch.arange(len(norm_z), device=norm_z.device)\n\n    loss = F.cross_entropy(logits, target, reduction=self.reduction)\n    loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n    return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss","title":"DenoisingLoss","text":"<pre><code>DenoisingLoss(\n    lambda_cat=1.0, lambda_cont=1.0, reduction=\"mean\"\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Denoising Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Parameters:</p> <ul> <li> <code>lambda_cat</code>             (<code>float</code>, default:                 <code>1.0</code> )         \u2013          <p>Multiplicative factor that will be applied to loss associated to the categorical features</p> </li> <li> <code>lambda_cont</code>             (<code>float</code>, default:                 <code>1.0</code> )         \u2013          <p>Multiplicative factor that will be applied to loss associated to the continuous features</p> </li> <li> <code>reduction</code>             (<code>str</code>, default:                 <code>'mean'</code> )         \u2013          <p>Loss reduction method</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self, lambda_cat: float = 1.0, lambda_cont: float = 1.0, reduction: str = \"mean\"\n):\n    super(DenoisingLoss, self).__init__()\n\n    self.lambda_cat = lambda_cat\n    self.lambda_cont = lambda_cont\n    self.reduction = reduction\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss.forward","title":"forward","text":"<pre><code>forward(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>x_cat_and_cat_</code>             (<code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code>)         \u2013          <p>Tuple of tensors containing the raw input features and their encodings, referred in the SAINT paper as \\(x\\) and \\(x''\\) respectively. If one denoising MLP is used per categorical feature <code>x_cat_and_cat_</code> will be a list of tuples, one per categorical feature</p> </li> <li> <code>x_cont_and_cont_</code>             (<code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code>)         \u2013          <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n&gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n&gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n&gt;&gt;&gt; loss = DenoisingLoss()\n&gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    x_cat_and_cat_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n    x_cont_and_cont_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_cat_and_cat_: tuple of Tensors or lists of tuples\n        Tuple of tensors containing the raw input features and their\n        encodings, referred in the SAINT paper as $x$ and $x''$\n        respectively. If one denoising MLP is used per categorical\n        feature `x_cat_and_cat_` will be a list of tuples, one per\n        categorical feature\n    x_cont_and_cont_: tuple of Tensors or lists of tuples\n        same as `x_cat_and_cat_` but for continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n    &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n    &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n    &gt;&gt;&gt; loss = DenoisingLoss()\n    &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n    \"\"\"\n\n    loss_cat = (\n        self._compute_cat_loss(x_cat_and_cat_)\n        if x_cat_and_cat_ is not None\n        else torch.tensor(0.0)\n    )\n    loss_cont = (\n        self._compute_cont_loss(x_cont_and_cont_)\n        if x_cont_and_cont_ is not None\n        else torch.tensor(0.0)\n    )\n\n    return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss","title":"EncoderDecoderLoss","text":"<pre><code>EncoderDecoderLoss(eps=1e-09)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>'Standard' Encoder Decoder Loss. Loss applied during the Endoder-Decoder  Self-Supervised Pre-Training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>The implementation of this lost is based on that at the tabnet repo, which is in itself an adaptation of that in the original paper TabNet: Attentive Interpretable Tabular Learning.</p> <p>Parameters:</p> <ul> <li> <code>eps</code>             (<code>float</code>, default:                 <code>1e-09</code> )         \u2013          <p>Simply a small number to avoid dividing by zero</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, eps: float = 1e-9):\n    super(EncoderDecoderLoss, self).__init__()\n    self.eps = eps\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss.forward","title":"forward","text":"<pre><code>forward(x_true, x_pred, mask)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>x_true</code>             (<code>Tensor</code>)         \u2013          <p>Embeddings of the input data</p> </li> <li> <code>x_pred</code>             (<code>Tensor</code>)         \u2013          <p>Reconstructed embeddings</p> </li> <li> <code>mask</code>             (<code>Tensor</code>)         \u2013          <p>Mask with 1s indicated that the reconstruction, and therefore the loss, is based on those features.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n&gt;&gt;&gt; x_true = torch.rand(3, 3)\n&gt;&gt;&gt; x_pred = torch.rand(3, 3)\n&gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n&gt;&gt;&gt; loss = EncoderDecoderLoss()\n&gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_true: Tensor\n        Embeddings of the input data\n    x_pred: Tensor\n        Reconstructed embeddings\n    mask: Tensor\n        Mask with 1s indicated that the reconstruction, and therefore the\n        loss, is based on those features.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n    &gt;&gt;&gt; x_true = torch.rand(3, 3)\n    &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n    &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n    &gt;&gt;&gt; loss = EncoderDecoderLoss()\n    &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n    \"\"\"\n\n    errors = x_pred - x_true\n\n    reconstruction_errors = torch.mul(errors, mask) ** 2\n\n    x_true_means = torch.mean(x_true, dim=0)\n    x_true_means[x_true_means == 0] = 1\n\n    x_true_stds = torch.std(x_true, dim=0) ** 2\n    x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n    features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n    nb_reconstructed_variables = torch.sum(mask, dim=1)\n    features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n    loss = torch.mean(features_loss_norm)\n\n    return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss","title":"MultiTargetRegressionLoss","text":"<pre><code>MultiTargetRegressionLoss(weights=None, reduction='mean')\n</code></pre> <p>             Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch MSELoss. It allows for multi-target regression problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>weights</code>             (<code>Optional[List[float]]</code>, default:                 <code>None</code> )         \u2013          <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>             (<code>Literal[mean, sum]</code>, default:                 <code>'mean'</code> )         \u2013          <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the MSELoss. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n&gt;&gt;&gt; input = torch.randn(3, 2)\n&gt;&gt;&gt; target = torch.randn(3, 2)\n&gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(\n    self,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n):\n    super(MultiTargetRegressionLoss, self).__init__()\n\n    self.weights = weights\n    self.reduction = reduction\n\n    if self.reduction not in [\"mean\", \"sum\"]:\n        raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss","title":"MultiTargetClassificationLoss","text":"<pre><code>MultiTargetClassificationLoss(\n    binary_config=None,\n    multiclass_config=None,\n    weights=None,\n    reduction=\"mean\",\n    binary_trick=False,\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and cross_entropy losses. It allows for multi-target classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>binary_config</code>             (<code>Optional[List[Union[int, Tuple[int, float]]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> </li> <li> <code>multiclass_config</code>             (<code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> </li> <li> <code>weights</code>             (<code>Optional[List[float]]</code>, default:                 <code>None</code> )         \u2013          <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>             (<code>Literal[mean, sum]</code>, default:                 <code>'mean'</code> )         \u2013          <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss or the binary_cross_entropy_with_logits. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> </li> <li> <code>binary_trick</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 4)\n&gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output_1 = loss_1(input, target)\n&gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n... reduction=\"sum\", weights=[0.5, 0.5])\n&gt;&gt;&gt; output_2 = loss_2(input, target)\n&gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n&gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(  # noqa: C901\n    self,\n    binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n    multiclass_config: Optional[\n        List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n    ] = None,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    binary_trick: bool = False,\n):\n    super(MultiTargetClassificationLoss, self).__init__()\n\n    if reduction not in [\"mean\", \"sum\"]:\n        raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n    self.binary_config = binary_config\n    self.multiclass_config = multiclass_config\n    self.weights = weights\n    self.reduction = reduction\n    self.binary_trick = binary_trick\n\n    if self.weights is not None:\n        if len(self.weights) != (\n            len(self.binary_config) if self.binary_config is not None else 0\n        ) + (\n            len(self.multiclass_config) if self.multiclass_config is not None else 0\n        ):\n            raise ValueError(\n                \"The number of weights must match the number of binary and multiclass targets\"\n            )\n\n    if self.binary_trick:\n        self._check_inputs_with_binary_trick()\n        self._binary_config: List[int] = binary_config  # type: ignore[assignment]\n        self._multiclass_config: List[Tuple[int, int]] = self.multiclass_config  # type: ignore[assignment]\n    else:\n        self.binary_config_with_pos_weights = (\n            (self._set_binary_config_without_binary_trick())\n            if self.binary_config is not None\n            else None\n        )\n        self.multiclass_config_with_weights = (\n            (self._set_multiclass_config_without_binary_trick())\n            if self.multiclass_config is not None\n            else None\n        )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss","title":"MutilTargetRegressionAndClassificationLoss","text":"<pre><code>MutilTargetRegressionAndClassificationLoss(\n    regression_config=[],\n    binary_config=None,\n    multiclass_config=None,\n    weights=None,\n    reduction=\"mean\",\n    binary_trick=False,\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>This class is a wrapper around the MultiTargetRegressionLoss and the MultiTargetClassificationLoss. It allows for multi-target regression and classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>regression_config</code>             (<code>List[int]</code>, default:                 <code>[]</code> )         \u2013          <p>List of integers with the indices of the regression targets</p> </li> <li> <code>binary_config</code>             (<code>Optional[List[Union[int, Tuple[int, float]]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> </li> <li> <code>multiclass_config</code>             (<code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> </li> <li> <code>weights</code>             (<code>Optional[List[float]]</code>, default:                 <code>None</code> )         \u2013          <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>             (<code>Literal[mean, sum]</code>, default:                 <code>'mean'</code> )         \u2013          <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss, the binary_cross_entropy_with_logits or the MSELoss. This reduction is applied after each target has been computed. Alias: 'target_reduction'</p> </li> <li> <code>binary_trick</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n... multiclass_config=[(2, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(  # noqa: C901\n    self,\n    regression_config: List[int] = [],\n    binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n    multiclass_config: Optional[\n        List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n    ] = None,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    binary_trick: bool = False,\n):\n\n    super(MutilTargetRegressionAndClassificationLoss, self).__init__()\n\n    self.regression_config = regression_config\n\n    assert binary_config is not None or multiclass_config is not None, (\n        \"Either binary_config or multiclass_config must be provided. \"\n        \"Otherwise, use the MultiTargetRegressionLoss\"\n    )\n\n    if binary_trick:\n        self._check_inputs_with_binary_trick(\n            regression_config, binary_config, multiclass_config\n        )\n\n    if weights is not None:\n        if len(weights) != (\n            len(regression_config)\n            + (len(binary_config) if binary_config is not None else 0)\n            + (len(multiclass_config) if multiclass_config is not None else 0)\n        ):\n            raise ValueError(\n                \"The number of weights must match the number of regression, binary and multiclass targets\"\n            )\n\n        self.weights_regression = self._prepare_weights_for_regression_targets(\n            weights, regression_config\n        )\n        self.weights_binary = self._prepare_weights_per_binary_targets(\n            weights, binary_config\n        )\n        self.weights_multiclass = self._prepare_weights_per_multiclass_targets(\n            weights, multiclass_config\n        )\n        self.weights = weights\n    else:\n        self.weights_regression = None\n        self.weights_binary = None\n        self.weights_multiclass = None\n\n    self.multi_target_regression_loss = MultiTargetRegressionLoss(\n        weights=self.weights_regression, reduction=reduction\n    )\n\n    self.multi_target_classification_loss = MultiTargetClassificationLoss(\n        binary_config=binary_config,\n        multiclass_config=multiclass_config,\n        weights=(\n            self.weights_binary + self.weights_multiclass\n            if self.weights_binary is not None\n            and self.weights_multiclass is not None\n            else (\n                self.weights_binary\n                if self.weights_binary is not None\n                else self.weights_multiclass\n            )\n        ),\n        reduction=reduction,\n        binary_trick=binary_trick,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html","title":"Metrics","text":"<p> NOTE: metrics in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems: \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p> <p>We have added the possibility of using the metrics available at the torchmetrics library. Note that this library is still in its early versions and therefore this option should be used with caution. To use <code>torchmetrics</code> simply import them and use them as any of the <code>pytorch-widedeep</code> metrics described below.</p> <pre><code>from torchmetrics import Accuracy, Precision\n\naccuracy = Accuracy(average=None, num_classes=2)\nprecision = Precision(average='micro', num_classes=2)\n\ntrainer = Trainer(model, objective=\"binary\", metrics=[accuracy, precision])\n</code></pre> <p>A functioning example for <code>pytorch-widedeep</code> using <code>torchmetrics</code> can be found in the Examples folder</p> <p> NOTE: the forward method for all metrics in this  module takes two tensors, <code>y_pred</code> and <code>y_true</code> (in that order). Therefore,  we do not include the method in the documentation.</p>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy","title":"Accuracy","text":"<pre><code>Accuracy(top_k=1)\n</code></pre> <p>             Bases: <code>Metric</code></p> <p>Class to calculate the accuracy for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>top_k</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Accuracy will be computed using the top k most likely classes in multiclass problems</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy(top_k=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.66666667)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, top_k: int = 1):\n    super(Accuracy, self).__init__()\n\n    self.top_k = top_k\n    self.correct_count = 0\n    self.total_count = 0\n    self._name = \"acc\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.correct_count = 0\n    self.total_count = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision","title":"Precision","text":"<pre><code>Precision(average=True)\n</code></pre> <p>             Bases: <code>Metric</code></p> <p>Class to calculate the precision for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>This applies only to multiclass problems. if <code>True</code> calculate precision for each label, and finds their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(Precision, self).__init__()\n\n    self.average = average\n    self.true_positives = 0\n    self.all_positives = 0\n    self.eps = 1e-20\n    self._name = \"prec\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.all_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall","title":"Recall","text":"<pre><code>Recall(average=True)\n</code></pre> <p>             Bases: <code>Metric</code></p> <p>Class to calculate the recall for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>This applies only to multiclass problems. if <code>True</code> calculate recall for each label, and finds their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(Recall, self).__init__()\n\n    self.average = average\n    self.true_positives = 0\n    self.actual_positives = 0\n    self.eps = 1e-20\n    self._name = \"rec\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.actual_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore","title":"FBetaScore","text":"<pre><code>FBetaScore(beta, average=True)\n</code></pre> <p>             Bases: <code>Metric</code></p> <p>Class to calculate the fbeta score for both binary and categorical problems</p> \\[ F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)} \\] <p>Parameters:</p> <ul> <li> <code>beta</code>             (<code>int</code>)         \u2013          <p>Coefficient to control the balance between precision and recall</p> </li> <li> <code>average</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta for each label, and find their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, beta: int, average: bool = True):\n    super(FBetaScore, self).__init__()\n\n    self.beta = beta\n    self.average = average\n    self.precision = Precision(average=False)\n    self.recall = Recall(average=False)\n    self.eps = 1e-20\n    self._name = \"\".join([\"f\", str(self.beta)])\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets precision and recall</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets precision and recall\n    \"\"\"\n    self.precision.reset()\n    self.recall.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score","title":"F1Score","text":"<pre><code>F1Score(average=True)\n</code></pre> <p>             Bases: <code>Metric</code></p> <p>Class to calculate the f1 score for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for each label, and find their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(F1Score, self).__init__()\n\n    self.average = average\n    self.f1 = FBetaScore(beta=1, average=self.average)\n    self._name = self.f1._name\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.f1.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score","title":"R2Score","text":"<pre><code>R2Score()\n</code></pre> <p>             Bases: <code>Metric</code></p> <p>Calculates R-Squared, the coefficient of determination:</p> \\[ R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2} \\] <p>where \\(\\hat{y_j}\\) is the ground truth, \\(y_j\\) is the predicted value and \\(\\bar{y}\\) is the mean of the ground truth.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; r2 = R2Score()\n&gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n&gt;&gt;&gt; r2(y_pred, y_true)\narray(0.94860814)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self):\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n\n    self._name = \"r2\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html","title":"The <code>models</code> module","text":"<p>This module contains the models that can be used as the four main components that will comprise a Wide and Deep model (<code>wide</code>, <code>deeptabular</code>, <code>deeptext</code>, <code>deepimage</code>), as well as the <code>WideDeep</code> \"constructor\" class. Note that each of the four components can be used independently. It also contains all the documentation for the models that can be used for self-supervised pre-training with tabular data.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide","title":"Wide","text":"<pre><code>Wide(input_dim, pred_dim=1)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Defines a <code>Wide</code> (linear) model where the non-linearities are captured via the so-called crossed-columns. This can be used as the <code>wide</code> component of a Wide &amp; Deep model.</p> <p>Parameters:</p> <ul> <li> <code>input_dim</code>             (<code>int</code>)         \u2013          <p>size of the Linear layer (implemented via an Embedding layer). <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide model. For example, if the wide model receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> </li> <li> <code>pred_dim</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>size of the ouput tensor containing the predictions. Note that unlike all the other models, the wide model is connected directly to the output neuron(s) when used to build a Wide and Deep model. Therefore, it requires the <code>pred_dim</code> parameter.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_linear</code>             (<code>Module</code>)         \u2013          <p>the linear layer that comprises the wide branch of the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Wide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(4)\n&gt;&gt;&gt; wide = Wide(input_dim=X.unique().size(0), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>@alias(\"pred_dim\", [\"pred_size\", \"num_class\"])\ndef __init__(self, input_dim: int, pred_dim: int = 1):\n    super(Wide, self).__init__()\n\n    self.input_dim = input_dim\n    self.pred_dim = pred_dim\n\n    # Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n    self.wide_linear = nn.Embedding(input_dim + 1, pred_dim, padding_idx=0)\n    # (Sum(Embedding) + bias) is equivalent to (OneHotVector + Linear)\n    self.bias = nn.Parameter(torch.zeros(pred_dim))\n    self._reset_parameters()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide.forward","title":"forward","text":"<pre><code>forward(X)\n</code></pre> <p>Forward pass. Simply connecting the Embedding layer with the ouput neuron(s)</p> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>def forward(self, X: Tensor) -&gt; Tensor:\n    r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n    neuron(s)\"\"\"\n    out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n    return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp","title":"TabMlp","text":"<pre><code>TabMlp(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=[200, 100],\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True\n)\n</code></pre> <p>             Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabMlp</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of dense layers (i.e. a MLP).</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>             (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                 <code>None</code> )         \u2013          <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>             (<code>List[int]</code>, default:                 <code>[200, 100]</code> )         \u2013          <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>             (<code>Union[float, List[float]]</code>, default:                 <code>0.1</code> )         \u2013          <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>             (<code>Module</code>)         \u2013          <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: List[int] = [200, 100],\n    mlp_activation: str = \"relu\",\n    mlp_dropout: Union[float, List[float]] = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabMlp, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n    # Mlp\n    mlp_input_dim = self.cat_out_dim + self.cont_out_dim\n    mlp_hidden_dims = [mlp_input_dim] + mlp_hidden_dims\n    self.encoder = MLP(\n        mlp_hidden_dims,\n        mlp_activation,\n        mlp_dropout,\n        mlp_batchnorm,\n        mlp_batchnorm_last,\n        mlp_linear_first,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder","title":"TabMlpDecoder","text":"<pre><code>TabMlpDecoder(\n    embed_dim,\n    mlp_hidden_dims=[100, 200],\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True,\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabMlp</code> model (which can be considered an encoder itself).</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). The <code>TabMlpDecoder</code> will receive the output from the MLP and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>             (<code>int</code>)         \u2013          <p>Size of the embeddings tensor that needs to be reconstructed.</p> </li> <li> <code>mlp_hidden_dims</code>             (<code>List[int]</code>, default:                 <code>[100, 200]</code> )         \u2013          <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>             (<code>Union[float, List[float]]</code>, default:                 <code>0.1</code> )         \u2013          <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>             (<code>Module</code>)         \u2013          <p>mlp model that will receive the output of the encoder</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    mlp_hidden_dims: List[int] = [100, 200],\n    mlp_activation: str = \"relu\",\n    mlp_dropout: Union[float, List[float]] = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabMlpDecoder, self).__init__()\n\n    self.embed_dim = embed_dim\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.decoder = MLP(\n        mlp_hidden_dims + [self.embed_dim],\n        mlp_activation,\n        mlp_dropout,\n        mlp_batchnorm,\n        mlp_batchnorm_last,\n        mlp_linear_first,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet","title":"TabResnet","text":"<pre><code>TabResnet(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    blocks_dims=[200, 100, 100],\n    blocks_dropout=0.1,\n    simplify_blocks=False,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>             Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabResnet</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of Resnet blocks. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>             (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                 <code>None</code> )         \u2013          <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>blocks_dims</code>             (<code>List[int]</code>, default:                 <code>[200, 100, 100]</code> )         \u2013          <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> </li> <li> <code>blocks_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Block's internal dropout.</p> </li> <li> <code>simplify_blocks</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> </li> <li> <code>mlp_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> </li> <li> <code>mlp_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>             (<code>Module</code>)         \u2013          <p>deep dense Resnet model that will receive the concatenation of the embeddings and the continuous columns</p> </li> <li> <code>mlp</code>             (<code>Module</code>)         \u2013          <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp model that will receive the results of the concatenation of the embeddings and the continuous columns -- if present --.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n&gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_deep)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    blocks_dims: List[int] = [200, 100, 100],\n    blocks_dropout: float = 0.1,\n    simplify_blocks: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabResnet, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    if len(blocks_dims) &lt; 2:\n        raise ValueError(\n            \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n        )\n\n    self.blocks_dims = blocks_dims\n    self.blocks_dropout = blocks_dropout\n    self.simplify_blocks = simplify_blocks\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n\n    # Resnet\n    dense_resnet_input_dim = self.cat_out_dim + self.cont_out_dim\n    self.encoder = DenseResnet(\n        dense_resnet_input_dim, blocks_dims, blocks_dropout, self.simplify_blocks\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.blocks_dims[-1]] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                True if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder","title":"TabResnetDecoder","text":"<pre><code>TabResnetDecoder(\n    embed_dim,\n    blocks_dims=[100, 100, 200],\n    blocks_dropout=0.1,\n    simplify_blocks=False,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None,\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabResnet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the ResNet blocks or the MLP(if present) and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>             (<code>int</code>)         \u2013          <p>Size of the embeddings tensor to be reconstructed.</p> </li> <li> <code>blocks_dims</code>             (<code>List[int]</code>, default:                 <code>[100, 100, 200]</code> )         \u2013          <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> </li> <li> <code>blocks_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Block's internal dropout.</p> </li> <li> <code>simplify_blocks</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> </li> <li> <code>mlp_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> </li> <li> <code>mlp_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>             (<code>Module</code>)         \u2013          <p>deep dense Resnet model that will receive the output of the encoder IF <code>mlp_hidden_dims</code> is None</p> </li> <li> <code>mlp</code>             (<code>Module</code>)         \u2013          <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist in an MLP that will receive the output of the encoder followed by the deep dense Resnet.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    blocks_dims: List[int] = [100, 100, 200],\n    blocks_dropout: float = 0.1,\n    simplify_blocks: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabResnetDecoder, self).__init__()\n\n    if len(blocks_dims) &lt; 2:\n        raise ValueError(\n            \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n        )\n\n    self.embed_dim = embed_dim\n\n    self.blocks_dims = blocks_dims\n    self.blocks_dropout = blocks_dropout\n    self.simplify_blocks = simplify_blocks\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                True if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n        self.decoder = DenseResnet(\n            self.mlp_hidden_dims[-1],\n            blocks_dims,\n            blocks_dropout,\n            self.simplify_blocks,\n        )\n    else:\n        self.mlp = None\n        self.decoder = DenseResnet(\n            blocks_dims[0], blocks_dims, blocks_dropout, self.simplify_blocks\n        )\n\n    self.reconstruction_layer = nn.Linear(blocks_dims[-1], embed_dim, bias=False)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet","title":"TabNet","text":"<pre><code>TabNet(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    n_steps=3,\n    step_dim=8,\n    attn_dim=8,\n    dropout=0.0,\n    n_glu_step_dependent=2,\n    n_glu_shared=2,\n    ghost_bn=True,\n    virtual_batch_size=128,\n    momentum=0.02,\n    gamma=1.3,\n    epsilon=1e-15,\n    mask_type=\"sparsemax\"\n)\n</code></pre> <p>             Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a TabNet model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>The implementation in this library is fully based on that here by the dreamquark-ai team, simply adapted so that it can work within the <code>WideDeep</code> frame. Therefore, ALL CREDIT TO THE DREAMQUARK-AI TEAM.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>             (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                 <code>None</code> )         \u2013          <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>n_steps</code>             (<code>int</code>, default:                 <code>3</code> )         \u2013          <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> </li> <li> <code>step_dim</code>             (<code>int</code>, default:                 <code>8</code> )         \u2013          <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> </li> <li> <code>attn_dim</code>             (<code>int</code>, default:                 <code>8</code> )         \u2013          <p>Attention dimension</p> </li> <li> <code>dropout</code>             (<code>float</code>, default:                 <code>0.0</code> )         \u2013          <p>GLU block's internal dropout</p> </li> <li> <code>n_glu_step_dependent</code>             (<code>int</code>, default:                 <code>2</code> )         \u2013          <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> </li> <li> <code>n_glu_shared</code>             (<code>int</code>, default:                 <code>2</code> )         \u2013          <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> </li> <li> <code>ghost_bn</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating if Ghost Batch Normalization will be used.</p> </li> <li> <code>virtual_batch_size</code>             (<code>int</code>, default:                 <code>128</code> )         \u2013          <p>Batch size when using Ghost Batch Normalization</p> </li> <li> <code>momentum</code>             (<code>float</code>, default:                 <code>0.02</code> )         \u2013          <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> </li> <li> <code>gamma</code>             (<code>float</code>, default:                 <code>1.3</code> )         \u2013          <p>Relaxation parameter in the paper. When gamma = 1, a feature is enforced to be used only at one decision step. As gamma increases, more flexibility is provided to use a feature at multiple decision steps</p> </li> <li> <code>epsilon</code>             (<code>float</code>, default:                 <code>1e-15</code> )         \u2013          <p>Float to avoid log(0). Always keep low</p> </li> <li> <code>mask_type</code>             (<code>str</code>, default:                 <code>'sparsemax'</code> )         \u2013          <p>Mask function to use. Either 'sparsemax' or 'entmax'</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>             (<code>Module</code>)         \u2013          <p>the TabNet encoder. For details see the original publication.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    n_steps: int = 3,\n    step_dim: int = 8,\n    attn_dim: int = 8,\n    dropout: float = 0.0,\n    n_glu_step_dependent: int = 2,\n    n_glu_shared: int = 2,\n    ghost_bn: bool = True,\n    virtual_batch_size: int = 128,\n    momentum: float = 0.02,\n    gamma: float = 1.3,\n    epsilon: float = 1e-15,\n    mask_type: str = \"sparsemax\",\n):\n    super(TabNet, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_steps = n_steps\n    self.step_dim = step_dim\n    self.attn_dim = attn_dim\n    self.dropout = dropout\n    self.n_glu_step_dependent = n_glu_step_dependent\n    self.n_glu_shared = n_glu_shared\n    self.ghost_bn = ghost_bn\n    self.virtual_batch_size = virtual_batch_size\n    self.momentum = momentum\n    self.gamma = gamma\n    self.epsilon = epsilon\n    self.mask_type = mask_type\n\n    # Embeddings are instantiated at the base model\n    self.embed_out_dim = self.cat_out_dim + self.cont_out_dim\n\n    # TabNet\n    self.encoder = TabNetEncoder(\n        self.embed_out_dim,\n        n_steps,\n        step_dim,\n        attn_dim,\n        dropout,\n        n_glu_step_dependent,\n        n_glu_shared,\n        ghost_bn,\n        virtual_batch_size,\n        momentum,\n        gamma,\n        epsilon,\n        mask_type,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder","title":"TabNetDecoder","text":"<pre><code>TabNetDecoder(\n    embed_dim,\n    n_steps=3,\n    step_dim=8,\n    dropout=0.0,\n    n_glu_step_dependent=2,\n    n_glu_shared=2,\n    ghost_bn=True,\n    virtual_batch_size=128,\n    momentum=0.02,\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabNet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the <code>TabNet</code> encoder (i.e. the output from the so called 'steps') and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>             (<code>int</code>)         \u2013          <p>Size of the embeddings tensor to be reconstructed.</p> </li> <li> <code>n_steps</code>             (<code>int</code>, default:                 <code>3</code> )         \u2013          <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> </li> <li> <code>step_dim</code>             (<code>int</code>, default:                 <code>8</code> )         \u2013          <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> </li> <li> <code>dropout</code>             (<code>float</code>, default:                 <code>0.0</code> )         \u2013          <p>GLU block's internal dropout</p> </li> <li> <code>n_glu_step_dependent</code>             (<code>int</code>, default:                 <code>2</code> )         \u2013          <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> </li> <li> <code>n_glu_shared</code>             (<code>int</code>, default:                 <code>2</code> )         \u2013          <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> </li> <li> <code>ghost_bn</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating if Ghost Batch Normalization will be used.</p> </li> <li> <code>virtual_batch_size</code>             (<code>int</code>, default:                 <code>128</code> )         \u2013          <p>Batch size when using Ghost Batch Normalization</p> </li> <li> <code>momentum</code>             (<code>float</code>, default:                 <code>0.02</code> )         \u2013          <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>             (<code>Module</code>)         \u2013          <p>decoder that will receive the output from the encoder's steps and will reconstruct the embeddings</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n&gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n&gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    n_steps: int = 3,\n    step_dim: int = 8,\n    dropout: float = 0.0,\n    n_glu_step_dependent: int = 2,\n    n_glu_shared: int = 2,\n    ghost_bn: bool = True,\n    virtual_batch_size: int = 128,\n    momentum: float = 0.02,\n):\n    super(TabNetDecoder, self).__init__()\n\n    self.n_steps = n_steps\n    self.step_dim = step_dim\n    self.dropout = dropout\n    self.n_glu_step_dependent = n_glu_step_dependent\n    self.n_glu_shared = n_glu_shared\n    self.ghost_bn = ghost_bn\n    self.virtual_batch_size = virtual_batch_size\n    self.momentum = momentum\n\n    shared_layers = nn.ModuleList()\n    for i in range(n_glu_shared):\n        if i == 0:\n            shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n        else:\n            shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n\n    self.decoder = nn.ModuleList()\n    for step in range(n_steps):\n        transformer = FeatTransformer(\n            step_dim,\n            step_dim,\n            dropout,\n            shared_layers,\n            n_glu_step_dependent,\n            ghost_bn,\n            virtual_batch_size,\n            momentum=momentum,\n        )\n        self.decoder.append(transformer)\n\n    self.reconstruction_layer = nn.Linear(step_dim, embed_dim, bias=False)\n    initialize_non_glu(self.reconstruction_layer, step_dim, embed_dim)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP","title":"ContextAttentionMLP","text":"<pre><code>ContextAttentionMLP(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    attn_dropout=0.2,\n    with_addnorm=False,\n    attn_activation=\"leaky_relu\",\n    n_blocks=3\n)\n</code></pre> <p>             Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>ContextAttentionMLP</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by a <code>ContextAttentionEncoder</code>. Such encoder is in part inspired by the attention mechanism described in Hierarchical Attention Networks for Document Classification. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>             (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                 <code>'standard'</code> )         \u2013          <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>             (<code>int</code>, default:                 <code>32</code> )         \u2013          <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>attn_dropout</code>             (<code>float</code>, default:                 <code>0.2</code> )         \u2013          <p>Dropout for each attention block</p> </li> <li> <code>with_addnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if residual connections will be used in the attention blocks</p> </li> <li> <code>attn_activation</code>             (<code>str</code>, default:                 <code>'leaky_relu'</code> )         \u2013          <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> </li> <li> <code>n_blocks</code>             (<code>int</code>, default:                 <code>3</code> )         \u2013          <p>Number of attention blocks</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>             (<code>Module</code>)         \u2013          <p>Sequence of attention encoders.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    attn_dropout: float = 0.2,\n    with_addnorm: bool = False,\n    attn_activation: str = \"leaky_relu\",\n    n_blocks: int = 3,\n):\n    super(ContextAttentionMLP, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.attn_dropout = attn_dropout\n    self.with_addnorm = with_addnorm\n    self.attn_activation = attn_activation\n    self.n_blocks = n_blocks\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    # Embeddings are instantiated at the base model\n    # Attention Blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"attention_block\" + str(i),\n            ContextAttentionEncoder(\n                input_dim,\n                attn_dropout,\n                with_addnorm,\n                attn_activation,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, F)\\), where \\(N\\) is the batch size and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP","title":"SelfAttentionMLP","text":"<pre><code>SelfAttentionMLP(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    attn_dropout=0.2,\n    n_heads=8,\n    use_bias=False,\n    with_addnorm=False,\n    attn_activation=\"leaky_relu\",\n    n_blocks=3\n)\n</code></pre> <p>             Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>SelfAttentionMLP</code> model that can be used as the deeptabular component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by what we would refer as a simplified <code>SelfAttentionEncoder</code>. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details. The reason to use a simplified version of self attention is because we observed that the 'standard' attention mechanism used in the TabTransformer has a notable tendency to overfit.</p> <p>In more detail, this model only uses Q and K (and not V). If we think about it as in terms of text (and intuitively), the Softmax(QK^T) is the attention mechanism that tells us how much, at each position in the input sentence, each word is represented or 'expressed'. We refer to that as 'attention weights'. These attention weighst are normally multiplied by a Value matrix to further strength the focus on the words that each word should be attending to (again, intuitively).</p> <p>In this implementation we skip this last multiplication and instead we multiply the attention weights directly by the input tensor. This is a simplification that we expect is beneficial in terms of avoiding overfitting for tabular data.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>             (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                 <code>'standard'</code> )         \u2013          <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>             (<code>int</code>, default:                 <code>32</code> )         \u2013          <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>attn_dropout</code>             (<code>float</code>, default:                 <code>0.2</code> )         \u2013          <p>Dropout for each attention block</p> </li> <li> <code>n_heads</code>             (<code>int</code>, default:                 <code>8</code> )         \u2013          <p>Number of attention heads per attention block.</p> </li> <li> <code>use_bias</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to use bias in the Q, K projection layers.</p> </li> <li> <code>with_addnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if residual connections will be used in the attention blocks</p> </li> <li> <code>attn_activation</code>             (<code>str</code>, default:                 <code>'leaky_relu'</code> )         \u2013          <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> </li> <li> <code>n_blocks</code>             (<code>int</code>, default:                 <code>3</code> )         \u2013          <p>Number of attention blocks</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cat_and_cont_embed</code>             (<code>Module</code>)         \u2013          <p>This is the module that processes the categorical and continuous columns</p> </li> <li> <code>encoder</code>             (<code>Module</code>)         \u2013          <p>Sequence of attention encoders.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    attn_dropout: float = 0.2,\n    n_heads: int = 8,\n    use_bias: bool = False,\n    with_addnorm: bool = False,\n    attn_activation: str = \"leaky_relu\",\n    n_blocks: int = 3,\n):\n    super(SelfAttentionMLP, self).__init__(\n        column_idx=column_idx,\n        input_dim=input_dim,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.attn_dropout = attn_dropout\n    self.n_heads = n_heads\n    self.use_bias = use_bias\n    self.with_addnorm = with_addnorm\n    self.attn_activation = attn_activation\n    self.n_blocks = n_blocks\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    # Embeddings are instantiated at the base model\n    # Attention Blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"attention_block\" + str(i),\n            SelfAttentionEncoder(\n                input_dim,\n                attn_dropout,\n                use_bias,\n                n_heads,\n                with_addnorm,\n                attn_activation,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the WideDeep class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer","title":"TabTransformer","text":"<pre><code>TabTransformer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_heads=8,\n    use_qkv_bias=False,\n    n_blocks=4,\n    attn_dropout=0.2,\n    ff_dropout=0.1,\n    ff_factor=4,\n    transformer_activation=\"gelu\",\n    use_linear_attention=False,\n    use_flash_attention=False,\n    mlp_hidden_dims=None,\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True\n)\n</code></pre> <p>             Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines our adptation of the TabTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an enhanced adaptation of the model described in the paper. It can be considered as the flagship of our transformer family of models for tabular data and offers mutiple, additional features relative to the original publication(and some other models in the library)</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>             (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                 <code>None</code> )         \u2013          <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>             (<code>int</code>, default:                 <code>32</code> )         \u2013          <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>             (<code>int</code>, default:                 <code>8</code> )         \u2013          <p>Number of attention heads per Transformer block</p> </li> <li> <code>use_qkv_bias</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers.</p> </li> <li> <code>n_blocks</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Number of Transformer blocks</p> </li> <li> <code>attn_dropout</code>             (<code>float</code>, default:                 <code>0.2</code> )         \u2013          <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>             (<code>str</code>, default:                 <code>'gelu'</code> )         \u2013          <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>use_linear_attention</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if Linear Attention (from Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention) will be used. The inclusing of this mode of attention is inspired by this post, where the Uber team finds that this attention mechanism leads to the best results for their tabular data.</p> </li> <li> <code>use_flash_attention</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if Flash Attention will be used. </p> </li> <li> <code>mlp_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>             (<code>Module</code>)         \u2013          <p>Sequence of Transformer blocks</p> </li> <li> <code>mlp</code>             (<code>Module</code>)         \u2013          <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_heads: int = 8,\n    use_qkv_bias: bool = False,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.2,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    transformer_activation: str = \"gelu\",\n    use_linear_attention: bool = False,\n    use_flash_attention: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: str = \"relu\",\n    mlp_dropout: float = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabTransformer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        input_dim=input_dim,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_heads = n_heads\n    self.use_qkv_bias = use_qkv_bias\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.transformer_activation = transformer_activation\n    self.use_linear_attention = use_linear_attention\n    self.use_flash_attention = use_flash_attention\n    self.ff_factor = ff_factor\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    if self.n_cont and not self.n_cat and not self.embed_continuous:\n        raise ValueError(\n            \"If only continuous features are used 'embed_continuous' must be set to 'True'\"\n        )\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"transformer_block\" + str(i),\n            TransformerEncoder(\n                input_dim,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                transformer_activation,\n                use_linear_attention,\n                use_flash_attention,\n            ),\n        )\n\n    self.mlp_first_hidden_dim = self._mlp_first_hidden_dim()\n\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p> <p> NOTE: if flash attention or linear attention are used, no attention weights are saved during the training process and calling this property will throw a ValueError</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT","title":"SAINT","text":"<pre><code>SAINT(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    use_qkv_bias=False,\n    n_heads=8,\n    n_blocks=2,\n    attn_dropout=0.1,\n    ff_dropout=0.2,\n    ff_factor=4,\n    transformer_activation=\"gelu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>             Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a SAINT model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an slightly modified and enhanced  version of the model described in the paper,</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>             (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                 <code>'standard'</code> )         \u2013          <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>             (<code>int</code>, default:                 <code>32</code> )         \u2013          <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>             (<code>int</code>, default:                 <code>8</code> )         \u2013          <p>Number of attention heads per Transformer block</p> </li> <li> <code>use_qkv_bias</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>             (<code>int</code>, default:                 <code>2</code> )         \u2013          <p>Number of SAINT-Transformer blocks.</p> </li> <li> <code>attn_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Dropout that will be applied to the Multi-Head Attention column and row layers</p> </li> <li> <code>ff_dropout</code>             (<code>float</code>, default:                 <code>0.2</code> )         \u2013          <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>             (<code>str</code>, default:                 <code>'gelu'</code> )         \u2013          <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>             (<code>Module</code>)         \u2013          <p>Sequence of SAINT-Transformer blocks</p> </li> <li> <code>mlp</code>             (<code>Module</code>)         \u2013          <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    use_qkv_bias: bool = False,\n    n_heads: int = 8,\n    n_blocks: int = 2,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.2,\n    ff_factor: int = 4,\n    transformer_activation: str = \"gelu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(SAINT, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.use_qkv_bias = use_qkv_bias\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"saint_block\" + str(i),\n            SaintEncoder(\n                input_dim,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                transformer_activation,\n                self.n_feats,\n            ),\n        )\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and the second elements are the column and row attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>column attention: \\((N, H, F, F)\\)</p> </li> <li> <p>row attention: \\((1, H, N, N)\\)</p> </li> </ul> <p>where \\(N\\) is the batch size, \\(H\\) is the number of heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer","title":"FTTransformer","text":"<pre><code>FTTransformer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=64,\n    kv_compression_factor=0.5,\n    kv_sharing=False,\n    use_qkv_bias=False,\n    n_heads=8,\n    n_blocks=4,\n    attn_dropout=0.2,\n    ff_dropout=0.1,\n    ff_factor=1.33,\n    transformer_activation=\"reglu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>             Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a FTTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>             (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                 <code>'standard'</code> )         \u2013          <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>             (<code>int</code>, default:                 <code>64</code> )         \u2013          <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> </li> <li> <code>kv_compression_factor</code>             (<code>float</code>, default:                 <code>0.5</code> )         \u2013          <p>By default, the FTTransformer uses Linear Attention (See Linformer: Self-Attention with Linear Complexity ). The compression factor that will be used to reduce the input sequence length. If we denote the resulting sequence length as \\(k = int(kv_{compression \\space factor} \\times s)\\) where \\(s\\) is the input sequence length.</p> </li> <li> <code>kv_sharing</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the \\(E\\) and \\(F\\) projection matrices will share weights.  See Linformer: Self-Attention with Linear Complexity for details</p> </li> <li> <code>n_heads</code>             (<code>int</code>, default:                 <code>8</code> )         \u2013          <p>Number of attention heads per FTTransformer block</p> </li> <li> <code>use_qkv_bias</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Number of FTTransformer blocks</p> </li> <li> <code>attn_dropout</code>             (<code>float</code>, default:                 <code>0.2</code> )         \u2013          <p>Dropout that will be applied to the Linear-Attention layers</p> </li> <li> <code>ff_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>             (<code>float</code>, default:                 <code>1.33</code> )         \u2013          <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4, but they use 4/3 in the paper.</p> </li> <li> <code>transformer_activation</code>             (<code>str</code>, default:                 <code>'reglu'</code> )         \u2013          <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final FTTransformer block will be used.</p> </li> <li> <code>mlp_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>             (<code>Module</code>)         \u2013          <p>Sequence of FTTransformer blocks</p> </li> <li> <code>mlp</code>             (<code>Module</code>)         \u2013          <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 64,\n    kv_compression_factor: float = 0.5,\n    kv_sharing: bool = False,\n    use_qkv_bias: bool = False,\n    n_heads: int = 8,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.2,\n    ff_dropout: float = 0.1,\n    ff_factor: float = 1.33,\n    transformer_activation: str = \"reglu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(FTTransformer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.kv_compression_factor = kv_compression_factor\n    self.kv_sharing = kv_sharing\n    self.use_qkv_bias = use_qkv_bias\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    is_first = True\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"fttransformer_block\" + str(i),\n            FTTransformerEncoder(\n                input_dim,\n                self.n_feats,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                kv_compression_factor,\n                kv_sharing,\n                transformer_activation,\n                is_first,\n            ),\n        )\n        is_first = False\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is: \\((N, H, F, k)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads, \\(F\\) is the number of features/columns and \\(k\\) is the reduced sequence length or dimension, i.e. \\(k = int(kv_{compression \\space factor} \\times s)\\)</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver","title":"TabPerceiver","text":"<pre><code>TabPerceiver(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_cross_attns=1,\n    n_cross_attn_heads=4,\n    n_latents=16,\n    latent_dim=128,\n    n_latent_heads=4,\n    n_latent_blocks=4,\n    n_perceiver_blocks=4,\n    share_weights=False,\n    attn_dropout=0.1,\n    ff_dropout=0.1,\n    ff_factor=4,\n    transformer_activation=\"geglu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>             Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a Perceiver  that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model  or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>             (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                 <code>'standard'</code> )         \u2013          <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>             (<code>int</code>, default:                 <code>32</code> )         \u2013          <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> </li> <li> <code>n_cross_attns</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Number of times each perceiver block will cross attend to the input data (i.e. number of cross attention components per perceiver block). This should normally be 1. However, in the paper they describe some architectures (normally computer vision-related problems) where the Perceiver attends multiple times to the input array. Therefore, maybe multiple cross attention to the input array is also useful in some cases for tabular data  .</p> </li> <li> <code>n_cross_attn_heads</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Number of attention heads for the cross attention component</p> </li> <li> <code>n_latents</code>             (<code>int</code>, default:                 <code>16</code> )         \u2013          <p>Number of latents. This is the \\(N\\) parameter in the paper. As indicated in the paper, this number should be significantly lower than \\(M\\) (the number of columns in the dataset). Setting \\(N\\) closer to \\(M\\) defies the main purpose of the Perceiver, which is to overcome the transformer quadratic bottleneck</p> </li> <li> <code>latent_dim</code>             (<code>int</code>, default:                 <code>128</code> )         \u2013          <p>Latent dimension.</p> </li> <li> <code>n_latent_heads</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Number of attention heads per Latent Transformer</p> </li> <li> <code>n_latent_blocks</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Number of transformer encoder blocks (normalised MHA + normalised FF) per Latent Transformer</p> </li> <li> <code>n_perceiver_blocks</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Number of Perceiver blocks defined as [Cross Attention + Latent Transformer]</p> </li> <li> <code>share_weights</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the weights will be shared between Perceiver blocks</p> </li> <li> <code>attn_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>             (<code>str</code>, default:                 <code>'geglu'</code> )         \u2013          <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>             (<code>ModuleDict</code>)         \u2013          <p>ModuleDict with the Perceiver blocks</p> </li> <li> <code>latents</code>             (<code>Parameter</code>)         \u2013          <p>Latents that will be used for prediction</p> </li> <li> <code>mlp</code>             (<code>Module</code>)         \u2013          <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n... n_perceiver_blocks=2)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_cross_attns: int = 1,\n    n_cross_attn_heads: int = 4,\n    n_latents: int = 16,\n    latent_dim: int = 128,\n    n_latent_heads: int = 4,\n    n_latent_blocks: int = 4,\n    n_perceiver_blocks: int = 4,\n    share_weights: bool = False,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    transformer_activation: str = \"geglu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabPerceiver, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_cross_attns = n_cross_attns\n    self.n_cross_attn_heads = n_cross_attn_heads\n    self.n_latents = n_latents\n    self.latent_dim = latent_dim\n    self.n_latent_heads = n_latent_heads\n    self.n_latent_blocks = n_latent_blocks\n    self.n_perceiver_blocks = n_perceiver_blocks\n    self.share_weights = share_weights\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.latents = nn.init.trunc_normal_(\n        nn.Parameter(torch.empty(n_latents, latent_dim))\n    )\n\n    self.encoder = nn.ModuleDict()\n    first_perceiver_block = self._build_perceiver_block()\n    self.encoder[\"perceiver_block0\"] = first_perceiver_block\n\n    if share_weights:\n        for n in range(1, n_perceiver_blocks):\n            self.encoder[\"perceiver_block\" + str(n)] = first_perceiver_block\n    else:\n        for n in range(1, n_perceiver_blocks):\n            self.encoder[\"perceiver_block\" + str(n)] = self._build_perceiver_block()\n\n    self.mlp_first_hidden_dim = self.latent_dim\n\n    # Mlp\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. If the weights are not shared between perceiver blocks each element of the list will be a list itself containing the Cross Attention and Latent Transformer attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>Cross Attention: \\((N, C, L, F)\\)</p> </li> <li> <p>Latent Attention: \\((N, T, L, L)\\)</p> </li> </ul> <p>WHere \\(N\\) is the batch size, \\(C\\) is the number of Cross Attention heads, \\(L\\) is the number of Latents, \\(F\\) is the number of features/columns in the dataset and \\(T\\) is the number of Latent Attention heads</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer","title":"TabFastFormer","text":"<pre><code>TabFastFormer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_heads=8,\n    use_bias=False,\n    n_blocks=4,\n    attn_dropout=0.1,\n    ff_dropout=0.2,\n    ff_factor=4,\n    share_qv_weights=False,\n    share_weights=False,\n    transformer_activation=\"relu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>             Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a FastFormer that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>             (<code>Dict[str, int]</code>)         \u2013          <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>             (<code>Optional[List[Tuple[str, int]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>             (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                 <code>None</code> )         \u2013          <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>             (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                 <code>'standard'</code> )         \u2013          <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>             (<code>int</code>, default:                 <code>32</code> )         \u2013          <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>             (<code>int</code>, default:                 <code>8</code> )         \u2013          <p>Number of attention heads per FastFormer block</p> </li> <li> <code>use_bias</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Number of FastFormer blocks</p> </li> <li> <code>attn_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Dropout that will be applied to the Additive Attention layers</p> </li> <li> <code>ff_dropout</code>             (<code>float</code>, default:                 <code>0.2</code> )         \u2013          <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>share_qv_weights</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Following the paper, this is a boolean indicating if the Value (\\(V\\)) and the Query (\\(Q\\)) transformation parameters will be shared.</p> </li> <li> <code>share_weights</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>In addition to sharing the \\(V\\) and \\(Q\\) transformation parameters, the parameters across different Fastformer layers can also be shared. Please, see <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> for details</p> </li> <li> <code>transformer_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>MLP hidden dimensions. If not provided no MLP on top of the final FTTransformer block will be used</p> </li> <li> <code>mlp_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>             (<code>Module</code>)         \u2013          <p>Sequence of FasFormer blocks.</p> </li> <li> <code>mlp</code>             (<code>Module</code>)         \u2013          <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_heads: int = 8,\n    use_bias: bool = False,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.2,\n    ff_factor: int = 4,\n    share_qv_weights: bool = False,\n    share_weights: bool = False,\n    transformer_activation: str = \"relu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabFastFormer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_heads = n_heads\n    self.use_bias = use_bias\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.share_qv_weights = share_qv_weights\n    self.share_weights = share_weights\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    first_fastformer_block = FastFormerEncoder(\n        input_dim,\n        n_heads,\n        use_bias,\n        attn_dropout,\n        ff_dropout,\n        ff_factor,\n        share_qv_weights,\n        transformer_activation,\n    )\n    self.encoder.add_module(\"fastformer_block0\", first_fastformer_block)\n    for i in range(1, n_blocks):\n        if share_weights:\n            self.encoder.add_module(\n                \"fastformer_block\" + str(i), first_fastformer_block\n            )\n        else:\n            self.encoder.add_module(\n                \"fastformer_block\" + str(i),\n                FastFormerEncoder(\n                    input_dim,\n                    n_heads,\n                    use_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    share_qv_weights,\n                    transformer_activation,\n                ),\n            )\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and second elements are the \\(\\alpha\\) and \\(\\beta\\) attention weights in the paper.</p> <p>The shape of the attention weights is \\((N, H, F)\\) where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN","title":"BasicRNN","text":"<pre><code>BasicRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.0,\n    bidirectional=False,\n    use_hidden_state=True,\n    padding_idx=1,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>             Bases: <code>BaseWDModelComponent</code></p> <p>Standard text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) that can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the stack of RNNs</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>             (<code>int</code>)         \u2013          <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>             (<code>Literal[lstm, gru]</code>, default:                 <code>'lstm'</code> )         \u2013          <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>             (<code>int</code>, default:                 <code>64</code> )         \u2013          <p>Hidden dim of the RNN</p> </li> <li> <code>n_layers</code>             (<code>int</code>, default:                 <code>3</code> )         \u2013          <p>Number of recurrent layers</p> </li> <li> <code>rnn_dropout</code>             (<code>float</code>, default:                 <code>0.0</code> )         \u2013          <p>Dropout for each RNN layer except the last layer</p> </li> <li> <code>bidirectional</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>use_hidden_state</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> </li> <li> <code>padding_idx</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>head_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>             (<code>Module</code>)         \u2013          <p>word embedding matrix</p> </li> <li> <code>rnn</code>             (<code>Module</code>)         \u2013          <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>             (<code>Module</code>)         \u2013          <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    n_layers: int = 3,\n    rnn_dropout: float = 0.0,\n    bidirectional: bool = False,\n    use_hidden_state: bool = True,\n    padding_idx: int = 1,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(BasicRNN, self).__init__()\n\n    if embed_dim is None and embed_matrix is None:\n        raise ValueError(\n            \"If no 'embed_matrix' is passed, the embedding dimension must\"\n            \"be specified with 'embed_dim'\"\n        )\n\n    if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n        )\n\n    if (\n        embed_dim is not None\n        and embed_matrix is not None\n        and not embed_dim == embed_matrix.shape[1]\n    ):\n        warnings.warn(\n            \"the input embedding dimension {} and the dimension of the \"\n            \"pretrained embeddings {} do not match. The pretrained embeddings \"\n            \"dimension ({}) will be used\".format(\n                embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n            ),\n            UserWarning,\n        )\n\n    self.vocab_size = vocab_size\n    self.embed_trainable = embed_trainable\n    self.embed_dim = embed_dim\n\n    self.rnn_type = rnn_type\n    self.hidden_dim = hidden_dim\n    self.n_layers = n_layers\n    self.rnn_dropout = rnn_dropout\n    self.bidirectional = bidirectional\n    self.use_hidden_state = use_hidden_state\n    self.padding_idx = padding_idx\n\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    # Embeddings\n    if embed_matrix is not None:\n        self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n    else:\n        assert self.embed_dim is not None\n        self.word_embed = nn.Embedding(\n            self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n        )\n\n    # RNN\n    rnn_params = {\n        \"input_size\": self.embed_dim,\n        \"hidden_size\": hidden_dim,\n        \"num_layers\": n_layers,\n        \"bidirectional\": bidirectional,\n        \"dropout\": rnn_dropout,\n        \"batch_first\": True,\n    }\n    if self.rnn_type.lower() == \"lstm\":\n        self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n    elif self.rnn_type.lower() == \"gru\":\n        self.rnn = nn.GRU(**rnn_params)\n    else:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n        )\n\n    self.rnn_output_dim = hidden_dim * 2 if bidirectional else hidden_dim\n\n    # FC-Head (Mlp)\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    else:\n        # simple hack to add readability in the forward pass\n        self.rnn_mlp = nn.Identity()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN","title":"AttentiveRNN","text":"<pre><code>AttentiveRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.1,\n    bidirectional=False,\n    use_hidden_state=True,\n    padding_idx=1,\n    attn_concatenate=True,\n    attn_dropout=0.1,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>             Bases: <code>BasicRNN</code></p> <p>Text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) plus an attention layer. This model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of attention layer</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>             (<code>int</code>)         \u2013          <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>             (<code>Literal[lstm, gru]</code>, default:                 <code>'lstm'</code> )         \u2013          <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>             (<code>int</code>, default:                 <code>64</code> )         \u2013          <p>Hidden dim of the RNN</p> </li> <li> <code>n_layers</code>             (<code>int</code>, default:                 <code>3</code> )         \u2013          <p>Number of recurrent layers</p> </li> <li> <code>rnn_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Dropout for each RNN layer except the last layer</p> </li> <li> <code>bidirectional</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>use_hidden_state</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> </li> <li> <code>padding_idx</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>attn_concatenate</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state.</p> </li> <li> <code>attn_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Internal dropout for the attention mechanism</p> </li> <li> <code>head_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>             (<code>Module</code>)         \u2013          <p>word embedding matrix</p> </li> <li> <code>rnn</code>             (<code>Module</code>)         \u2013          <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>             (<code>Module</code>)         \u2013          <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    n_layers: int = 3,\n    rnn_dropout: float = 0.1,\n    bidirectional: bool = False,\n    use_hidden_state: bool = True,\n    padding_idx: int = 1,\n    attn_concatenate: bool = True,\n    attn_dropout: float = 0.1,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(AttentiveRNN, self).__init__(\n        vocab_size=vocab_size,\n        embed_dim=embed_dim,\n        embed_matrix=embed_matrix,\n        embed_trainable=embed_trainable,\n        rnn_type=rnn_type,\n        hidden_dim=hidden_dim,\n        n_layers=n_layers,\n        rnn_dropout=rnn_dropout,\n        bidirectional=bidirectional,\n        use_hidden_state=use_hidden_state,\n        padding_idx=padding_idx,\n        head_hidden_dims=head_hidden_dims,\n        head_activation=head_activation,\n        head_dropout=head_dropout,\n        head_batchnorm=head_batchnorm,\n        head_batchnorm_last=head_batchnorm_last,\n        head_linear_first=head_linear_first,\n    )\n\n    # Embeddings and RNN defined in the BasicRNN inherited class\n\n    # Attention\n    self.attn_concatenate = attn_concatenate\n    self.attn_dropout = attn_dropout\n\n    if bidirectional and attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 4\n    elif bidirectional or attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 2\n    else:\n        self.rnn_output_dim = hidden_dim\n    self.attn = ContextAttention(\n        self.rnn_output_dim, attn_dropout, sum_along_seq=True\n    )\n\n    # FC-Head (Mlp)\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights</p> <p>The shape of the attention weights is \\((N, S)\\), where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN","title":"StackedAttentiveRNN","text":"<pre><code>StackedAttentiveRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    bidirectional=False,\n    padding_idx=1,\n    n_blocks=3,\n    attn_concatenate=False,\n    attn_dropout=0.1,\n    with_addnorm=False,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>             Bases: <code>BaseWDModelComponent</code></p> <p>Text classifier/regressor comprised by a stack of blocks: <code>[RNN + Attention]</code>. This can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the attentiob blocks</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>             (<code>int</code>)         \u2013          <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>             (<code>Literal[lstm, gru]</code>, default:                 <code>'lstm'</code> )         \u2013          <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>             (<code>int</code>, default:                 <code>64</code> )         \u2013          <p>Hidden dim of the RNN</p> </li> <li> <code>bidirectional</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>padding_idx</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>n_blocks</code>             (<code>int</code>, default:                 <code>3</code> )         \u2013          <p>Number of attention blocks. Each block is comprised by an RNN and a Context Attention Encoder</p> </li> <li> <code>attn_concatenate</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state or simply</p> </li> <li> <code>attn_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Internal dropout for the attention mechanism</p> </li> <li> <code>with_addnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the output of each block will be added to the input and normalised</p> </li> <li> <code>head_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>             (<code>Module</code>)         \u2013          <p>word embedding matrix</p> </li> <li> <code>rnn</code>             (<code>Module</code>)         \u2013          <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>             (<code>Module</code>)         \u2013          <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    bidirectional: bool = False,\n    padding_idx: int = 1,\n    n_blocks: int = 3,\n    attn_concatenate: bool = False,\n    attn_dropout: float = 0.1,\n    with_addnorm: bool = False,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(StackedAttentiveRNN, self).__init__()\n\n    if (\n        embed_dim is not None\n        and embed_matrix is not None\n        and not embed_dim == embed_matrix.shape[1]\n    ):\n        warnings.warn(\n            \"the input embedding dimension {} and the dimension of the \"\n            \"pretrained embeddings {} do not match. The pretrained embeddings \"\n            \"dimension ({}) will be used\".format(\n                embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n            ),\n            UserWarning,\n        )\n\n    if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n        )\n\n    self.vocab_size = vocab_size\n    self.embed_trainable = embed_trainable\n    self.embed_dim = embed_dim\n\n    self.rnn_type = rnn_type\n    self.hidden_dim = hidden_dim\n    self.bidirectional = bidirectional\n    self.padding_idx = padding_idx\n\n    self.n_blocks = n_blocks\n    self.attn_concatenate = attn_concatenate\n    self.attn_dropout = attn_dropout\n    self.with_addnorm = with_addnorm\n\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    # Embeddings\n    self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n\n    # Linear Projection: if embed_dim is different that the input of the\n    # attention blocks we add a linear projection\n    if bidirectional and attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 4\n    elif bidirectional or attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 2\n    else:\n        self.rnn_output_dim = hidden_dim\n\n    if self.rnn_output_dim != self.embed_dim:\n        self.embed_proj: Union[nn.Linear, nn.Identity] = nn.Linear(\n            self.embed_dim, self.rnn_output_dim\n        )\n    else:\n        self.embed_proj = nn.Identity()\n\n    # RNN\n    rnn_params = {\n        \"input_size\": self.rnn_output_dim,\n        \"hidden_size\": hidden_dim,\n        \"bidirectional\": bidirectional,\n        \"batch_first\": True,\n    }\n    if self.rnn_type.lower() == \"lstm\":\n        self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n    elif self.rnn_type.lower() == \"gru\":\n        self.rnn = nn.GRU(**rnn_params)\n\n    # FC-Head (Mlp)\n    self.attention_blks = nn.ModuleList()\n    for i in range(n_blocks):\n        self.attention_blks.append(\n            ContextAttentionEncoder(\n                self.rnn,\n                self.rnn_output_dim,\n                attn_dropout,\n                attn_concatenate,\n                with_addnorm=with_addnorm if i != n_blocks - 1 else False,\n                sum_along_seq=i == n_blocks - 1,\n            )\n        )\n\n    # Mlp\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    else:\n        # simple hack to add readability in the forward pass\n        self.rnn_mlp = nn.Identity()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, S)\\) Where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer","title":"Transformer","text":"<pre><code>Transformer(\n    vocab_size,\n    seq_length,\n    input_dim,\n    n_heads,\n    n_blocks,\n    attn_dropout=0.1,\n    ff_dropout=0.1,\n    ff_factor=4,\n    activation=\"gelu\",\n    use_linear_attention=False,\n    use_flash_attention=False,\n    padding_idx=0,\n    with_cls_token=False,\n    *,\n    with_pos_encoding=True,\n    pos_encoding_dropout=0.1,\n    pos_encoder=None\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Basic Encoder-Only Transformer Model for text classification/regression. As all other models in the library this model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: This model is introduced in the context of recommendation systems and thought for sequences of any nature (e.g. items). It can, of course, still be used for text. However, at this stage, we have decided to not include the possibility of loading pretrained word vectors since we aim to integrate the library wit Huggingface in the (hopefully) near future</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>             (<code>int</code>)         \u2013          <p>Number of words in the vocabulary</p> </li> <li> <code>input_dim</code>             (<code>int</code>)         \u2013          <p>Dimension of the token embeddings</p> <p>Param aliases: <code>embed_dim</code>, <code>d_model</code>. </p> </li> <li> <code>seq_length</code>             (<code>int</code>)         \u2013          <p>Input sequence length</p> </li> <li> <code>n_heads</code>             (<code>int</code>)         \u2013          <p>Number of attention heads per Transformer block</p> </li> <li> <code>n_blocks</code>             (<code>int</code>)         \u2013          <p>Number of Transformer blocks</p> </li> <li> <code>attn_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>             (<code>int</code>, default:                 <code>4</code> )         \u2013          <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>activation</code>             (<code>str</code>, default:                 <code>'gelu'</code> )         \u2013          <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>padding_idx</code>             (<code>int</code>, default:                 <code>0</code> )         \u2013          <p>index of the padding token in the padded-tokenised sequences.</p> </li> <li> <code>with_cls_token</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if a <code>'[CLS]'</code> token is included in the tokenized sequences. If present, the final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. NOTE: if included in the tokenized sequences it must be inserted as the first token in the sequences.</p> </li> <li> <code>with_pos_encoding</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating if positional encoding will be used</p> </li> <li> <code>pos_encoding_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Positional encoding dropout</p> </li> <li> <code>pos_encoder</code>             (<code>Optional[Module]</code>, default:                 <code>None</code> )         \u2013          <p>This model uses by default a standard positional encoding approach. However, any custom positional encoder can also be used and pass to the Transformer model via the 'pos_encoder' parameter</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embedding</code>             (<code>Module</code>)         \u2013          <p>Standard token embedding layer</p> </li> <li> <code>pos_encoder</code>             (<code>Module</code>)         \u2013          <p>Positional Encoder</p> </li> <li> <code>encoder</code>             (<code>Module</code>)         \u2013          <p>Sequence of Transformer blocks</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Transformer\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/miscellaneous/basic_transformer.py</code> <pre><code>@alias(\"input_dim\", [\"embed_dim\", \"d_model\"])\n@alias(\"seq_length\", [\"max_length\", \"maxlen\"])\ndef __init__(\n    self,\n    vocab_size: int,\n    seq_length: int,\n    input_dim: int,\n    n_heads: int,\n    n_blocks: int,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    activation: str = \"gelu\",\n    use_linear_attention: bool = False,\n    use_flash_attention: bool = False,\n    padding_idx: int = 0,\n    with_cls_token: bool = False,\n    *,  # from here on pos encoding args\n    with_pos_encoding: bool = True,\n    pos_encoding_dropout: float = 0.1,\n    pos_encoder: Optional[nn.Module] = None,\n):\n    super().__init__()\n\n    self.input_dim = input_dim\n    self.seq_length = seq_length\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.activation = activation\n    self.use_linear_attention = use_linear_attention\n    self.use_flash_attention = use_flash_attention\n    self.padding_idx = padding_idx\n    self.with_cls_token = with_cls_token\n    self.with_pos_encoding = with_pos_encoding\n    self.pos_encoding_dropout = pos_encoding_dropout\n\n    self.embedding = nn.Embedding(\n        vocab_size, input_dim, padding_idx=self.padding_idx\n    )\n\n    if with_pos_encoding:\n        if pos_encoder is not None:\n            self.pos_encoder: Union[nn.Module, nn.Identity, PositionalEncoding] = (\n                pos_encoder\n            )\n        else:\n            self.pos_encoder = PositionalEncoding(\n                input_dim, pos_encoding_dropout, seq_length\n            )\n    else:\n        self.pos_encoder = nn.Identity()\n\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"transformer_block\" + str(i),\n            TransformerEncoder(\n                input_dim,\n                n_heads,\n                False,  # use_qkv_bias\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                activation,\n                use_linear_attention,\n                use_flash_attention,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel","title":"HFModel","text":"<pre><code>HFModel(\n    model_name,\n    use_cls_token=True,\n    trainable_parameters=None,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n    verbose=False,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around the Hugging Face transformers library. It can be used as the text component of a Wide &amp; Deep model or independently by itself.</p> <p>At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>             (<code>str</code>)         \u2013          <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>use_cls_token</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating whether to use the [CLS] token or the mean of the sequence of hidden states as the sentence embedding</p> </li> <li> <code>trainable_parameters</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the names of the model parameters that will be trained. If None, none of the parameters will be trainable</p> </li> <li> <code>head_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the head</p> </li> <li> <code>head_batchnorm_last</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> <li> <code>verbose</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>If True, it will print information about the model</p> </li> <li> <code>**kwargs</code>         \u2013          <p>Additional kwargs to be passed to the model</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>head</code>             (<code>Module</code>)         \u2013          <p>Stack of dense layers on top of the transformer. This will only exists if <code>head_layers_dim</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n&gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code> <pre><code>@alias(\"use_cls_token\", [\"use_special_token\"])\ndef __init__(\n    self,\n    model_name: str,\n    use_cls_token: bool = True,\n    trainable_parameters: Optional[List[str]] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n    verbose: bool = False,\n    **kwargs,\n):\n    super().__init__()\n\n    # TO DO: add warning regarging ELECTRA as ELECTRA does not have a cls\n    # token.  Research what happens with ELECTRA\n    self.model_name = model_name\n    self.use_cls_token = use_cls_token\n    self.trainable_parameters = trainable_parameters\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n    self.verbose = verbose\n    self.kwargs = kwargs\n\n    if self.verbose and self.use_cls_token:\n        warnings.warn(\n            \"The model will use the [CLS] token. Make sure the tokenizer \"\n            \"was run with add_special_tokens=True\",\n            UserWarning,\n        )\n\n    self.model_class = get_model_class(model_name)\n\n    self.config, self.model = get_config_and_model(self.model_name)\n\n    self.output_attention_weights = kwargs.get(\"output_attentions\", False)\n\n    if self.trainable_parameters is not None:\n        for n, p in self.model.named_parameters():\n            p.requires_grad = any([tl in n for tl in self.trainable_parameters])\n\n    # FC-Head (Mlp). Note that the FC head will always be trainable\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.config.hidden_size] + self.head_hidden_dims\n        self.head = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight","title":"attention_weight  <code>property</code>","text":"<pre><code>attention_weight\n</code></pre> <p>Returns the attention weights if the model was created with the output_attention_weights=True argument. If not, it will raise an AttributeError.</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the sequence length.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision","title":"Vision","text":"<pre><code>Vision(\n    pretrained_model_setup=None,\n    n_trainable=None,\n    trainable_params=None,\n    channel_sizes=[64, 128, 256, 512],\n    kernel_sizes=[7, 3, 3, 3],\n    strides=[2, 1, 1, 1],\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=0.1,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>             Bases: <code>BaseWDModelComponent</code></p> <p>Defines a standard image classifier/regressor using a pretrained network or a sequence of convolution layers that can be used as the <code>deepimage</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: this class represents the integration  between <code>pytorch-widedeep</code> and <code>torchvision</code>. New architectures will be  available as they are added to <code>torchvision</code>. In a distant future we aim  to bring transformer-based architectures as well. However, simple  CNN-based architectures (and even MLP-based) seem to produce SoTA  results. For the time being, we describe below the options available  through this class</p> <p>Parameters:</p> <ul> <li> <code>pretrained_model_setup</code>             (<code>Union[str, Dict[str, Union[str, WeightsEnum]]]</code>, default:                 <code>None</code> )         \u2013          <p>Name of the pretrained model. Should be a variant of the following architectures: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3', 'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'. if <code>pretrained_model_setup = None</code> a basic, fully trainable CNN will be used. Alternatively, since Torchvision 0.13 one can use pretrained models with different weigths. Therefore, <code>pretrained_model_setup</code> can also be dictionary with the name of the model and the weights (e.g. <code>{'resnet50': ResNet50_Weights.DEFAULT}</code> or <code>{'resnet50': \"IMAGENET1K_V2\"}</code>).  Aliased as <code>pretrained_model_name</code>.</p> </li> <li> <code>n_trainable</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>Number of trainable layers starting from the layer closer to the output neuron(s). Note that this number DOES NOT take into account the so-called 'head' which is ALWAYS trainable. If <code>trainable_params</code> is not None this parameter will be ignored</p> </li> <li> <code>trainable_params</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List of strings containing the names (or substring within the name) of the parameters that will be trained. For example, if we use a 'resnet18' pretrained model and we set <code>trainable_params = ['layer4']</code> only the parameters of 'layer4' of the network (and the head, as mentioned before) will be trained. Note that setting this or the previous parameter involves some knowledge of the architecture used.</p> </li> <li> <code>channel_sizes</code>             (<code>List[int]</code>, default:                 <code>[64, 128, 256, 512]</code> )         \u2013          <p>List of integers with the channel sizes of a CNN in case we choose not to use a pretrained model</p> </li> <li> <code>kernel_sizes</code>             (<code>Union[int, List[int]]</code>, default:                 <code>[7, 3, 3, 3]</code> )         \u2013          <p>List of integers with the kernel sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> </li> <li> <code>strides</code>             (<code>Union[int, List[int]]</code>, default:                 <code>[2, 1, 1, 1]</code> )         \u2013          <p>List of integers with the stride sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> </li> <li> <code>head_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the number of neurons per dense layer in the head. e.g: [64,32]</p> </li> <li> <code>head_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>             (<code>Union[float, List[float]]</code>, default:                 <code>0.1</code> )         \u2013          <p>float indicating the dropout between the dense layers.</p> </li> <li> <code>head_batchnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>head_batchnorm_last</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>head_linear_first</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>features</code>             (<code>Module</code>)         \u2013          <p>The pretrained model or Standard CNN plus the optional head</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Vision\n&gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n&gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n&gt;&gt;&gt; out = model(X_img)\n</code></pre> Source code in <code>pytorch_widedeep/models/image/vision.py</code> <pre><code>@alias(\"pretrained_model_setup\", [\"pretrained_model_name\"])\ndef __init__(\n    self,\n    pretrained_model_setup: Union[str, Dict[str, Union[str, WeightsEnum]]] = None,\n    n_trainable: Optional[int] = None,\n    trainable_params: Optional[List[str]] = None,\n    channel_sizes: List[int] = [64, 128, 256, 512],\n    kernel_sizes: Union[int, List[int]] = [7, 3, 3, 3],\n    strides: Union[int, List[int]] = [2, 1, 1, 1],\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Union[float, List[float]] = 0.1,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(Vision, self).__init__()\n\n    self._check_pretrained_model_setup(\n        pretrained_model_setup, n_trainable, trainable_params\n    )\n\n    self.pretrained_model_setup = pretrained_model_setup\n    self.n_trainable = n_trainable\n    self.trainable_params = trainable_params\n    self.channel_sizes = channel_sizes\n    self.kernel_sizes = kernel_sizes\n    self.strides = strides\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    self.features, self.backbone_output_dim = self._get_features()\n\n    if pretrained_model_setup is not None:\n        self._freeze(self.features)\n\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.backbone_output_dim] + self.head_hidden_dims\n        self.vision_mlp = MLP(\n            head_hidden_dims,\n            self.head_activation,\n            self.head_dropout,\n            self.head_batchnorm,\n            self.head_batchnorm_last,\n            self.head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser","title":"ModelFuser","text":"<pre><code>ModelFuser(\n    models,\n    *,\n    fusion_method,\n    projection_method=None,\n    custom_head=None,\n    head_hidden_dims=None,\n    head_activation=None,\n    head_dropout=None,\n    head_batchnorm=None,\n    head_batchnorm_last=None,\n    head_linear_first=None\n)\n</code></pre> <p>             Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around a list of models that are associated to the different text and/or image columns (and datasets) The class is designed to 'fuse' the models using a variety of methods.</p> <p>Parameters:</p> <ul> <li> <code>models</code>             (<code>List[BaseWDModelComponent]</code>)         \u2013          <p>List of models whose outputs will be fused</p> </li> <li> <code>fusion_method</code>             (<code>Union[Literal[concatenate, mean, max, sum, mult, head], List[Literal[concatenate, mean, max, sum, mult, head]]]</code>)         \u2013          <p>Method to fuse the output of the models. It can be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'head'] or a list of those. If a list is provided the output of the models will be fused using all the methods in the list and the final output will be the concatenation of the outputs of each method</p> </li> <li> <code>projection_method</code>             (<code>Optional[Literal[min, max, mean]]</code>, default:                 <code>None</code> )         \u2013          <p>If the fusion_method is not 'concatenate', this parameter will determine how to project the output of the models to a common dimension. It can be one of ['min', 'max', 'mean']. Default is None</p> </li> <li> <code>custom_head</code>             (<code>Optional[Union[BaseWDModelComponent, Module]]</code>, default:                 <code>None</code> )         \u2013          <p>Custom head to be used to fuse the output of the models. If provided, this will take precedence over head_hidden_dims. Also, if provided, 'projection_method' will be ignored.</p> </li> <li> <code>head_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the number of neurons per layer in the custom head. If custom_head is provided, this parameter will be ignored</p> </li> <li> <code>head_activation</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Activation function to be used in the custom head. Default is None</p> </li> <li> <code>head_dropout</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>Dropout to be used in the custom head. Default is None</p> </li> <li> <code>head_batchnorm</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Whether to use batchnorm in the custom head. Default is None</p> </li> <li> <code>head_batchnorm_last</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>head_linear_first</code>             (<code>Optional[bool]</code>, default:                 <code>None</code> )         \u2013          <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>head</code>             (<code>Module or BaseWDModelComponent</code>)         \u2013          <p>Custom head to be used to fuse the output of the models. If custom_head is provided, this will take precedence over head_hidden_dims</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n... 'text_col2': ['goodbye world', 'this is another test']})\n&gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n...     text_col=\"text_col1\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n...     text_col=\"text_col2\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n&gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n&gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n&gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n&gt;&gt;&gt; rnn1 = BasicRNN(\n...     vocab_size=len(text_preprocessor_1.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; rnn2 = BasicRNN(\n...     vocab_size=len(text_preprocessor_2.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n&gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n</code></pre> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>def __init__(\n    self,\n    models: List[BaseWDModelComponent],\n    *,\n    fusion_method: Union[\n        Literal[\n            \"concatenate\",\n            \"mean\",\n            \"max\",\n            \"sum\",\n            \"mult\",\n            \"head\",\n        ],\n        List[Literal[\"concatenate\", \"mean\", \"max\", \"sum\", \"mult\", \"head\"]],\n    ],\n    projection_method: Optional[Literal[\"min\", \"max\", \"mean\"]] = None,\n    custom_head: Optional[Union[BaseWDModelComponent, nn.Module]] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: Optional[str] = None,\n    head_dropout: Optional[float] = None,\n    head_batchnorm: Optional[bool] = None,\n    head_batchnorm_last: Optional[bool] = None,\n    head_linear_first: Optional[bool] = None,\n) -&gt; None:\n    super(ModelFuser, self).__init__()\n\n    self.models = nn.ModuleList(models)\n    self.fusion_method = fusion_method\n    self.projection_method = projection_method\n\n    self.all_output_dim_equal = all(\n        model.output_dim == self.models[0].output_dim for model in self.models\n    )\n\n    self.check_input_parameters()\n\n    if self.fusion_method == \"head\":\n        assert (\n            head_hidden_dims is not None or custom_head is not None\n        ), \"When using 'head' as fusion_method, either head_hidden_dims or custom_head must be provided\"\n        if custom_head is not None:\n            # custom_head takes precedence over head_hidden_dims (in case\n            # both are provided)\n            assert hasattr(\n                custom_head, \"output_dim\"\n            ), \"custom_head must have an 'output_dim' property\"\n            self.head: Union[BaseWDModelComponent, nn.Module] = custom_head\n        else:\n            assert head_hidden_dims is not None\n            self.head_hidden_dims = head_hidden_dims\n            self.head_activation = head_activation\n            self.head_dropout = head_dropout\n            self.head_batchnorm = head_batchnorm\n            self.head_batchnorm_last = head_batchnorm_last\n            self.head_linear_first = head_linear_first\n\n            self.head = MLP(\n                d_hidden=[sum([model.output_dim for model in self.models])]\n                + self.head_hidden_dims,\n                activation=(\n                    \"relu\" if self.head_activation is None else self.head_activation\n                ),\n                dropout=0.0 if self.head_dropout is None else self.head_dropout,\n                batchnorm=(\n                    False if self.head_batchnorm is None else self.head_batchnorm\n                ),\n                batchnorm_last=(\n                    False\n                    if self.head_batchnorm_last is None\n                    else self.head_batchnorm_last\n                ),\n                linear_first=(\n                    True\n                    if self.head_linear_first is None\n                    else self.head_linear_first\n                ),\n            )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>Returns the output dimension of the model.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.project","title":"project","text":"<pre><code>project(X)\n</code></pre> <p>Projects the output of the models to a common dimension.</p> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>def project(self, X: List[Tensor]) -&gt; List[Tensor]:\n    r\"\"\"Projects the output of the models to a common dimension.\"\"\"\n\n    if self.all_output_dim_equal and self.projection_method is None:\n        return X\n\n    output_dims = [model.output_dim for model in self.models]\n\n    if self.projection_method == \"min\":\n        proj_dim = min(output_dims)\n        idx = output_dims.index(proj_dim)\n    elif self.projection_method == \"max\":\n        proj_dim = max(output_dims)\n        idx = output_dims.index(proj_dim)\n    elif self.projection_method == \"mean\":\n        proj_dim = int(sum(output_dims) / len(output_dims))\n        idx = None\n    else:\n        raise ValueError(\"projection_method must be one of ['min', 'max', 'mean']\")\n\n    x_proj: List[Tensor] = []\n    for i, x in enumerate(X):\n        if i == idx:\n            x_proj.append(x)\n        else:\n            x_proj.append(\n                nn.Linear(output_dims[i], proj_dim, bias=False, device=x.device)(x)\n            )\n\n    return x_proj\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.wide_deep.WideDeep","title":"WideDeep","text":"<pre><code>WideDeep(\n    wide=None,\n    deeptabular=None,\n    deeptext=None,\n    deepimage=None,\n    deephead=None,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=0.1,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=True,\n    enforce_positive=False,\n    enforce_positive_activation=\"softplus\",\n    pred_dim=1,\n    with_fds=False,\n    **fds_config\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code> <code>deeptext</code> and <code>deepimage</code> models.</p> <p>Note that all models described so far in this library must be passed to the <code>WideDeep</code> class once constructed. This is because the models output the last layer before the prediction layer. Such prediction layer is added by the <code>WideDeep</code> class as it collects the components for every data mode.</p> <p>There are two options to combine these models that correspond to the two main architectures that <code>pytorch-widedeep</code> can build.</p> <ul> <li> <p>Directly connecting the output of the model components to an ouput neuron(s).</p> </li> <li> <p>Adding a <code>Fully-Connected Head</code> (FC-Head) on top of the deep models.   This FC-Head will combine the output form the <code>deeptabular</code>, <code>deeptext</code> and   <code>deepimage</code> and will be then connected to the output neuron(s).</p> </li> </ul> <p>Parameters:</p> <ul> <li> <code>wide</code>             (<code>Optional[Module]</code>, default:                 <code>None</code> )         \u2013          <p><code>Wide</code> model. This is a linear model where the non-linearities are captured via crossed-columns.</p> </li> <li> <code>deeptabular</code>             (<code>Optional[BaseWDModelComponent]</code>, default:                 <code>None</code> )         \u2013          <p>Currently this library implements a number of possible architectures for the <code>deeptabular</code> component. See the documenation of the package.</p> </li> <li> <code>deeptext</code>             (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                 <code>None</code> )         \u2013          <p>Currently this library implements a number of possible architectures for the <code>deeptext</code> component. See the documenation of the package. Note that <code>deeptext</code> can be a list of models. This is useful when using multiple text inputs.</p> </li> <li> <code>deepimage</code>             (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                 <code>None</code> )         \u2013          <p>Currently this library uses <code>torchvision</code> and implements a number of possible architectures for the <code>deepimage</code> component. See the documenation of the package. Note that <code>deepimage</code> can be a list of models. This is useful when using multiple image inputs.</p> </li> <li> <code>deephead</code>             (<code>Optional[BaseWDModelComponent]</code>, default:                 <code>None</code> )         \u2013          <p>Alternatively, the user can pass a custom model that will receive the output of the deep component. If <code>deephead</code> is not None all the previous fc-head parameters will be ignored</p> </li> <li> <code>head_hidden_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Activation function for the dense layers in the head. Currently <code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p> </li> <li> <code>head_dropout</code>             (<code>float</code>, default:                 <code>0.1</code> )         \u2013          <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the <code>'rnn_mlp'</code></p> </li> <li> <code>head_batchnorm_last</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> <li> <code>enforce_positive</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the output from the final layer must be positive. This is important if you are using loss functions with non-negative input restrictions, e.g. RMSLE, or if you know your predictions are bounded in between 0 and inf</p> </li> <li> <code>enforce_positive_activation</code>             (<code>str</code>, default:                 <code>'softplus'</code> )         \u2013          <p>Activation function to enforce that the final layer has a positive output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p> </li> <li> <code>pred_dim</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Size of the final wide and deep output layer containing the predictions. <code>1</code> for regression and binary classification or number of classes for multiclass classification.</p> </li> <li> <code>with_fds</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if Feature Distribution Smoothing (FDS) will be applied before the final prediction layer. Only available for regression problems. See Delving into Deep Imbalanced Regression for details.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**fds_config</code>         \u2013          <p>Dictionary with the parameters to be used when using Feature Distribution Smoothing. Please, see the docs for the <code>FDSLayer</code>.  NOTE: Feature Distribution Smoothing  is available when using ONLY a <code>deeptabular</code> component  NOTE: We consider Feature Distribution Smoothing absolutely experimental and we recommend the user to not use it unless the corresponding publication is well understood</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n</code></pre> <p> NOTE: It is possible to use custom components to  build Wide &amp; Deep models. Simply, build them and pass them as the  corresponding parameters. Note that the custom models MUST return a last  layer of activations(i.e. not the final prediction) so that  these  activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with  the size of these last layers of activations. See for example  <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p> Source code in <code>pytorch_widedeep/models/wide_deep.py</code> <pre><code>@alias(  # noqa: C901\n    \"pred_dim\",\n    [\"num_class\", \"pred_size\"],\n)\ndef __init__(\n    self,\n    wide: Optional[nn.Module] = None,\n    deeptabular: Optional[BaseWDModelComponent] = None,\n    deeptext: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deepimage: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deephead: Optional[BaseWDModelComponent] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: float = 0.1,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = True,\n    enforce_positive: bool = False,\n    enforce_positive_activation: str = \"softplus\",\n    pred_dim: int = 1,\n    with_fds: bool = False,\n    **fds_config,\n):\n    super(WideDeep, self).__init__()\n\n    self._check_inputs(\n        wide,\n        deeptabular,\n        deeptext,\n        deepimage,\n        deephead,\n        head_hidden_dims,\n        pred_dim,\n        with_fds,\n    )\n\n    # this attribute will be eventually over-written by the Trainer's\n    # device. Acts here as a 'placeholder'.\n    self.wd_device: Optional[str] = None\n\n    # required as attribute just in case we pass a deephead\n    self.pred_dim = pred_dim\n\n    self.with_fds = with_fds\n    self.enforce_positive = enforce_positive\n\n    # The main 5 components of the wide and deep assemble: wide,\n    # deeptabular, deeptext, deepimage and deephead\n    self.with_deephead = deephead is not None or head_hidden_dims is not None\n    if deephead is None and head_hidden_dims is not None:\n        self.deephead = self._build_deephead(\n            deeptabular,\n            deeptext,\n            deepimage,\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    elif deephead is not None:\n        self.deephead = nn.Sequential(\n            deephead, nn.Linear(deephead.output_dim, self.pred_dim)\n        )\n    else:\n        # for consistency with other components we default to None\n        self.deephead = None\n\n    self.wide = wide\n    self.deeptabular, self.deeptext, self.deepimage = self._set_model_components(\n        deeptabular, deeptext, deepimage, self.with_deephead\n    )\n\n    if self.with_fds:\n        self.fds_layer = FDSLayer(feature_dim=self.deeptabular.output_dim, **fds_config)  # type: ignore[arg-type]\n\n    if self.enforce_positive:\n        self.enf_pos = get_activation_fn(enforce_positive_activation)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.fds_layer.FDSLayer","title":"FDSLayer","text":"<pre><code>FDSLayer(\n    feature_dim,\n    granularity=100,\n    y_max=None,\n    y_min=None,\n    start_update=0,\n    start_smooth=2,\n    kernel=\"gaussian\",\n    ks=5,\n    sigma=2,\n    momentum=0.9,\n    clip_min=None,\n    clip_max=None,\n)\n</code></pre> <p>             Bases: <code>Module</code></p> <p>Feature Distribution Smoothing layer. Please, see Delving into Deep Imbalanced Regression for details.</p> <p> NOTE: this is NOT an available model per se,  but more a utility that can be used as we run a <code>WideDeep</code> model.  The parameters of this extra layers can be set as the class  <code>WideDeep</code> is instantiated via the keyword arguments <code>fds_config</code>.</p> <p> NOTE: Feature Distribution Smoothing is  available when using ONLY a <code>deeptabular</code> component</p> <p> NOTE: We consider this feature absolutely experimental and we recommend the user to not use it unless the corresponding publication is well understood</p> <p>The code here is based on the code at the official repo</p> <p>Parameters:</p> <ul> <li> <code>feature_dim</code>             (<code>int</code>)         \u2013          <p>input dimension size, i.e. output size of previous layer. This will be the dimension of the output from the <code>deeptabular</code> component</p> </li> <li> <code>granularity</code>             (<code>int</code>, default:                 <code>100</code> )         \u2013          <p>number of bins that the target \\(y\\) is divided into and that will be used to compute the features' statistics (mean and variance)</p> </li> <li> <code>y_max</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>\\(y\\) upper limit to be considered when binning</p> </li> <li> <code>y_min</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>\\(y\\) lower limit to be considered when binning</p> </li> <li> <code>start_update</code>             (<code>int</code>, default:                 <code>0</code> )         \u2013          <p>number of _'waiting epochs' after which the FDS layer will start to update its statistics</p> </li> <li> <code>start_smooth</code>             (<code>int</code>, default:                 <code>2</code> )         \u2013          <p>number of _'waiting epochs' after which the FDS layer will start smoothing the feature distributions</p> </li> <li> <code>kernel</code>             (<code>Literal[gaussian, triang, laplace]</code>, default:                 <code>'gaussian'</code> )         \u2013          <p>choice of smoothing kernel</p> </li> <li> <code>ks</code>             (<code>int</code>, default:                 <code>5</code> )         \u2013          <p>kernel window size</p> </li> <li> <code>sigma</code>             (<code>float</code>, default:                 <code>2</code> )         \u2013          <p>if a 'gaussian' or 'laplace' kernels are used, this is the corresponding standard deviation</p> </li> <li> <code>momentum</code>             (<code>Optional[float]</code>, default:                 <code>0.9</code> )         \u2013          <p>to train the layer the authors used a momentum update of the running statistics across each epoch. Set to 0.9 in the paper.</p> </li> <li> <code>clip_min</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>this parameter is used to clip the ratio between the so called running variance and the smoothed variance, and is introduced for numerical stability. We leave it as optional as we did not find a notable improvement in our experiments. The authors used a value of 0.1</p> </li> <li> <code>clip_max</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>same as <code>clip_min</code> but for the upper limit.We leave it as optional as we did not find a notable improvement in our experiments. The authors used a value of 10.</p> </li> </ul> Source code in <code>pytorch_widedeep/models/fds_layer.py</code> <pre><code>def __init__(\n    self,\n    feature_dim: int,\n    granularity: int = 100,\n    y_max: Optional[float] = None,\n    y_min: Optional[float] = None,\n    start_update: int = 0,\n    start_smooth: int = 2,\n    kernel: Literal[\"gaussian\", \"triang\", \"laplace\"] = \"gaussian\",\n    ks: int = 5,\n    sigma: float = 2,\n    momentum: Optional[float] = 0.9,\n    clip_min: Optional[float] = None,\n    clip_max: Optional[float] = None,\n):\n    \"\"\"\n    Feature Distribution Smoothing layer. Please, see\n    [Delving into Deep Imbalanced Regression](https:/arxiv.org/abs/2102.09554)\n    for details.\n\n    :information_source: **NOTE**: this is NOT an available model per se,\n     but more a utility that can be used as we run a `WideDeep` model.\n     The parameters of this extra layers can be set as the class\n     `WideDeep` is instantiated via the keyword arguments `fds_config`.\n\n    :information_source: **NOTE**: Feature Distribution Smoothing is\n     available when using ONLY a `deeptabular` component\n\n    :information_source: **NOTE**: We consider this feature absolutely\n    experimental and we recommend the user to not use it unless the\n    corresponding [publication](https://arxiv.org/abs/2102.09554) is\n    well understood\n\n    The code here is based on the code at the\n    [official repo](https://github.com/YyzHarry/imbalanced-regression)\n\n    Parameters\n    ----------\n    feature_dim: int,\n        input dimension size, i.e. output size of previous layer. This\n        will be the dimension of the output from the `deeptabular`\n        component\n    granularity: int = 100,\n        number of bins that the target $y$ is divided into and that will\n        be used to compute the features' statistics (mean and variance)\n    y_max: Optional[float] = None,\n        $y$ upper limit to be considered when binning\n    y_min: Optional[float] = None,\n        $y$ lower limit to be considered when binning\n    start_update: int = 0,\n        number of _'waiting epochs' after which the FDS layer will start\n        to update its statistics\n    start_smooth: int = 1,\n        number of _'waiting epochs' after which the FDS layer will start\n        smoothing the feature distributions\n    kernel: Literal[\"gaussian\", \"triang\", \"laplace\", None] = \"gaussian\",\n        choice of smoothing kernel\n    ks: int = 5,\n        kernel window size\n    sigma: Union[int, float] = 2,\n        if a _'gaussian'_ or _'laplace'_ kernels are used, this is the\n        corresponding standard deviation\n    momentum: float = 0.9,\n        to train the layer the authors used a momentum update of the running\n        statistics across each epoch. Set to 0.9 in the paper.\n    clip_min: Optional[float] = None,\n        this parameter is used to clip the ratio between the so called\n        running variance and the smoothed variance, and is introduced for\n        numerical stability. We leave it as optional as we did not find a\n        notable improvement in our experiments. The authors used a value\n        of 0.1\n    clip_max: Optional[float] = None,\n        same as `clip_min` but for the upper limit.We leave it as optional\n        as we did not find a notable improvement in our experiments. The\n        authors used a value of 10.\n    \"\"\"\n    super(FDSLayer, self).__init__()\n    assert (\n        start_update + 1 &lt; start_smooth\n    ), \"initial update must start at least 2 epoch before smoothing\"\n\n    self.feature_dim = feature_dim\n    self.granularity = granularity\n    self.y_max = y_max\n    self.y_min = y_min\n    self.kernel_window = torch.tensor(\n        get_kernel_window(kernel, ks, sigma), dtype=torch.float32\n    )\n    self.half_ks = (ks - 1) // 2\n    self.momentum = momentum\n    self.start_update = start_update\n    self.start_smooth = start_smooth\n    self.clip_min = clip_min\n    self.clip_max = clip_max\n\n    self.pred_layer = nn.Linear(feature_dim, 1)\n\n    self._register_buffers()\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html","title":"The <code>preprocessing</code> module","text":"<p>This module contains the classes that are used to prepare the data before being passed to the models. There is one Preprocessor per data mode or model component (<code>wide</code>, <code>deeptabular</code>, <code>deepimage</code> and <code>deeptext</code>) with the exception of the <code>deeptext</code> component. In this case, two processors are available: one for the case when no Hugging Face model is used (<code>TextPreprocessor</code>) and another one when a Hugging Face model is used (<code>HFPreprocessor</code>).</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor","title":"WidePreprocessor","text":"<pre><code>WidePreprocessor(wide_cols, crossed_cols=None)\n</code></pre> <p>             Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>WidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> <ul> <li> <code>wide_cols</code>             (<code>List[str]</code>)         \u2013          <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> </li> <li> <code>crossed_cols</code>             (<code>Optional[List[Tuple[str, str]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_crossed_cols</code>             (<code>List</code>)         \u2013          <p>List with the names of all columns that will be label encoded</p> </li> <li> <code>encoding_dict</code>             (<code>Dict</code>)         \u2013          <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> </li> <li> <code>inverse_encoding_dict</code>             (<code>Dict</code>)         \u2013          <p>the inverse encoding dictionary</p> </li> <li> <code>wide_dim</code>             (<code>int</code>)         \u2013          <p>Dimension of the wide model (i.e. dim of the linear layer)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n&gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n&gt;&gt;&gt; X_wide\narray([[1, 4],\n       [2, 5],\n       [3, 6]])\n&gt;&gt;&gt; wide_preprocessor.encoding_dict\n{'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n&gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n  color color_size\n0     r        r-s\n1     b        b-n\n2     g        g-l\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def __init__(\n    self, wide_cols: List[str], crossed_cols: Optional[List[Tuple[str, str]]] = None\n):\n    super(WidePreprocessor, self).__init__()\n\n    self.wide_cols = wide_cols\n    self.crossed_cols = crossed_cols\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>WidePreprocessor</code>         \u2013          <p><code>WidePreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    WidePreprocessor\n        `WidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(df)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n    glob_feature_list = self._make_global_feature_list(\n        df_wide[self.wide_crossed_cols]\n    )\n    # leave 0 for padding/\"unseen\" categories\n    self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n    self.wide_dim = len(self.encoding_dict)\n    self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n    self.inverse_encoding_dict[0] = \"unseen\"\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    r\"\"\"\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, attributes=[\"encoding_dict\"])\n    df_wide = self._prepare_wide(df)\n    encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n    for col_i, col in enumerate(self.wide_crossed_cols):\n        encoded[:, col_i] = df_wide[col].apply(\n            lambda x: (\n                self.encoding_dict[col + \"_\" + str(x)]\n                if col + \"_\" + str(x) in self.encoding_dict\n                else 0\n            )\n        )\n    return encoded.astype(\"int64\")\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> <ul> <li> <code>encoded</code>             (<code>ndarray</code>)         \u2013          <p>numpy array with the encoded values that are the output from the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>         \u2013          <p>Pandas dataframe with the original values</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        numpy array with the encoded values that are the output from the\n        `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n    if pd.__version__ &gt;= \"2.1.0\":\n        decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n    else:\n        decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n    for col in decoded.columns:\n        rm_str = \"\".join([col, \"_\"])\n        decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor","title":"TabPreprocessor","text":"<pre><code>TabPreprocessor(\n    cat_embed_cols=None,\n    continuous_cols=None,\n    quantization_setup=None,\n    cols_to_scale=None,\n    auto_embed_dim=True,\n    embedding_rule=\"fastai_new\",\n    default_embed_dim=16,\n    with_attention=False,\n    with_cls_token=False,\n    shared_embed=False,\n    verbose=1,\n    *,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> <ul> <li> <code>cat_embed_cols</code>             (<code>Optional[Union[List[str], List[Tuple[str, int]]]]</code>, default:                 <code>None</code> )         \u2013          <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the continuous cols</p> </li> <li> <code>quantization_setup</code>             (<code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code>, default:                 <code>None</code> )         \u2013          <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used.</p> </li> <li> <code>cols_to_scale</code>             (<code>Optional[Union[List[str], str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> </li> <li> <code>auto_embed_dim</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> </li> <li> <code>embedding_rule</code>             (<code>Literal[google, fastai_old, fastai_new]</code>, default:                 <code>'fastai_new'</code> )         \u2013          <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6 \\times n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> </li> <li> <code>default_embed_dim</code>             (<code>int</code>, default:                 <code>16</code> )         \u2013          <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> </li> <li> <code>with_attention</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> </li> <li> <code>with_cls_token</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical and/or continuous embeddings will be concatenated before being passed to the final MLP (if present).</p> </li> <li> <code>shared_embed</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          </li> <li> <code>scale</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> </li> <li> <code>already_standard</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>         \u2013          <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embed_dim</code>             (<code>Dict</code>)         \u2013          <p>Dictionary where keys are the embed cols and values are the embedding dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute is not generated during the <code>fit</code> process</p> </li> <li> <code>label_encoder</code>             (<code>LabelEncoder</code>)         \u2013          <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p> </li> <li> <code>cat_embed_input</code>             (<code>List</code>)         \u2013          <p>List of Tuples with the column name, number of individual values for that column and, If <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> </li> <li> <code>standardize_cols</code>             (<code>List</code>)         \u2013          <p>List of the columns that will be standarized</p> </li> <li> <code>scaler</code>             (<code>StandardScaler</code>)         \u2013          <p>an instance of <code>sklearn.preprocessing.StandardScaler</code></p> </li> <li> <code>column_idx</code>             (<code>Dict</code>)         \u2013          <p>Dictionary where keys are column names and values are column indexes. This is neccesary to slice tensors</p> </li> <li> <code>quantizer</code>             (<code>Quantizer</code>)         \u2013          <p>an instance of <code>Quantizer</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n&gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n&gt;&gt;&gt; cont_cols = ['age']\n&gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n&gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n[('color', 5), ('size', 5)]\n&gt;&gt;&gt; deep_preprocessor.column_idx\n{'color': 0, 'size': 1, 'age': 2}\n&gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n&gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n&gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n&gt;&gt;&gt; # or...\n&gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n&gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n&gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\n@alias(\"cat_embed_cols\", [\"embed_cols\"])\n@alias(\"scale\", [\"scale_cont_cols\"])\n@alias(\"quantization_setup\", [\"cols_and_bins\"])\ndef __init__(\n    self,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    quantization_setup: Optional[\n        Union[int, Dict[str, Union[int, List[float]]]]\n    ] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    auto_embed_dim: bool = True,\n    embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n    default_embed_dim: int = 16,\n    with_attention: bool = False,\n    with_cls_token: bool = False,\n    shared_embed: bool = False,\n    verbose: int = 1,\n    *,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    super(TabPreprocessor, self).__init__()\n\n    self.continuous_cols = continuous_cols\n    self.quantization_setup = quantization_setup\n    self.cols_to_scale = cols_to_scale\n    self.scale = scale\n    self.already_standard = already_standard\n    self.auto_embed_dim = auto_embed_dim\n    self.embedding_rule = embedding_rule\n    self.default_embed_dim = default_embed_dim\n    self.with_attention = with_attention\n    self.with_cls_token = with_cls_token\n    self.shared_embed = shared_embed\n    self.verbose = verbose\n\n    self.quant_args = {\n        k: v for k, v in kwargs.items() if k in pd.cut.__code__.co_varnames\n    }\n    self.scale_args = {\n        k: v for k, v in kwargs.items() if k in StandardScaler().get_params()\n    }\n\n    self._check_inputs(cat_embed_cols)\n\n    if with_cls_token:\n        self.cat_embed_cols = (\n            [\"cls_token\"] + cat_embed_cols  # type: ignore[operator]\n            if cat_embed_cols is not None\n            else [\"cls_token\"]\n        )\n    else:\n        self.cat_embed_cols = cat_embed_cols  # type: ignore[assignment]\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>TabPreprocessor</code>         \u2013          <p><code>TabPreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n    \"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TabPreprocessor\n        `TabPreprocessor` fitted object\n    \"\"\"\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    self.column_idx: Dict[str, int] = {}\n\n    # Categorical embeddings logic\n    if self.cat_embed_cols is not None or self.quantization_setup is not None:\n        self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n            []\n        )\n\n    if self.cat_embed_cols is not None:\n        df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n        self.label_encoder = LabelEncoder(\n            columns_to_encode=df_cat.columns.tolist(),\n            shared_embed=self.shared_embed,\n            with_attention=self.with_attention,\n        )\n        self.label_encoder.fit(df_cat)\n\n        for k, v in self.label_encoder.encoding_dict.items():\n            if self.with_attention:\n                self.cat_embed_input.append((k, len(v)))\n            else:\n                self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n        self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n    # Continuous columns logic\n    if self.continuous_cols is not None:\n        df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n        # Standardization logic\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args).fit(\n                df_cont[self.standardize_cols].values\n            )\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # we do not run 'Quantizer.fit' here since in the wild case\n            # someone wants standardization and quantization for the same\n            # columns, the Quantizer will run on the scaled data\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n            if self.with_attention:\n                for col, n_cat, _ in cont_embed_dim:\n                    self.cat_embed_input.append((col, n_cat))\n            else:\n                self.cat_embed_input.extend(cont_embed_dim)\n\n        self.column_idx.update(\n            {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the processed <code>dataframe</code> as a np.ndarray</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n    \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, condition=self.is_fitted)\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    if self.cat_embed_cols is not None:\n        df_cat = df_adj[self.cat_cols]\n        df_cat = self.label_encoder.transform(df_cat)\n    if self.continuous_cols is not None:\n        df_cont = df_adj[self.continuous_cols]\n        # Standardization logic\n        if self.standardize_cols:\n            df_cont[self.standardize_cols] = self.scaler.transform(\n                df_cont[self.standardize_cols].values\n            )\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # Adjustment so I don't have to override the method\n            # in 'ChunkTabPreprocessor'\n            if self.quantizer.is_fitted:\n                df_cont = self.quantizer.transform(df_cont)\n            else:\n                df_cont = self.quantizer.fit_transform(df_cont)\n    try:\n        df_deep = pd.concat([df_cat, df_cont], axis=1)\n    except NameError:\n        try:\n            df_deep = df_cat.copy()\n        except NameError:\n            df_deep = df_cont.copy()\n\n    return df_deep.values\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> <ul> <li> <code>encoded</code>             (<code>ndarray</code>)         \u2013          <p>array with the output of the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>         \u2013          <p>Pandas dataframe with the original values</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n    # embeddings back to original category\n    if self.cat_embed_cols is not None:\n        decoded = self.label_encoder.inverse_transform(decoded)\n    if self.continuous_cols is not None:\n        # quantized cols to the mid point\n        if self.cols_and_bins is not None:\n            if self.verbose:\n                print(\n                    \"Note that quantized cols will be turned into the mid point of \"\n                    \"the corresponding bin\"\n                )\n            for k, v in self.quantizer.inversed_bins.items():\n                decoded[k] = decoded[k].map(v)\n        # continuous_cols back to non-standarised\n        try:\n            decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                decoded[self.standardize_cols]\n            )\n        except Exception:  # KeyError:\n            pass\n\n    if \"cls_token\" in decoded.columns:\n        decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer","title":"Quantizer","text":"<pre><code>Quantizer(quantization_setup, **kwargs)\n</code></pre> <p>Helper class to perform the quantization of continuous columns. It is included in this docs for completion, since depending on the value of the parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that class might have an attribute of type <code>Quantizer</code>. However, this class is designed to always run internally within the <code>TabPreprocessor</code> class.</p> <p>Parameters:</p> <ul> <li> <code>quantization_setup</code>             (<code>Dict[str, Union[int, List[float]]]</code>)         \u2013          <p>Dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges.</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    quantization_setup: Dict[str, Union[int, List[float]]],\n    **kwargs,\n):\n    self.quantization_setup = quantization_setup\n    self.quant_args = kwargs\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor","title":"TextPreprocessor","text":"<pre><code>TextPreprocessor(\n    text_col,\n    max_vocab=30000,\n    min_freq=5,\n    maxlen=80,\n    pad_first=True,\n    pad_idx=1,\n    already_processed=False,\n    word_vectors_path=None,\n    n_cpus=None,\n    verbose=1,\n)\n</code></pre> <p>             Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> <ul> <li> <code>text_col</code>             (<code>str</code>)         \u2013          <p>column in the input dataframe containing the texts</p> </li> <li> <code>max_vocab</code>             (<code>int</code>, default:                 <code>30000</code> )         \u2013          <p>Maximum number of tokens in the vocabulary</p> </li> <li> <code>min_freq</code>             (<code>int</code>, default:                 <code>5</code> )         \u2013          <p>Minimum frequency for a token to be part of the vocabulary</p> </li> <li> <code>maxlen</code>             (<code>int</code>, default:                 <code>80</code> )         \u2013          <p>Maximum length of the tokenized sequences</p> </li> <li> <code>pad_first</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> <li> <code>already_processed</code>             (<code>Optional[bool]</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the sequence of elements is already processed or prepared. If this is the case, this Preprocessor will simply tokenize and pad the sequence. </p> <pre><code>Param aliases: `not_text`. &lt;br/&gt;\n</code></pre> <p>This parameter is thought for those cases where the input sequences are already fully processed or are directly not text (e.g. IDs)</p> </li> <li> <code>word_vectors_path</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Path to the pretrained word vectors</p> </li> <li> <code>n_cpus</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>number of CPUs to used during the tokenization process</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vocab</code>             (<code>Vocab</code>)         \u2013          <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p> </li> <li> <code>embedding_matrix</code>             (<code>ndarray</code>)         \u2013          <p>Array with the pretrained embeddings</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n&gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\nThe vocabulary contains 24 tokens\narray([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n&gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n&gt;&gt;&gt; text_preprocessor.transform(df_te)\narray([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>@alias(\"already_processed\", [\"not_text\"])\ndef __init__(\n    self,\n    text_col: str,\n    max_vocab: int = 30000,\n    min_freq: int = 5,\n    maxlen: int = 80,\n    pad_first: bool = True,\n    pad_idx: int = 1,\n    already_processed: Optional[bool] = False,\n    word_vectors_path: Optional[str] = None,\n    n_cpus: Optional[int] = None,\n    verbose: int = 1,\n):\n    super(TextPreprocessor, self).__init__()\n\n    self.text_col = text_col\n    self.max_vocab = max_vocab\n    self.min_freq = min_freq\n    self.maxlen = maxlen\n    self.pad_first = pad_first\n    self.pad_idx = pad_idx\n    self.already_processed = already_processed\n    self.word_vectors_path = word_vectors_path\n    self.verbose = verbose\n    self.n_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Builds the vocabulary</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>TextPreprocessor</code>         \u2013          <p><code>TextPreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n    \"\"\"Builds the vocabulary\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TextPreprocessor\n        `TextPreprocessor` fitted object\n    \"\"\"\n    texts = self._read_texts(df)\n\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n    self.vocab: TVocab = Vocab(\n        max_vocab=self.max_vocab,\n        min_freq=self.min_freq,\n        pad_idx=self.pad_idx,\n    ).fit(\n        tokens,\n    )\n\n    if self.verbose:\n        print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n    if self.word_vectors_path is not None:\n        self.embedding_matrix = build_embeddings_matrix(\n            self.vocab, self.word_vectors_path, self.min_freq\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the padded, 'numericalised' sequences</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>Padded, 'numericalised' sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequences\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    texts = self._read_texts(df)\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Returns the padded, 'numericalised' sequence</p> <p>Parameters:</p> <ul> <li> <code>text</code>             (<code>str</code>)         \u2013          <p>text to be tokenized and padded</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>Padded, 'numericalised' sequence</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequence\n\n    Parameters\n    ----------\n    text: str\n        text to be tokenized and padded\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequence\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    tokens = get_texts([text], self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>Padded, 'numericalised' sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(padded_seq)\n</code></pre> <p>Returns the original text plus the added 'special' tokens</p> <p>Parameters:</p> <ul> <li> <code>padded_seq</code>             (<code>ndarray</code>)         \u2013          <p>array with the output of the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>         \u2013          <p>Pandas dataframe with the original text plus the added 'special' tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n    \"\"\"Returns the original text plus the added 'special' tokens\n\n    Parameters\n    ----------\n    padded_seq: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original text plus the added 'special' tokens\n    \"\"\"\n    texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n    return pd.DataFrame({self.text_col: texts})\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor","title":"HFPreprocessor","text":"<pre><code>HFPreprocessor(\n    model_name,\n    *,\n    use_fast_tokenizer=False,\n    text_col=None,\n    root_dir=None,\n    num_workers=None,\n    preprocessing_rules=None,\n    tokenizer_params=None,\n    encode_params=None,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>BasePreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this class is designed to be as flexible as possible. Therefore, it is coded so that the user can use it as one would use any HuggingFace tokenizers, or following the API call 'protocol' of the rest of the library.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>             (<code>str</code>)         \u2013          <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>use_fast_tokenizer</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Whether to use the fast tokenizer from HuggingFace or not</p> </li> <li> <code>text_col</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>The column in the input dataframe containing the text data. If this tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this argument is mandatory. If the tokenizer is used via the <code>encode</code> method, this argument is not needed since the input text is passed directly to the <code>encode</code> method.</p> </li> <li> <code>num_workers</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> </li> <li> <code>preprocessing_rules</code>             (<code>Optional[List[Callable[[str], str]]]</code>, default:                 <code>None</code> )         \u2013          <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> </li> <li> <code>tokenizer_params</code>             (<code>Optional[Dict[str, Any]]</code>, default:                 <code>None</code> )         \u2013          <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code> can also be passed via the <code>**kwargs</code> argument</p> </li> <li> <code>encode_params</code>             (<code>Optional[Dict[str, Any]]</code>, default:                 <code>None</code> )         \u2013          <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code> methods are used, the <code>encode_params</code> dict parameter is mandatory. If the <code>encode</code> method is used, this parameter is not needed since the input text is passed directly to the <code>encode</code> method.</p> </li> <li> <code>**kwargs</code>         \u2013          <p>Additional kwargs to be passed to the model, in particular to the <code>PreTrainedTokenizer</code> class.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>is_fitted</code>             (<code>bool</code>)         \u2013          <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n&gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n&gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n&gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n&gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n&gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    model_name: str,\n    *,\n    use_fast_tokenizer: bool = False,\n    text_col: Optional[str] = None,\n    root_dir: Optional[str] = None,\n    num_workers: Optional[int] = None,\n    preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n    tokenizer_params: Optional[Dict[str, Any]] = None,\n    encode_params: Optional[Dict[str, Any]] = None,\n    **kwargs,\n):\n    self.model_name = model_name\n    self.use_fast_tokenizer = use_fast_tokenizer\n    self.text_col = text_col\n    self.root_dir = root_dir\n    self.num_workers = num_workers\n    self.preprocessing_rules = preprocessing_rules\n    self.tokenizer_params = tokenizer_params if tokenizer_params is not None else {}\n    self.encode_params = encode_params if encode_params is not None else {}\n\n    self._multiprocessing = (\n        num_workers is not None and num_workers &gt; 1 and not use_fast_tokenizer\n    )\n\n    if kwargs:\n        self.tokenizer_params.update(kwargs)\n\n    self.tokenizer = get_tokenizer(\n        model_name=self.model_name,\n        use_fast_tokenizer=self.use_fast_tokenizer,\n        **self.tokenizer_params,\n    )\n\n    # A HuggingFace tokenizer is already trained, since we need this\n    # attribute elsewhere in the library, we simply set it to True\n    self.is_fitted = True\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode","title":"encode","text":"<pre><code>encode(texts, **kwargs)\n</code></pre> <p>Encodes a list of texts. The method is a wrapper around the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p> <p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p> <p>Parameters:</p> <ul> <li> <code>texts</code>             (<code>List[str]</code>)         \u2013          <p>List of texts to be encoded</p> </li> <li> <code>**kwargs</code>         \u2013          <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer. If the 'encode_params' dict was passed when instantiating the class, that dictionaly will be updated with the kwargs passed here.</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>         \u2013          <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a list of texts. The method is a wrapper around the\n    `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n    if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n    Parameters\n    ----------\n    texts: List[str]\n        List of texts to be encoded\n    **kwargs\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n        when instantiating the class, that dictionaly will be updated with\n        the kwargs passed here.\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if kwargs:\n        self.encode_params.update(kwargs)\n\n    if self.preprocessing_rules:\n        if self._multiprocessing:\n            texts = self._process_text_parallel(texts)\n        else:\n            texts = [self._preprocess_text(text) for text in texts]\n\n    if self._multiprocessing:\n        input_ids = self._encode_paralell(texts, **self.encode_params)\n    else:\n        encoded_texts = self.tokenizer.batch_encode_plus(\n            texts,\n            **self.encode_params,\n        )\n        input_ids = encoded_texts.get(\"input_ids\")\n\n    self.is_fitted = True\n\n    try:\n        output = np.array(input_ids)\n    except ValueError:\n        warnings.warn(\n            \"Padding and Truncating parameters were not passed and all input arrays \"\n            \"do not have the same shape. Padding to the longest sequence. \"\n            \"Padding will be done with the index of the pad token for the model\",\n            UserWarning,\n        )\n        max_len = max([len(ids) for ids in input_ids])\n        output = np.array(\n            [\n                np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                for ids in input_ids\n            ]\n        )\n\n    return output\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode","title":"decode","text":"<pre><code>decode(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method is a wrapper around the <code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the HuggingFace's tokenizer.</p> <p>Parameters:</p> <ul> <li> <code>input_ids</code>             (<code>NDArray[int64]</code>)         \u2013          <p>The input_ids to be decoded</p> </li> <li> <code>skip_special_tokens</code>             (<code>bool</code>)         \u2013          <p>Whether to skip the special tokens or not</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>         \u2013          <p>The decoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def decode(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method is a wrapper around the\n    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n    HuggingFace's tokenizer.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    texts = [\n        self.tokenizer.convert_tokens_to_string(\n            self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n        )\n        for i in range(input_ids.shape[0])\n    ]\n    return texts\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>This method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's tokenizers and models are already trained. Therefore, the 'fit' method here does nothing other than checking that the 'text_col' parameter is not <code>None</code>.</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n    \"\"\"\n    This method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular. HuggingFace's\n    tokenizers and models are already trained. Therefore, the 'fit' method\n    here does nothing other than checking that the 'text_col' parameter is\n    not `None`.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe. This method simply calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method, this method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular.</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>         \u2013          <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe. This method simply\n    calls the `encode` method under the hood. Similar to the `fit` method,\n    this method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n\n    texts = self._read_texts(df, self.root_dir)\n\n    return self.encode(texts)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Encodes a single text sample.</p> <p>Parameters:</p> <ul> <li> <code>text</code>             (<code>str</code>)         \u2013          <p>The text sample to be encoded</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>         \u2013          <p>The encoded text</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a single text sample.\n\n    Parameters\n    ----------\n    text: str\n        The text sample to be encoded\n\n    Returns\n    -------\n    np.array\n        The encoded text\n    \"\"\"\n\n    if not self.is_fitted:\n        raise ValueError(\n            \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n        )\n    return self.encode([text])[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe.</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>         \u2013          <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method under the hood.</p> <p>Parameters:</p> <ul> <li> <code>input_ids</code>             (<code>NDArray[int64]</code>)         \u2013          <p>The input_ids to be decoded</p> </li> <li> <code>skip_special_tokens</code>             (<code>bool</code>)         \u2013          <p>Whether to skip the special tokens or not</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>         \u2013          <p>The decoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def inverse_transform(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method simply calls the `decode` method\n    under the hood.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    return self.decode(input_ids, skip_special_tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor","title":"ImagePreprocessor","text":"<pre><code>ImagePreprocessor(\n    img_col, img_path, width=224, height=224, verbose=1\n)\n</code></pre> <p>             Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p> <p>The Preprocessing consists simply on resizing according to their aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>img_col</code>             (<code>str</code>)         \u2013          <p>name of the column with the images filenames</p> </li> <li> <code>img_path</code>             (<code>str</code>)         \u2013          <p>path to the dicrectory where the images are stored</p> </li> <li> <code>width</code>             (<code>int</code>, default:                 <code>224</code> )         \u2013          <p>width of the resulting processed image.</p> </li> <li> <code>height</code>             (<code>int</code>, default:                 <code>224</code> )         \u2013          <p>width of the resulting processed image.</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>aap</code>             (<code>AspectAwarePreprocessor</code>)         \u2013          <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p> </li> <li> <code>spp</code>             (<code>SimplePreprocessor</code>)         \u2013          <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p> </li> <li> <code>normalise_metrics</code>             (<code>Dict</code>)         \u2013          <p>Dict containing the normalisation metrics of the image dataset, i.e. mean and std for the R, G and B channels</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n&gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n&gt;&gt;&gt;\n&gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n&gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n&gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n&gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n&gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n</code></pre> <p> NOTE: Normalising metrics will only be computed when the <code>fit_transform</code> method is run. Running <code>transform</code> only will not change the computed metrics and running <code>fit</code> only simply instantiates the resizing functions.</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    img_col: str,\n    img_path: str,\n    width: int = 224,\n    height: int = 224,\n    verbose: int = 1,\n):\n    super(ImagePreprocessor, self).__init__()\n\n    self.img_col = img_col\n    self.img_path = img_path\n    self.width = width\n    self.height = height\n    self.verbose = verbose\n\n    self.aap = AspectAwarePreprocessor(self.width, self.height)\n    self.spp = SimplePreprocessor(self.width, self.height)\n\n    self.compute_normalising_computed = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Resizes the images to the input height and width.</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe with the <code>img_col</code></p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>Resized images to the input height and width</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Resizes the images to the input height and width.\n\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe with the `img_col`\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    image_list = df[self.img_col].tolist()\n    if self.verbose:\n        print(\"Reading Images from {}\".format(self.img_path))\n    imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n    # finding images with different height and width\n    aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n    aspect_r = [a[0] / a[1] for a in aspect]\n    diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n    if self.verbose:\n        print(\"Resizing\")\n    resized_imgs = []\n    for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n        if i in diff_idx:\n            resized_imgs.append(self.aap.preprocess(img))\n        else:\n            # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n            resized_imgs.append(self.spp.preprocess(img))\n\n    if not self.compute_normalising_computed:\n        if self.verbose:\n            print(\"Computing normalisation metrics\")\n        # mean and std deviation will only be computed when the fit method\n        # is called\n        mean_R, mean_G, mean_B = [], [], []\n        std_R, std_G, std_B = [], [], []\n        for rsz_img in resized_imgs:\n            (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                rsz_img\n            )\n            mean_R.append(mean_r)\n            mean_G.append(mean_g)\n            mean_B.append(mean_b)\n            std_R.append(std_r)\n            std_G.append(std_g)\n            std_B.append(std_b)\n        self.normalise_metrics = dict(\n            mean={\n                \"R\": np.mean(mean_R) / 255.0,\n                \"G\": np.mean(mean_G) / 255.0,\n                \"B\": np.mean(mean_B) / 255.0,\n            },\n            std={\n                \"R\": np.mean(std_R) / 255.0,\n                \"G\": np.mean(std_G) / 255.0,\n                \"B\": np.mean(std_B) / 255.0,\n            },\n        )\n        self.compute_normalising_computed = True\n    return np.asarray(resized_imgs)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>Resized images to the input height and width</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#chunked-versions","title":"Chunked versions","text":"<p>Chunked versions of the preprocessors are also available. These are useful when the data is too big to fit in memory. See also the <code>load_from_folder</code> module in the library and the corresponding section here in the documentation.</p> <p>Note that there is not a <code>ChunkImagePreprocessor</code>. This is because the processing of the images will occur inside the <code>ImageFromFolder</code> class in the <code>load_from_folder</code> module.</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor","title":"ChunkWidePreprocessor","text":"<pre><code>ChunkWidePreprocessor(\n    wide_cols, n_chunks, crossed_cols=None\n)\n</code></pre> <p>             Bases: <code>WidePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>ChunkWidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> <ul> <li> <code>wide_cols</code>             (<code>List[str]</code>)         \u2013          <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> </li> <li> <code>crossed_cols</code>             (<code>Optional[List[Tuple[str, str]]]</code>, default:                 <code>None</code> )         \u2013          <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_crossed_cols</code>             (<code>List</code>)         \u2013          <p>List with the names of all columns that will be label encoded</p> </li> <li> <code>encoding_dict</code>             (<code>Dict</code>)         \u2013          <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> </li> <li> <code>inverse_encoding_dict</code>             (<code>Dict</code>)         \u2013          <p>the inverse encoding dictionary</p> </li> <li> <code>wide_dim</code>             (<code>int</code>)         \u2013          <p>Dimension of the wide model (i.e. dim of the linear layer)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n&gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n... n_chunks=1)\n&gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    wide_cols: List[str],\n    n_chunks: int,\n    crossed_cols: Optional[List[Tuple[str, str]]] = None,\n):\n    super(ChunkWidePreprocessor, self).__init__(wide_cols, crossed_cols)\n\n    self.n_chunks = n_chunks\n\n    self.chunk_counter = 0\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(chunk)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>chunk</code>             (<code>DataFrame</code>)         \u2013          <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ChunkWidePreprocessor</code>         \u2013          <p><code>ChunkWidePreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    chunk: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    ChunkWidePreprocessor\n        `ChunkWidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(chunk)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n\n    if self.chunk_counter == 0:\n        self.glob_feature_set = set(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n    else:\n        self.glob_feature_set.update(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n\n    self.chunk_counter += 1\n\n    if self.chunk_counter == self.n_chunks:\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Runs <code>partial_fit</code>. This is just to override the fit method in the base class. This class is not designed or thought to run fit</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    \"\"\"\n    Runs `partial_fit`. This is just to override the fit method in the base\n    class. This class is not designed or thought to run fit\n    \"\"\"\n    return self.partial_fit(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor","title":"ChunkTabPreprocessor","text":"<pre><code>ChunkTabPreprocessor(\n    n_chunks,\n    cat_embed_cols=None,\n    continuous_cols=None,\n    cols_and_bins=None,\n    cols_to_scale=None,\n    default_embed_dim=16,\n    with_attention=False,\n    with_cls_token=False,\n    shared_embed=False,\n    verbose=1,\n    *,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>TabPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> <ul> <li> <code>n_chunks</code>             (<code>int</code>)         \u2013          <p>Number of chunks that the tabular dataset is divided by.</p> </li> <li> <code>cat_embed_cols</code>             (<code>Optional[Union[List[str], List[Tuple[str, int]]]]</code>, default:                 <code>None</code> )         \u2013          <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> </li> <li> <code>continuous_cols</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the name of the continuous cols</p> </li> <li> <code>cols_and_bins</code>             (<code>Optional[Dict[str, List[float]]]</code>, default:                 <code>None</code> )         \u2013          <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column names to quantize and the values are a list of scalars indicating the bin edges.</p> </li> <li> <code>cols_to_scale</code>             (<code>Optional[Union[List[str], str]]</code>, default:                 <code>None</code> )         \u2013          <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code></p> </li> <li> <code>default_embed_dim</code>             (<code>int</code>, default:                 <code>16</code> )         \u2013          <p>Dimension for the embeddings if the embed_dim is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> </li> <li> <code>with_attention</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> </li> <li> <code>with_cls_token</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical (and continuous embeddings if present) will be concatenated before being passed to the final MLP (if present).</p> </li> <li> <code>shared_embed</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          </li> <li> <code>scale</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> </li> <li> <code>already_standard</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>         \u2013          <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embed_dim</code>             (<code>Dict</code>)         \u2013          <p>Dictionary where keys are the embed cols and values are the embedding dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute is not generated during the <code>fit</code> process</p> </li> <li> <code>label_encoder</code>             (<code>LabelEncoder</code>)         \u2013          <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p> </li> <li> <code>cat_embed_input</code>             (<code>List</code>)         \u2013          <p>List of Tuples with the column name, number of individual values for that column and, If <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> </li> <li> <code>standardize_cols</code>             (<code>List</code>)         \u2013          <p>List of the columns that will be standarized</p> </li> <li> <code>scaler</code>             (<code>StandardScaler</code>)         \u2013          <p>an instance of <code>sklearn.preprocessing.StandardScaler</code> if 'cols_to_scale' is not None or 'scale' is 'True'</p> </li> <li> <code>column_idx</code>             (<code>Dict</code>)         \u2013          <p>Dictionary where keys are column names and values are column indexes. This is neccesary to slice tensors</p> </li> <li> <code>quantizer</code>             (<code>Quantizer</code>)         \u2013          <p>an instance of <code>Quantizer</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n&gt;&gt;&gt; np.random.seed(42)\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n... 'cont_col': np.random.uniform(1, 100, size=8)})\n&gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n&gt;&gt;&gt; cont_cols = ['cont_col']\n&gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n... )\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n&gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n[('cat_col', 4)]\n&gt;&gt;&gt; tab_preprocessor.column_idx\n{'cat_col': 0, 'cont_col': 1}\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\n@alias(\"cat_embed_cols\", [\"embed_cols\"])\n@alias(\"scale\", [\"scale_cont_cols\"])\n@alias(\"cols_and_bins\", [\"quantization_setup\"])\ndef __init__(\n    self,\n    n_chunks: int,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cols_and_bins: Optional[Dict[str, List[float]]] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    default_embed_dim: int = 16,\n    with_attention: bool = False,\n    with_cls_token: bool = False,\n    shared_embed: bool = False,\n    verbose: int = 1,\n    *,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    super(ChunkTabPreprocessor, self).__init__(\n        cat_embed_cols=cat_embed_cols,\n        continuous_cols=continuous_cols,\n        quantization_setup=None,\n        cols_to_scale=cols_to_scale,\n        auto_embed_dim=False,\n        embedding_rule=\"google\",  # does not matter, irrelevant\n        default_embed_dim=default_embed_dim,\n        with_attention=with_attention,\n        with_cls_token=with_cls_token,\n        shared_embed=shared_embed,\n        verbose=verbose,\n        scale=scale,\n        already_standard=already_standard,\n        **kwargs,\n    )\n\n    self.n_chunks = n_chunks\n    self.chunk_counter = 0\n\n    self.cols_and_bins = cols_and_bins  # type: ignore[assignment]\n    if self.cols_and_bins is not None:\n        self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n    self.embed_prepared = False\n    self.continuous_prepared = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor","title":"ChunkTextPreprocessor","text":"<pre><code>ChunkTextPreprocessor(\n    text_col,\n    n_chunks,\n    root_dir=None,\n    max_vocab=30000,\n    min_freq=5,\n    maxlen=80,\n    pad_first=True,\n    pad_idx=1,\n    already_processed=False,\n    word_vectors_path=None,\n    n_cpus=None,\n    verbose=1,\n)\n</code></pre> <p>             Bases: <code>TextPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> <ul> <li> <code>text_col</code>             (<code>str</code>)         \u2013          <p>column in the input dataframe containing either the texts or the filenames where the text documents are stored</p> </li> <li> <code>n_chunks</code>             (<code>int</code>)         \u2013          <p>Number of chunks that the text dataset is divided by.</p> </li> <li> <code>root_dir</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>If 'text_col' contains the filenames with the text documents, this is the path to the directory where those documents are stored.</p> </li> <li> <code>max_vocab</code>             (<code>int</code>, default:                 <code>30000</code> )         \u2013          <p>Maximum number of tokens in the vocabulary</p> </li> <li> <code>min_freq</code>             (<code>int</code>, default:                 <code>5</code> )         \u2013          <p>Minimum frequency for a token to be part of the vocabulary</p> </li> <li> <code>maxlen</code>             (<code>int</code>, default:                 <code>80</code> )         \u2013          <p>Maximum length of the tokenized sequences</p> </li> <li> <code>pad_first</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> <li> <code>word_vectors_path</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Path to the pretrained word vectors</p> </li> <li> <code>n_cpus</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>number of CPUs to used during the tokenization process</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vocab</code>             (<code>Vocab</code>)         \u2013          <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p> </li> <li> <code>embedding_matrix</code>             (<code>ndarray</code>)         \u2013          <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n&gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    text_col: str,\n    n_chunks: int,\n    root_dir: Optional[str] = None,\n    max_vocab: int = 30000,\n    min_freq: int = 5,\n    maxlen: int = 80,\n    pad_first: bool = True,\n    pad_idx: int = 1,\n    already_processed: Optional[bool] = False,\n    word_vectors_path: Optional[str] = None,\n    n_cpus: Optional[int] = None,\n    verbose: int = 1,\n):\n    super(ChunkTextPreprocessor, self).__init__(\n        text_col=text_col,\n        max_vocab=max_vocab,\n        min_freq=min_freq,\n        maxlen=maxlen,\n        pad_first=pad_first,\n        pad_idx=pad_idx,\n        already_processed=already_processed,\n        word_vectors_path=word_vectors_path,\n        n_cpus=n_cpus,\n        verbose=verbose,\n    )\n\n    self.n_chunks = n_chunks\n    self.root_dir = root_dir\n\n    self.chunk_counter = 0\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor","title":"ChunkHFPreprocessor","text":"<pre><code>ChunkHFPreprocessor(\n    model_name,\n    *,\n    text_col,\n    root_dir=None,\n    use_fast_tokenizer=True,\n    num_workers=None,\n    preprocessing_rules=None,\n    tokenizer_params=None,\n    encode_params=None\n)\n</code></pre> <p>             Bases: <code>HFPreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Hugginface Tokenizer's are already 'trained'. Therefore, unlike the <code>ChunkTextPreprocessor</code> this is mostly identical to the <code>HFPreprocessor</code> with the only difference that the class needs a 'text_col' parameter to be passed. Also the parameter <code>encode_params</code> is not really optional when using this class. It must be passed containing at least the 'max_length' encoding parameter. This is because we need to ensure that  all sequences have the same length when encoding in chunks.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>             (<code>str</code>)         \u2013          <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>text_col</code>             (<code>str</code>)         \u2013          <p>The column in the input dataframe containing the text data. When using the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p> </li> <li> <code>root_dir</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>The root directory where the text files are located. This is only needed if the text data is stored in text files. If the text data is stored in a column in the input dataframe, this parameter is not needed.</p> </li> <li> <code>use_fast_tokenizer</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Whether to use the fast tokenizer from HuggingFace or not</p> </li> <li> <code>num_workers</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> </li> <li> <code>preprocessing_rules</code>             (<code>Optional[List[Callable[[str], str]]]</code>, default:                 <code>None</code> )         \u2013          <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> </li> <li> <code>tokenizer_params</code>             (<code>Optional[Dict[str, Any]]</code>, default:                 <code>None</code> )         \u2013          <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>.</p> </li> <li> <code>encode_params</code>             (<code>Optional[Dict[str, Any]]</code>, default:                 <code>None</code> )         \u2013          <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the <code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It must be passed containing at least the 'max_length' encoding parameter</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>is_fitted</code>             (<code>bool</code>)         \u2013          <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    model_name: str,\n    *,\n    text_col: str,\n    root_dir: Optional[str] = None,\n    use_fast_tokenizer: bool = True,\n    num_workers: Optional[int] = None,\n    preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n    tokenizer_params: Optional[Dict[str, Any]] = None,\n    encode_params: Optional[Dict[str, Any]] = None,\n):\n    super().__init__(\n        model_name=model_name,\n        use_fast_tokenizer=use_fast_tokenizer,\n        text_col=text_col,\n        num_workers=num_workers,\n        preprocessing_rules=preprocessing_rules,\n        tokenizer_params=tokenizer_params,\n        encode_params=encode_params,\n    )\n\n    self.root_dir = root_dir\n\n    # when using in chunks encode_params is not really optional. I will\n    # review types in due time\n    if self.encode_params is None:\n        raise ValueError(\n            \"The 'encode_params' dict must be passed to the ChunkHFTokenizer \"\n            \"containing at least the 'max_length' encoding parameter\"\n        )\n\n    if \"padding\" not in self.encode_params or not self.encode_params[\"padding\"]:\n        self.encode_params[\"padding\"] = True\n\n    if (\n        \"truncation\" not in self.encode_params\n        or not self.encode_params[\"truncation\"]\n    ):\n        self.encode_params[\"truncation\"] = True\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html","title":"Self Supervised Pre-training for tabular data","text":"<p>In this library we have implemented two methods or routines that allow the user to use self-suerpvised pre-training for all tabular models in the library with the exception of the <code>TabPerceiver</code> (this is a particular model and self-supervised pre-training requires some adjustments that will be implemented in future versions). Please see the examples folder in the repo or the examples section in the docs for details on how to use self-supervised pre-training with this library.</p> <p>The two routines implemented are illustrated in the figures below. The first is from TabNet: Attentive Interpretable Tabular Learning. It is a 'standard' encoder-decoder architecture and and is designed here for models that do not use transformer-based architectures (or when the embeddings can all have different dimensions). The second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, it is based on Contrastive and Denoising learning and is designed for models that use transformer-based architectures (or when the embeddings all need to have the same dimension):</p> <p> </p> <p>Figure 1. Figure 2 in their paper. The caption of the original paper is included in case it is useful.</p> <p> </p> <p>Figure 2. Figure 1 in their paper. The caption of the original paper is included in case it is useful.</p> <p>Note that the self-supervised pre-trainers described below focus, of course, on the self-supervised pre-training phase, i.e. the left side in Figure 1 and the upper part in Figure 2. When combined with the <code>Trainer</code> described earlier in the documenation, one can reproduce the full process illustrated in the figures above.</p> <p>Also Note that it is beyond the scope of this docs to explain in detail these routines. In addition, to fully utilise the self-supervised trainers implemented in this library a minimum understanding of the processes as described in the papers is required. Therefore, we strongly encourage the users to have a look to the papers.</p>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer","title":"EncoderDecoderTrainer","text":"<pre><code>EncoderDecoderTrainer(\n    encoder,\n    decoder=None,\n    masked_prob=0.2,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>BaseEncoderDecoderTrainer</code></p> <p>This class implements an Encoder-Decoder self-supervised 'routine' inspired by TabNet: Attentive Interpretable Tabular Learning. See Figure 1 above.</p> <p>Parameters:</p> <ul> <li> <code>encoder</code>             (<code>ModelWithoutAttention</code>)         \u2013          <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p> </li> <li> <code>decoder</code>             (<code>Optional[DecoderWithoutAttention]</code>, default:                 <code>None</code> )         \u2013          <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or <code>TabNetDecoder</code> model. if <code>None</code> the decoder will be automatically built as a 'simetric' model to the Encoder</p> </li> <li> <code>masked_prob</code>             (<code>float</code>, default:                 <code>0.2</code> )         \u2013          <p>Indicates the fraction of elements in the embedding tensor that will be masked and hence used for reconstruction</p> </li> <li> <code>optimizer</code>             (<code>Optional[Optimizer]</code>, default:                 <code>None</code> )         \u2013          <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>             (<code>Optional[LRScheduler]</code>, default:                 <code>None</code> )         \u2013          <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>             (<code>Optional[List[Callback]]</code>, default:                 <code>None</code> )         \u2013          <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>         \u2013          <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def __init__(\n    self,\n    encoder: ModelWithoutAttention,\n    decoder: Optional[DecoderWithoutAttention] = None,\n    masked_prob: float = 0.2,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        encoder=encoder,\n        decoder=decoder,\n        masked_prob=masked_prob,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>             (<code>ndarray</code>)         \u2013          <p>tabular dataset</p> </li> <li> <code>X_tab_val</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>validation data</p> </li> <li> <code>val_split</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>validation_freq</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>epochs validation frequency</p> </li> <li> <code>n_epochs</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>number of epochs</p> </li> <li> <code>batch_size</code>             (<code>int</code>, default:                 <code>32</code> )         \u2013          <p>batch size</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.ed_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"ed_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history (if any) to disk</p> <p>Parameters:</p> <ul> <li> <code>path</code>             (<code>str</code>)         \u2013          <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> </li> <li> <code>model_filename</code>             (<code>str</code>, default:                 <code>'ed_model.pt'</code> )         \u2013          <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"ed_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history (if any) to disk\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"ed_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.ed_model.state_dict(), model_path)\n    else:\n        torch.save(self.ed_model, model_path)\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer","title":"ContrastiveDenoisingTrainer","text":"<pre><code>ContrastiveDenoisingTrainer(\n    model,\n    preprocessor,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    loss_type=\"both\",\n    projection_head1_dims=None,\n    projection_head2_dims=None,\n    projection_heads_activation=\"relu\",\n    cat_mlp_type=\"multiple\",\n    cont_mlp_type=\"multiple\",\n    denoise_mlps_activation=\"relu\",\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>BaseContrastiveDenoisingTrainer</code></p> <p>This class trains a Contrastive, Denoising Self Supervised 'routine' that is based on the one described in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their Figure 1.</p> <p>Parameters:</p> <ul> <li> <code>model</code>             (<code>ModelWithAttention</code>)         \u2013          <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>, <code>TabFastFormer</code>, <code>TabPerceiver</code>, <code>ContextAttentionMLP</code> and <code>SelfAttentionMLP</code>.</p> </li> <li> <code>preprocessor</code>             (<code>TabPreprocessor</code>)         \u2013          <p>A fitted <code>TabPreprocessor</code> object. See <code>pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor</code></p> </li> <li> <code>optimizer</code>             (<code>Optional[Optimizer]</code>, default:                 <code>None</code> )         \u2013          <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>             (<code>Optional[LRScheduler]</code>, default:                 <code>None</code> )         \u2013          <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>             (<code>Optional[List[Callback]]</code>, default:                 <code>None</code> )         \u2013          <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>loss_type</code>             (<code>Literal[contrastive, denoising, both]</code>, default:                 <code>'both'</code> )         \u2013          <p>One of 'contrastive', 'denoising' or 'both'. See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their figure (1) and their equation (5).</p> </li> <li> <code>projection_head1_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>The projection heads are simply MLPs. This parameter is a list of integers with the dimensions of the MLP hidden layers. See the paper for details. Note that setting up this parameter requires some knowledge of the architecture one is using. For example, if we are representing the features with embeddings of dim 32 (i.e. the so called dimension of the model is 32), then the first dimension of the projection head must be 32 (e.g. [32, 16])</p> </li> <li> <code>projection_head2_dims</code>             (<code>Optional[List[int]]</code>, default:                 <code>None</code> )         \u2013          <p>Same as 'projection_head1_dims' for the second head</p> </li> <li> <code>projection_heads_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>Activation function for the projection heads</p> </li> <li> <code>cat_mlp_type</code>             (<code>Literal[single, multiple]</code>, default:                 <code>'multiple'</code> )         \u2013          <p>If 'denoising' loss is used, one can choose two types of 'stacked' MLPs to process the output from the transformer-based encoder that receives 'corrupted' (cut-mixed and mixed-up) features. These are 'single' or 'multiple'. The former approach will apply a single MLP to all the categorical features while the latter will use one MLP per categorical feature</p> </li> <li> <code>cont_mlp_type</code>             (<code>Literal[single, multiple]</code>, default:                 <code>'multiple'</code> )         \u2013          <p>Same as 'cat_mlp_type' but for the continuous features</p> </li> <li> <code>denoise_mlps_activation</code>             (<code>str</code>, default:                 <code>'relu'</code> )         \u2013          <p>activation function for the so called 'denoising mlps'.</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>         \u2013          <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def __init__(\n    self,\n    model: ModelWithAttention,\n    preprocessor: TabPreprocessor,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    loss_type: Literal[\"contrastive\", \"denoising\", \"both\"] = \"both\",\n    projection_head1_dims: Optional[List[int]] = None,\n    projection_head2_dims: Optional[List[int]] = None,\n    projection_heads_activation: str = \"relu\",\n    cat_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n    cont_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n    denoise_mlps_activation: str = \"relu\",\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        preprocessor=preprocessor,\n        loss_type=loss_type,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        projection_head1_dims=projection_head1_dims,\n        projection_head2_dims=projection_head2_dims,\n        projection_heads_activation=projection_heads_activation,\n        cat_mlp_type=cat_mlp_type,\n        cont_mlp_type=cont_mlp_type,\n        denoise_mlps_activation=denoise_mlps_activation,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>             (<code>ndarray</code>)         \u2013          <p>tabular dataset</p> </li> <li> <code>X_tab_val</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>validation data. Note that, although it is possible to use contrastive-denoising training with a validation set, such set must include feature values that are all seen in the training set in the case of the categorical columns. This is because the values of the columns themselves will be used as targets when computing the loss. Therefore, if a new category is present in the validation set that was not seen in training this will effectively be like trying to predict a new, never seen category (and Pytorch will throw an error)</p> </li> <li> <code>val_split</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>validation_freq</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>epochs validation frequency</p> </li> <li> <code>n_epochs</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>number of epochs</p> </li> <li> <code>batch_size</code>             (<code>int</code>, default:                 <code>32</code> )         \u2013          <p>batch size</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data. Note that, although it is possible to use\n        contrastive-denoising training with a validation set, such set\n        must include feature values that are _all_ seen in the training\n        set in the case of the categorical columns. This is because the\n        values of the columns themselves will be used as targets when\n        computing the loss. Therefore, if a new category is present in\n        the validation set that was not seen in training this will\n        effectively be like trying to predict a new, never seen category\n        (and Pytorch will throw an error)\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.cd_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"cd_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history (if any) to disk</p> <p>Parameters:</p> <ul> <li> <code>path</code>             (<code>str</code>)         \u2013          <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> </li> <li> <code>model_filename</code>             (<code>str</code>, default:                 <code>'cd_model.pt'</code> )         \u2013          <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"cd_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history (if any) to disk\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"cd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.cd_model.state_dict(), model_path)\n    else:\n        torch.save(self.cd_model, model_path)\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html","title":"Tab2Vec","text":""},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec","title":"Tab2Vec","text":"<pre><code>Tab2Vec(\n    tab_preprocessor,\n    model,\n    return_dataframe=False,\n    verbose=False,\n)\n</code></pre> <p>Class to transform an input dataframe into vectorized form.</p> <p>This class will take an input dataframe in the form of the dataframe used for training, and it will turn it into a vectorised form based on the processing applied by the model to the categorical and continuous columns.</p> <p> NOTE: Currently this class is only implemented  for the deeptabular component. Therefore, if the input dataframe has a  text column or a column with the path to images, these will be ignored.  We will be adding these functionalities in future versions</p> <p>Parameters:</p> <ul> <li> <code>model</code>             (<code>Union[WideDeep, BayesianWide, BayesianTabMlp]</code>)         \u2013          <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p> </li> <li> <code>tab_preprocessor</code>             (<code>TabPreprocessor</code>)         \u2013          <p><code>TabPreprocessor</code> object. Must be fitted.</p> </li> <li> <code>return_dataframe</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating of the returned object(s) will be array(s) or pandas dataframe(s)</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vectorizer</code>             (<code>Module</code>)         \u2013          <p>Torch module with the categorical and continuous encoding process</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import string\n&gt;&gt;&gt; from random import choices\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n&gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n&gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n&gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n&gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # fit the TabPreprocessor\n&gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n&gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # define the model (and let's assume we train it)\n&gt;&gt;&gt; tabmlp = TabMlp(\n... column_idx=tab_preprocessor.column_idx,\n... cat_embed_input=tab_preprocessor.cat_embed_input,\n... continuous_cols=tab_preprocessor.continuous_cols,\n... mlp_hidden_dims=[8, 4])\n&gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n&gt;&gt;&gt; # ...train the model...\n&gt;&gt;&gt;\n&gt;&gt;&gt; # vectorise the dataframe\n&gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model)\n&gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n</code></pre> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def __init__(\n    self,\n    tab_preprocessor: TabPreprocessor,\n    model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n    return_dataframe: bool = False,\n    verbose: bool = False,\n):\n    super(Tab2Vec, self).__init__()\n\n    self._check_inputs(tab_preprocessor, model, verbose)\n\n    self.tab_preprocessor = tab_preprocessor\n    self.return_dataframe = return_dataframe\n    self.verbose = verbose\n\n    self.vectorizer = self._set_vectorizer(model)\n\n    self._set_dim_attributes(tab_preprocessor, model)\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit","title":"fit","text":"<pre><code>fit(df, target_col=None)\n</code></pre> <p>This is an empty method i.e. Returns the unchanged object itself. Is only included for consistency in case <code>Tab2Vec</code> is used as part of a Pipeline</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> </li> <li> <code>target_col</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>Tab2Vec</code>         \u2013          </li> </ul> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n    r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n    only included for consistency in case `Tab2Vec` is used as part of a\n    Pipeline\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Tab2Vec\n    \"\"\"\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.transform","title":"transform","text":"<pre><code>transform(df, target_col=None)\n</code></pre> <p>Transforms the input dataframe into vectorized form. If a target column name is passed the target values will be returned separately in their corresponding type (np.ndarray or pd.DataFrame)</p> <p>Parameters:</p> <ul> <li> <code>df</code>             (<code>DataFrame</code>)         \u2013          <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> </li> <li> <code>target_col</code>             (<code>Optional[str]</code>, default:                 <code>None</code> )         \u2013          <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code>         \u2013          <p>Returns eiter a numpy array with the vectorised values, or a Tuple of numpy arrays with the vectorised values and the target. The same applies to dataframes in case we choose to set <code>return_dataframe = True</code></p> </li> </ul> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def transform(\n    self,\n    df: pd.DataFrame,\n    target_col: Optional[str] = None,\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Transforms the input dataframe into vectorized form. If a target\n    column name is passed the target values will be returned separately\n    in their corresponding type (np.ndarray or pd.DataFrame)\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n        Returns eiter a numpy array with the vectorised values, or a Tuple\n        of numpy arrays with the vectorised values and the target. The\n        same applies to dataframes in case we choose to set\n        `return_dataframe = True`\n    \"\"\"\n\n    X_tab = self.tab_preprocessor.transform(df)\n    X = torch.from_numpy(X_tab.astype(\"float\")).to(device)\n\n    with torch.no_grad():\n        if self.is_tab_transformer:\n            x_vec, x_cont_not_embed = self.vectorizer(X)\n        else:\n            x_vec = self.vectorizer(X)\n            x_cont_not_embed = None\n\n    if self.tab_preprocessor.with_cls_token:\n        x_vec = x_vec[:, 1:, :]\n\n    if self.tab_preprocessor.with_attention:\n        x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n    if x_cont_not_embed is not None:\n        x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n    else:\n        x_vec = x_vec.detach().cpu().numpy()\n\n    if self.return_dataframe:\n        new_colnames = self._new_colnames()\n        if target_col:\n            return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n        else:\n            return pd.DataFrame(data=x_vec, columns=new_colnames)\n    else:\n        if target_col:\n            return x_vec, df[target_col].values\n        else:\n            return x_vec\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df, target_col=None)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit_transform(\n    self, df: pd.DataFrame, target_col: Optional[str] = None\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Combines `fit` and `transform`\"\"\"\n    return self.fit(df, target_col).transform(df, target_col)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html","title":"Training multimodal Deep Learning Models","text":"<p>Here is the documentation for the <code>Trainer</code> class, that will do all the heavy lifting.</p> <p>Trainer is also available from <code>pytorch-widedeep</code> directly, for example, one could do:</p> <pre><code>    from pytorch-widedeep.training import Trainer\n</code></pre> <p>or also:</p> <pre><code>    from pytorch-widedeep import Trainer\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer","title":"Trainer","text":"<pre><code>Trainer(\n    model,\n    objective,\n    custom_loss_function=None,\n    optimizers=None,\n    lr_schedulers=None,\n    initializers=None,\n    transforms=None,\n    callbacks=None,\n    metrics=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>             Bases: <code>BaseTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Parameters:</p> <ul> <li> <code>model</code>             (<code>WideDeep</code>)         \u2013          <p>An object of class <code>WideDeep</code></p> </li> <li> <code>objective</code>             (<code>str</code>)         \u2013          <p>Defines the objective, loss or cost function. </p> <p>Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code>. </p> <p>Possible values are:</p> <ul> <li> <p><code>binary</code>, aliases: <code>logistic</code>, <code>binary_logloss</code>, <code>binary_cross_entropy</code></p> </li> <li> <p><code>binary_focal_loss</code></p> </li> <li> <p><code>multiclass</code>, aliases: <code>multi_logloss</code>, <code>cross_entropy</code>, <code>categorical_cross_entropy</code>,</p> </li> <li> <p><code>multiclass_focal_loss</code></p> </li> <li> <p><code>regression</code>, aliases: <code>mse</code>, <code>l2</code>, <code>mean_squared_error</code></p> </li> <li> <p><code>mean_absolute_error</code>, aliases: <code>mae</code>, <code>l1</code></p> </li> <li> <p><code>mean_squared_log_error</code>, aliases: <code>msle</code></p> </li> <li> <p><code>root_mean_squared_error</code>, aliases:  <code>rmse</code></p> </li> <li> <p><code>root_mean_squared_log_error</code>, aliases: <code>rmsle</code></p> </li> <li> <p><code>zero_inflated_lognormal</code>, aliases: <code>ziln</code></p> </li> <li> <p><code>quantile</code></p> </li> <li> <p><code>tweedie</code></p> </li> <li> <p><code>multitarget</code>, aliases: <code>multi_target</code></p> </li> </ul> <p>NOTE: For <code>multitarget</code> a custom loss function must be passed</p> </li> <li> <code>custom_loss_function</code>             (<code>Optional[Module]</code>, default:                 <code>None</code> )         \u2013          <p>It is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples section in this documentation or in the repo. Note that if <code>custom_loss_function</code> is not <code>None</code>, <code>objective</code> must be 'binary', 'multiclass' or 'regression', consistent with the loss function</p> </li> <li> <code>optimizers</code>             (<code>Optional[Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]]</code>, default:                 <code>None</code> )         \u2013          <ul> <li>An instance of Pytorch's <code>Optimizer</code> object   (e.g. <code>torch.optim.Adam()</code>) or</li> <li>a dictionary where there keys are the model components (i.e.   'wide', 'deeptabular', 'deeptext', 'deepimage'   and/or 'deephead')  and the values are the corresponding   optimizers or list of optimizers if multiple models are used for   the given data mode (e.g. two text columns/models for the deeptext   component). If multiple optimizers are used the   dictionary MUST contain an optimizer per model component.</li> </ul> <p>if no optimizers are passed it will default to <code>Adam</code> for all model components</p> </li> <li> <code>lr_schedulers</code>             (<code>Optional[Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]]</code>, default:                 <code>None</code> )         \u2013          <ul> <li>An instance of Pytorch's <code>LRScheduler</code> object (e.g   <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>) or</li> <li>a dictionary where there keys are the model componenst (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead') and the   values are the corresponding learning rate schedulers or list of     learning rate schedulers if multiple models are used for the given     data mode (e.g. two text columns/models for the deeptext component).</li> </ul> </li> <li> <code>initializers</code>             (<code>Optional[Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]]</code>, default:                 <code>None</code> )         \u2013          <ul> <li>An instance of an <code>Initializer</code> object see <code>pytorch-widedeep.initializers</code> or</li> <li>a dictionary where there keys are the model components (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead')   and the values are the corresponding initializers or list of     initializers if multiple models are used for the given data mode (e.g.     two text columns/models for the deeptext component).</li> </ul> </li> <li> <code>transforms</code>             (<code>Optional[List[Transforms]]</code>, default:                 <code>None</code> )         \u2013          <p>List with <code>torchvision.transforms</code> to be applied to the image component of the model (i.e. <code>deepimage</code>) See torchvision transforms.</p> </li> <li> <code>callbacks</code>             (<code>Optional[List[Callback]]</code>, default:                 <code>None</code> )         \u2013          <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. The <code>History</code> and the <code>LRShedulerCallback</code> callbacks are used by default. This can also be a custom callback as long as the object of type <code>Callback</code>. See <code>pytorch_widedeep.callbacks.Callback</code> or the examples folder in the repo.</p> </li> <li> <code>metrics</code>             (<code>Optional[Union[List[Metric], List[Metric]]]</code>, default:                 <code>None</code> )         \u2013          <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as long   as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the examples folder in the   repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library   Examples.   This can also be a custom metric as long as   it is an object of type <code>Metric</code>. See   the instructions.</li> </ul> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Verbosity level. If set to 0 nothing will be printed during training</p> </li> <li> <code>seed</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>Random seed to be used internally for train/test split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>         \u2013          <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>lambda_sparse: <code>float</code>     lambda sparse parameter in case the <code>deeptabular</code> component is <code>TabNet</code></p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</li> </ul> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cyclic_lr</code>             (<code>bool</code>)         \u2013          <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See Pytorch schedulers.</p> </li> <li> <code>feature_importance</code>             (<code>dict</code>)         \u2013          <p>dict where the keys are the column names and the values are the corresponding feature importances. This attribute will only exist if the <code>deeptabular</code> component is a Tabnet model.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torchvision.transforms import ToTensor\n&gt;&gt;&gt;\n&gt;&gt;&gt; # wide deep imports\n&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # build the model\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set optimizers and schedulers\n&gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n&gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n&gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n&gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n&gt;&gt;&gt;\n&gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n&gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n&gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n&gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n&gt;&gt;&gt;\n&gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n&gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set initializers and callbacks\n&gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n&gt;&gt;&gt; transforms = [ToTensor]\n&gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set the trainer\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n</code></pre> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(  # noqa: C901\n    \"objective\",\n    [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n)\ndef __init__(\n    self,\n    model: WideDeep,\n    objective: str,\n    custom_loss_function: Optional[nn.Module] = None,\n    optimizers: Optional[\n        Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]\n    ] = None,\n    lr_schedulers: Optional[\n        Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]\n    ] = None,\n    initializers: Optional[\n        Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]\n    ] = None,\n    transforms: Optional[List[Transforms]] = None,\n    callbacks: Optional[List[Callback]] = None,\n    metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        objective=objective,\n        custom_loss_function=custom_loss_function,\n        optimizers=optimizers,\n        lr_schedulers=lr_schedulers,\n        initializers=initializers,\n        transforms=transforms,\n        callbacks=callbacks,\n        metrics=metrics,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.fit","title":"fit","text":"<pre><code>fit(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_train=None,\n    X_val=None,\n    val_split=None,\n    target=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    custom_dataloader=None,\n    feature_importance_sample_size=None,\n    finetune=False,\n    with_lds=False,\n    **kwargs\n)\n</code></pre> <p>Fit method.</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in dictionaries (<code>X_train</code> or <code>X_val</code>).</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>             (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code>. If multiple text columns/models are used, this should be a list of numpy arrays</p> </li> <li> <code>X_img</code>             (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code>. If multiple image columns/models are used, this should be a list of numpy arrays</p> </li> <li> <code>X_train</code>             (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                 <code>None</code> )         \u2013          <p>The training dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> </li> <li> <code>X_val</code>             (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                 <code>None</code> )         \u2013          <p>The validation dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> </li> <li> <code>val_split</code>             (<code>Optional[float]</code>, default:                 <code>None</code> )         \u2013          <p>train/val split fraction</p> </li> <li> <code>target</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>target values</p> </li> <li> <code>n_epochs</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>number of epochs</p> </li> <li> <code>validation_freq</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>epochs validation frequency</p> </li> <li> <code>batch_size</code>             (<code>int</code>, default:                 <code>32</code> )         \u2013          <p>batch size</p> </li> <li> <code>custom_dataloader</code>             (<code>Optional[DataLoader]</code>, default:                 <code>None</code> )         \u2013          <p>object of class <code>torch.utils.data.DataLoader</code>. Available predefined dataloaders are in <code>pytorch-widedeep.dataloaders</code>.If <code>None</code>, a standard torch <code>DataLoader</code> is used.</p> </li> <li> <code>finetune</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>fine-tune individual model components. This functionality can also be used to 'warm-up' (and hence the alias <code>warmup</code>) individual components before the joined training starts, and hence its alias. See the Examples folder in the repo for more details</p> <p><code>pytorch_widedeep</code> implements 3 fine-tune routines.</p> <ul> <li>fine-tune all trainable layers at once. This routine is   inspired by the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper. Using a   Slanted Triangular learing (see   Leslie N. Smith paper ) ,   the process is the following: i) the learning rate will   gradually increase for 10% of the training steps from max_lr/10   to max_lr. ii) It will then gradually decrease to max_lr/10   for the remaining 90% of the steps. The optimizer used in the   process is <code>Adam</code>.</li> </ul> <p>and two gradual fine-tune routines, where only certain layers are trained at a time.</p> <ul> <li>The so called <code>Felbo</code> gradual fine-tune rourine, based on the the   Felbo et al., 2017 DeepEmoji paper.</li> <li>The <code>Howard</code> routine based on the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper.</li> </ul> <p>For details on how these routines work, please see the Examples section in this documentation and the Examples folder in the repo.  Param Alias: <code>warmup</code></p> </li> <li> <code>with_lds</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if Label Distribution Smoothing will be used.  information_source: NOTE: We consider this feature absolutely experimental and we recommend the user to not use it unless the corresponding publication is well understood</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>         \u2013          <p>Other keyword arguments are:</p> <ul> <li> <p>DataLoader related parameters:     For example,  <code>sampler</code>, <code>batch_sampler</code>, <code>collate_fn</code>, etc.     Please, see the pytorch     DataLoader docs     for details.</p> </li> <li> <p>Label Distribution Smoothing related parameters:</p> <ul> <li>lds_kernel (<code>Literal['gaussian', 'triang', 'laplace']</code>):     choice of kernel for Label Distribution Smoothing</li> <li>lds_ks (<code>int</code>):     LDS kernel window size</li> <li>lds_sigma (<code>float</code>):     standard deviation of ['gaussian','laplace'] kernel for LDS</li> <li>lds_granularity (<code>int</code>):     number of bins in histogram used in LDS to count occurence of sample values</li> <li>lds_reweight (<code>bool</code>):     option to reweight bin frequency counts in LDS</li> <li>lds_y_max (<code>Optional[float]</code>):     option to restrict LDS bins by upper label limit</li> <li>lds_y_min (<code>Optional[float]</code>):     option to restrict LDS bins by lower label limit</li> </ul> <p>See <code>pytorch_widedeep.trainer._wd_dataset</code> for more details on the implications of these parameters</p> </li> <li> <p>Finetune related parameters:     see the source code at <code>pytorch_widedeep._finetune</code>. Namely, these are:</p> <ul> <li><code>finetune_epochs</code> (<code>int</code>):     number of epochs use for fine tuning</li> <li><code>finetune_max_lr</code> (<code>float</code>):    max lr during fine tuning</li> <li><code>routine</code> (<code>str</code>):    one of 'howard' or 'felbo'</li> <li><code>deeptabular_gradual</code> (<code>bool</code>):    boolean indicating if the <code>deeptabular</code> component will be fine tuned gradually</li> <li><code>deeptabular_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    List of pytorch modules indicating the layers of the    <code>deeptabular</code> that will be fine tuned</li> <li><code>deeptabular_max_lr</code> (<code>Union[float, List[float]]</code>):    max lr for the <code>deeptabular</code> componet during fine tuning</li> <li><code>deeptext_gradual</code> (<code>bool</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component</li> <li><code>deeptext_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component.    If there are multiple text columns/models, this should be a list of lists</li> <li><code>deeptext_max_lr</code> (<code>Union[float, List[float]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component    If there are multiple text columns/models, this should be a list of floats</li> <li><code>deepimage_gradual</code> (<code>bool</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_max_lr</code> (<code>Union[float, List[float]]</code>):     same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> </ul> </li> </ul> </li> </ul> <p>Examples:</p> <p>For a series of comprehensive examples on how to use the <code>fit</code> method, please see the Examples folder in the repo</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(\"finetune\", [\"warmup\"])\ndef fit(  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[np.ndarray] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    val_split: Optional[float] = None,\n    target: Optional[np.ndarray] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    custom_dataloader: Optional[DataLoader] = None,\n    feature_importance_sample_size: Optional[int] = None,\n    finetune: bool = False,\n    with_lds: bool = False,\n    **kwargs,\n):\n    r\"\"\"Fit method.\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    dictionaries (`X_train` or `X_val`).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n        If multiple text columns/models are used, this should be a list of\n        numpy arrays\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n        If multiple image columns/models are used, this should be a list of\n        numpy arrays\n    X_train: Dict, Optional. default=None\n        The training dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices. Note that of multiple text or image\n        columns/models are used, the corresponding values should be lists\n        of numpy arrays\n    X_val: Dict, Optional. default=None\n        The validation dataset can also be passed in a dictionary. Keys\n        are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n        Values are the corresponding matrices. Note that of multiple text\n        or image columns/models are used, the corresponding values should\n        be lists of numpy arrays\n    val_split: float, Optional. default=None\n        train/val split fraction\n    target: np.ndarray, Optional. default=None\n        target values\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    custom_dataloader: `DataLoader`, Optional, default=None\n        object of class `torch.utils.data.DataLoader`. Available\n        predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n        `None`, a standard torch `DataLoader` is used.\n    finetune: bool, default=False\n        fine-tune individual model components. This functionality can also\n        be used to 'warm-up' (and hence the alias `warmup`) individual\n        components before the joined training starts, and hence its\n        alias. See the Examples folder in the repo for more details\n\n        `pytorch_widedeep` implements 3 fine-tune routines.\n\n        - fine-tune all trainable layers at once. This routine is\n          inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n          Slanted Triangular learing (see\n          [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n          the process is the following: *i*) the learning rate will\n          gradually increase for 10% of the training steps from max_lr/10\n          to max_lr. *ii*) It will then gradually decrease to max_lr/10\n          for the remaining 90% of the steps. The optimizer used in the\n          process is `Adam`.\n\n        and two gradual fine-tune routines, where only certain layers are\n        trained at a time.\n\n        - The so called `Felbo` gradual fine-tune rourine, based on the the\n          Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n        - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n        For details on how these routines work, please see the Examples\n        section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n        Param Alias: `warmup`\n    with_lds: bool, default=False\n        Boolean indicating if Label Distribution Smoothing will be used. &lt;br/&gt;\n        information_source: **NOTE**: We consider this feature absolutely\n        experimental and we recommend the user to not use it unless the\n        corresponding [publication](https://arxiv.org/abs/2102.09554) is\n        well understood\n\n    Other Parameters\n    ----------------\n    **kwargs:\n        Other keyword arguments are:\n\n        - **DataLoader related parameters**:&lt;br/&gt;\n            For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n            Please, see the pytorch\n            [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n            for details.\n\n        - **Label Distribution Smoothing related parameters**:&lt;br/&gt;\n\n            - lds_kernel (`Literal['gaussian', 'triang', 'laplace']`):\n                choice of kernel for Label Distribution Smoothing\n            - lds_ks (`int`):\n                LDS kernel window size\n            - lds_sigma (`float`):\n                standard deviation of ['gaussian','laplace'] kernel for LDS\n            - lds_granularity (`int`):\n                number of bins in histogram used in LDS to count occurence of sample values\n            - lds_reweight (`bool`):\n                option to reweight bin frequency counts in LDS\n            - lds_y_max (`Optional[float]`):\n                option to restrict LDS bins by upper label limit\n            - lds_y_min (`Optional[float]`):\n                option to restrict LDS bins by lower label limit\n\n            See `pytorch_widedeep.trainer._wd_dataset` for more details on\n            the implications of these parameters\n\n        - **Finetune related parameters**:&lt;br/&gt;\n            see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n            - `finetune_epochs` (`int`):\n                number of epochs use for fine tuning\n            - `finetune_max_lr` (`float`):\n               max lr during fine tuning\n            - `routine` (`str`):\n               one of _'howard'_ or _'felbo'_\n            - `deeptabular_gradual` (`bool`):\n               boolean indicating if the `deeptabular` component will be fine tuned gradually\n            - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               List of pytorch modules indicating the layers of the\n               `deeptabular` that will be fine tuned\n            - `deeptabular_max_lr` (`Union[float, List[float]]`):\n               max lr for the `deeptabular` componet during fine tuning\n            - `deeptext_gradual` (`bool`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n            - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component.\n               If there are multiple text columns/models, this should be a list of lists\n            - `deeptext_max_lr` (`Union[float, List[float]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n               If there are multiple text columns/models, this should be a list of floats\n            - `deepimage_gradual` (`bool`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_max_lr` (`Union[float, List[float]]`):\n                same as `deeptext_layers` but for the `deepimage` component\n\n    Examples\n    --------\n\n    For a series of comprehensive examples on how to use the `fit` method, please see the\n    [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n    folder in the repo\n    \"\"\"\n\n    lds_args, dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n    lds_args[\"with_lds\"] = with_lds\n    self.with_lds = with_lds\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = wd_train_val_split(\n        self.seed,\n        self.method,  # type: ignore\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_train,\n        X_val,\n        val_split,\n        target,\n        self.transforms,\n        **lds_args,\n    )\n    if isinstance(custom_dataloader, type):\n        if issubclass(custom_dataloader, DataLoader):\n            train_loader = custom_dataloader(  # type: ignore[misc]\n                dataset=train_set,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                **dataloader_args,\n            )\n        else:\n            NotImplementedError(\n                \"Custom DataLoader must be a subclass of \"\n                \"torch.utils.data.DataLoader, please see the \"\n                \"pytorch documentation or examples in \"\n                \"pytorch_widedeep.dataloaders\"\n            )\n    else:\n        train_loader = DataLoaderDefault(\n            dataset=train_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            **dataloader_args,\n        )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    if finetune:\n        self.with_finetuning: bool = True\n        self._finetune(train_loader, **finetune_args)\n        if self.verbose:\n            print(\n                \"Fine-tuning (or warmup) of individual components completed. \"\n                \"Training the whole model for {} epochs\".format(n_epochs)\n            )\n    else:\n        self.with_finetuning = False\n\n    self.callback_container.on_train_begin(\n        {\"batch_size\": batch_size, \"train_steps\": train_steps, \"n_epochs\": n_epochs}\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (data, targett, lds_weightt) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(\n                    data, targett, batch_idx, epoch, lds_weightt\n                )\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (data, targett) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(data, targett, i)\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            # self.callback_container.on_train_end(epoch_logs)\n            break\n\n        if self.model.with_fds:\n            self._update_fds_stats(train_loader, epoch)\n\n    self.callback_container.on_train_end(epoch_logs)\n\n    if feature_importance_sample_size is not None:\n        self.feature_importance = FeatureImportance(\n            self.device, feature_importance_sample_size\n        ).feature_importance(train_loader, self.model)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict","title":"predict","text":"<pre><code>predict(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predictions</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>             (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>             (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>             (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                 <code>None</code> )         \u2013          <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>         \u2013          <p>array with the predictions</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict(  # type: ignore[override, return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[np.ndarray] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray, Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"regression\":\n        return np.vstack(preds_l).squeeze(1)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        return (preds &gt; 0.5).astype(\"int\")\n    if self.method == \"qregression\":\n        return np.vstack(preds_l)\n    if self.method == \"multiclass\":\n        preds = np.vstack(preds_l)\n        return np.argmax(preds, 1)  # type: ignore[return-value]\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_uncertainty","title":"predict_uncertainty","text":"<pre><code>predict_uncertainty(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n    uncertainty_granularity=1000,\n)\n</code></pre> <p>Returns the predicted ucnertainty of the model for the test dataset using a Monte Carlo method during which dropout layers are activated in the evaluation/prediction phase and each sample is predicted N times (<code>uncertainty_granularity</code> times).</p> <p>This is based on Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning.</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>             (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>             (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>             (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                 <code>None</code> )         \u2013          <p>The test dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> <li> <code>uncertainty_granularity</code>         \u2013          <p>number of times the model does prediction for each sample</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>         \u2013          <ul> <li>if <code>method = regression</code>, it will return an array with <code>(max, min, mean, stdev)</code>   values for each sample.</li> <li>if <code>method = binary</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)</code> for each sample.</li> <li>if <code>method = multiclass</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)</code>   values for each sample.</li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_uncertainty(  # type: ignore[return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[np.ndarray] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n    uncertainty_granularity=1000,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n    using a Monte Carlo method during which dropout layers are activated\n    in the evaluation/prediction phase and each sample is predicted N\n    times (`uncertainty_granularity` times).\n\n    This is based on\n    [Dropout as a Bayesian Approximation: Representing\n    Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray, Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n    uncertainty_granularity: int default = 1000\n        number of times the model does prediction for each sample\n\n    Returns\n    -------\n    np.ndarray:\n        - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n          values for each sample.\n        - if `method = binary` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n        - if `method = multiclass` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n          values for each sample.\n\n    \"\"\"\n    preds_l = self._predict(\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_test,\n        batch_size,\n        uncertainty_granularity,\n        uncertainty=True,\n    )\n    preds = np.vstack(preds_l)\n    samples_num = int(preds.shape[0] / uncertainty_granularity)\n    if self.method == \"regression\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        return np.array(\n            (\n                preds.max(axis=0),\n                preds.min(axis=0),\n                preds.mean(axis=0),\n                preds.std(axis=0),\n            )\n        ).T\n    if self.method == \"qregression\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for qregression method\"\n        )\n    if self.method == \"binary\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        preds = preds.mean(axis=0)\n        probs = np.zeros([preds.shape[0], 3])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n        preds = preds.mean(axis=0)\n        preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n        return preds\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predicted probabilities for the test dataset for  binary and multiclass methods</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>             (<code>Optional[ndarray]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>             (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>             (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                 <code>None</code> )         \u2013          <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>             (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                 <code>None</code> )         \u2013          <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>array with the probabilities per class</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_proba(  # type: ignore[override, return]  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[np.ndarray] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n    and multiclass methods\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray, Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        probs = np.zeros([preds.shape[0], 2])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"wd_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history, and the <code>feature_importance</code> attribute (if the <code>deeptabular</code> component is a Tabnet model) to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers, learning rate schedulers, initializers, etc) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>The exception is Tabnet. If the <code>deeptabular</code> component is a Tabnet model, an attribute (a dict) called <code>feature_importance</code> will be created at the end of the training process. Therefore, a <code>save</code> method was created that will save the feature importance dictionary to a json file and, since we are here, the model weights, training history and learning rate history.</p> <p>Parameters:</p> <ul> <li> <code>path</code>             (<code>str</code>)         \u2013          <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> </li> <li> <code>model_filename</code>             (<code>str</code>, default:                 <code>'wd_model.pt'</code> )         \u2013          <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"wd_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history, and the\n    `feature_importance` attribute (if the `deeptabular` component is a\n    Tabnet model) to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers,\n    learning rate schedulers, initializers, etc) have to be defined\n    externally and then passed to the `Trainer`. As a result, the\n    `Trainer` does not generate any attribute or additional data\n    products that need to be saved other than the `model` object itself,\n    which can be saved as any other torch model (e.g. `torch.save(model,\n    path)`).\n\n    The exception is Tabnet. If the `deeptabular` component is a Tabnet\n    model, an attribute (a dict) called `feature_importance` will be\n    created at the end of the training process. Therefore, a `save`\n    method was created that will save the feature importance dictionary\n    to a json file and, since we are here, the model weights, training\n    history and learning rate history.\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.model.state_dict(), model_path)\n    else:\n        torch.save(self.model, model_path)\n\n    if self.model.is_tabnet:\n        with open(save_dir / \"feature_importance.json\", \"w\") as fi:\n            json.dump(self.feature_importance, fi)\n</code></pre>"},{"location":"pytorch-widedeep/utils/index.html","title":"The <code>utils</code> module","text":"<p>These are a series of utilities that might be useful for a number of preprocessing tasks, even not directly related to <code>pytorch-widedeep</code>. All the classes and functions discussed here are available directly from the <code>utils</code> module. For example, the <code>LabelEncoder</code> within the <code>deeptabular_utils</code> submodule can be imported as:</p> <pre><code>from pytorch_widedeep.utils import LabelEncoder\n</code></pre> <p>These are classes and functions that are internally used in the library. We include them here in case the user finds them useful for other purposes.</p>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html","title":"deeptabular utils","text":""},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder","title":"LabelEncoder","text":"<pre><code>LabelEncoder(\n    columns_to_encode=None,\n    with_attention=False,\n    shared_embed=False,\n)\n</code></pre> <p>Label Encode categorical values for multiple columns at once</p> <p> NOTE: LabelEncoder reserves 0 for <code>unseen</code> new categories. This is convenient when defining the embedding layers, since we can just set padding idx to 0.</p> <p>Parameters:</p> <ul> <li> <code>columns_to_encode</code>             (<code>Optional[List[str]]</code>, default:                 <code>None</code> )         \u2013          <p>List of strings containing the names of the columns to encode. If <code>None</code> all columns of type <code>object</code> in the dataframe will be label encoded.</p> </li> <li> <code>with_attention</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model. Aliased as <code>for_transformer</code>.</p> </li> <li> <code>shared_embed</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoding_dict</code>             (<code>Dict</code>)         \u2013          <p>Dictionary containing the encoding mappings in the format, e.g. :  <code>{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}</code></p> </li> <li> <code>inverse_encoding_dict</code>             (<code>Dict</code>)         \u2013          <p>Dictionary containing the inverse encoding mappings in the format, e.g. :  <code>{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}</code></p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\ndef __init__(\n    self,\n    columns_to_encode: Optional[List[str]] = None,\n    with_attention: bool = False,\n    shared_embed: bool = False,\n):\n    self.columns_to_encode = columns_to_encode\n\n    self.shared_embed = shared_embed\n    self.with_attention = with_attention\n\n    self.reset_embed_idx = not self.with_attention or self.shared_embed\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(df)\n</code></pre> <p>Main method. Creates encoding attributes.</p> <p>Returns:</p> <ul> <li> <code>LabelEncoder</code>         \u2013          <p><code>LabelEncoder</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n    \"\"\"Main method. Creates encoding attributes.\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # here df is a chunk of the data. this is meant to be run when the\n    # data is large and we pass a chunk at a time. Therefore, we do not\n    # copy the input chunk as mutating a chunk is ok\n    if self.columns_to_encode is None:\n        self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n    else:\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:\n            df[col] = df[col].astype(\"O\")\n\n    unique_column_vals: Dict[str, List[str]] = {}\n    for c in self.columns_to_encode:\n        unique_column_vals[c] = df[c].unique().tolist()\n\n    if not hasattr(self, \"encoding_dict\"):\n        # we run the method 'partial_fit' for the 1st time\n        self.encoding_dict: Dict[str, Dict[str, int]] = {}\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n            del unique_column_vals[\"cls_token\"]\n\n        # leave 0 for padding/\"unseen\" categories. Also we need an\n        # attribute to keep track of the encoding in case we use\n        # attention and we do not re-start the index/counter\n        self.cum_idx: int = 1\n        for k, v in unique_column_vals.items():\n            self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n            self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n    else:\n        # the 'partial_fit' method has already run.\n        # \"cls_token\" will have been added already\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            del unique_column_vals[\"cls_token\"]\n\n        # Classes in the new df/chunk of the dataset that have not been seen\n        # before\n        unseen_classes: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unseen_classes[c] = list(\n                np.setdiff1d(\n                    unique_column_vals[c], list(self.encoding_dict[c].keys())\n                )\n            )\n\n        # leave 0 for padding/\"unseen\" categories\n        for k, v in unique_column_vals.items():\n            # if we use attention we need to start encoding from the\n            # last 'overall' encoding index. Otherwise, we use the max\n            # encoding index per categorical col\n            _idx = (\n                max(self.encoding_dict[k].values()) + 1\n                if self.reset_embed_idx\n                else self.cum_idx\n            )\n            if len(unseen_classes[k]) != 0:\n                for i, o in enumerate(unseen_classes[k]):\n                    if o not in self.encoding_dict[k]:\n                        self.encoding_dict[k][o] = i + _idx\n                # if self.reset_embed_idx is True it will be 1 anyway\n                self.cum_idx = (\n                    1\n                    if self.reset_embed_idx\n                    else self.cum_idx + len(unseen_classes[k])\n                )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p> <p>Returns:</p> <ul> <li> <code>LabelEncoder</code>         \u2013          <p><code>LabelEncoder</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n    \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # this is meant to be run when the data fits in memory and therefore,\n    # we do not want to mutate the original df, so we copy it\n    self.partial_fit(df.copy())\n\n    self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Label Encoded the categories in <code>columns_to_encode</code></p> <p>Returns:</p> <ul> <li> <code>DataFrame</code>         \u2013          <p>label-encoded dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Label Encoded the categories in `columns_to_encode`\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    try:\n        self.encoding_dict\n    except AttributeError:\n        raise NotFittedError(\n            \"This LabelEncoder instance is not fitted yet. \"\n            \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n        )\n\n    df_inp = df.copy()\n    # sanity check to make sure all categorical columns are in an adequate\n    # format\n    for col in self.columns_to_encode:  # type: ignore\n        df_inp[col] = df_inp[col].astype(\"O\")\n\n    for k, v in self.encoding_dict.items():\n        df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n    return df_inp\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; encoder.fit_transform(df)\n   col1  col2\n0     1     1\n1     2     2\n2     3     3\n&gt;&gt;&gt; encoder.encoding_dict\n{'col2': {'me': 1, 'you': 2, 'him': 3}}\n</code></pre> <p>Returns:</p> <ul> <li> <code>DataFrame</code>         \u2013          <p>label-encoded dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Combines `fit` and `transform`\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; encoder.fit_transform(df)\n       col1  col2\n    0     1     1\n    1     2     2\n    2     3     3\n    &gt;&gt;&gt; encoder.encoding_dict\n    {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(df)\n</code></pre> <p>Returns the original categories</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n&gt;&gt;&gt; encoder.inverse_transform(df_enc)\n   col1 col2\n0     1   me\n1     2  you\n2     3  him\n</code></pre> <p>Returns:</p> <ul> <li> <code>DataFrame</code>         \u2013          <p>DataFrame with original categories</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Returns the original categories\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n    &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n       col1 col2\n    0     1   me\n    1     2  you\n    2     3  him\n\n    Returns\n    -------\n    pd.DataFrame\n        DataFrame with original categories\n    \"\"\"\n\n    if not hasattr(self, \"inverse_encoding_dict\"):\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    for k, v in self.inverse_encoding_dict.items():\n        df[k] = df[k].apply(lambda x: v[x])\n\n    return df\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html","title":"Fastai transforms","text":"<p>I directly copied and pasted part of the <code>transforms.py</code> module from the <code>fastai</code> library (from an old version). The reason to do such a thing is because <code>pytorch_widedeep</code> only needs the <code>Tokenizer</code> and the <code>Vocab</code> classes there. This way I avoid extra dependencies. Credit for all the code in the <code>fastai_transforms</code> module in this <code>pytorch-widedeep</code> package goes to Jeremy Howard and the <code>fastai</code> team. I only include the documentation here for completion, but I strongly advise the user to read the <code>fastai</code> documentation.</p>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer","title":"Tokenizer","text":"<pre><code>Tokenizer(\n    tok_func=SpacyTokenizer,\n    lang=\"en\",\n    pre_rules=None,\n    post_rules=None,\n    special_cases=None,\n    n_cpus=None,\n)\n</code></pre> <p>Class to combine a series of rules and a tokenizer function to tokenize text with multiprocessing.</p> <p>Setting some of the parameters of this class require perhaps some familiarity with the source code.</p> <p>Parameters:</p> <ul> <li> <code>tok_func</code>             (<code>Callable</code>, default:                 <code>SpacyTokenizer</code> )         \u2013          <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p> </li> <li> <code>lang</code>             (<code>str</code>, default:                 <code>'en'</code> )         \u2013          <p>Text's Language</p> </li> <li> <code>pre_rules</code>             (<code>Optional[ListRules]</code>, default:                 <code>None</code> )         \u2013          <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the text (str) directly as <code>rule(tok)</code> before being tokenized.</p> </li> <li> <code>post_rules</code>             (<code>Optional[ListRules]</code>, default:                 <code>None</code> )         \u2013          <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the tokens as <code>rule(tokens)</code> after the text has been tokenized.</p> </li> <li> <code>special_cases</code>             (<code>Optional[Collection[str]]</code>, default:                 <code>None</code> )         \u2013          <p>special cases to be added to the tokenizer via <code>Spacy</code>'s <code>add_special_case</code> method</p> </li> <li> <code>n_cpus</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>number of CPUs to used during the tokenization process</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def __init__(\n    self,\n    tok_func: Callable = SpacyTokenizer,\n    lang: str = \"en\",\n    pre_rules: Optional[ListRules] = None,\n    post_rules: Optional[ListRules] = None,\n    special_cases: Optional[Collection[str]] = None,\n    n_cpus: Optional[int] = None,\n):\n    self.tok_func, self.lang, self.special_cases = tok_func, lang, special_cases\n    self.pre_rules = ifnone(pre_rules, defaults.text_pre_rules)\n    self.post_rules = ifnone(post_rules, defaults.text_post_rules)\n    self.special_cases = (\n        special_cases if special_cases is not None else defaults.text_spec_tok\n    )\n    self.n_cpus = ifnone(n_cpus, defaults.cpus)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text","title":"process_text","text":"<pre><code>process_text(t, tok)\n</code></pre> <p>Process and tokenize one text <code>t</code> with tokenizer <code>tok</code>.</p> <p>Parameters:</p> <ul> <li> <code>t</code>             (<code>str</code>)         \u2013          <p>text to be processed and tokenized</p> </li> <li> <code>tok</code>             (<code>BaseTokenizer</code>)         \u2013          <p>Instance of <code>BaseTokenizer</code>. See <code>pytorch_widedeep.utils.fastai_transforms.BaseTokenizer</code></p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>         \u2013          <p>List of tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n    r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n    Parameters\n    ----------\n    t: str\n        text to be processed and tokenized\n    tok: ``BaseTokenizer``\n        Instance of `BaseTokenizer`. See\n        `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    for rule in self.pre_rules:\n        t = rule(t)\n    toks = tok.tokenizer(t)\n    for rule in self.post_rules:\n        toks = rule(toks)\n    return toks\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all","title":"process_all","text":"<pre><code>process_all(texts)\n</code></pre> <p>Process a list of texts. Parallel execution of <code>process_text</code>.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tok = Tokenizer()\n&gt;&gt;&gt; tok.process_all(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p> NOTE: Note the token <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> docs.</p> <p>Returns:</p> <ul> <li> <code>List[List[str]]</code>         \u2013          <p>List containing lists of tokens. One list per \"document\"</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n    r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tok = Tokenizer()\n    &gt;&gt;&gt; tok.process_all(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    :information_source: **NOTE**:\n    Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n    next word begins with a capital in the original text. For more\n    details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    List[List[str]]\n        List containing lists of tokens. One list per \"_document_\"\n\n    \"\"\"\n\n    if self.n_cpus &lt;= 1:\n        return self._process_all_1(texts)\n    with ProcessPoolExecutor(self.n_cpus) as e:\n        return sum(\n            e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []\n        )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab","title":"Vocab","text":"<pre><code>Vocab(max_vocab, min_freq, pad_idx=None)\n</code></pre> <p>Contains the correspondence between numbers and tokens.</p> <p>Parameters:</p> <ul> <li> <code>max_vocab</code>             (<code>int</code>)         \u2013          <p>maximum vocabulary size</p> </li> <li> <code>min_freq</code>             (<code>int</code>)         \u2013          <p>minimum frequency for a token to be considereds</p> </li> <li> <code>pad_idx</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index for the unknown token ('xxunk') and defaults to 1 for the padding token ('xxpad').</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>itos</code>             (<code>Collection</code>)         \u2013          <p><code>index to str</code>. Collection of strings that are the tokens of the vocabulary</p> </li> <li> <code>stoi</code>             (<code>defaultdict</code>)         \u2013          <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and their corresponding index</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def __init__(\n    self,\n    max_vocab: int,\n    min_freq: int,\n    pad_idx: Optional[int] = None,\n):\n    self.max_vocab = max_vocab\n    self.min_freq = min_freq\n    self.pad_idx = pad_idx\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.create","title":"create","text":"<pre><code>create(tokens)\n</code></pre> <p>Create a vocabulary object from a set of tokens.</p> <p>Parameters:</p> <ul> <li> <code>tokens</code>             (<code>Tokens</code>)         \u2013          <p>Custom type: <code>Collection[Collection[str]]</code>  see <code>pytorch_widedeep.wdtypes</code>. Collection of collection of strings (e.g. list of tokenized sentences)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n&gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n&gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n[10, 11, 9, 12]\n&gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n'machine learning is great'\n</code></pre> <p> NOTE: Note the many special tokens that <code>fastai</code>'s' tokenizer adds. These are particularly useful when building Language models and/or in classification/Regression tasks. Please see the <code>fastai</code> docs.</p> <p>Returns:</p> <ul> <li> <code>Vocab</code>         \u2013          <p>An instance of a <code>Vocab</code> object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def create(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    r\"\"\"Create a vocabulary object from a set of tokens.\n\n    Parameters\n    ----------\n    tokens: Tokens\n        Custom type: ``Collection[Collection[str]]``  see\n        `pytorch_widedeep.wdtypes`. Collection of collection of\n        strings (e.g. list of tokenized sentences)\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n    &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n    &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n    [10, 11, 9, 12]\n    &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n    'machine learning is great'\n\n    :information_source: **NOTE**:\n    Note the many special tokens that ``fastai``'s' tokenizer adds. These\n    are particularly useful when building Language models and/or in\n    classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    Vocab\n        An instance of a `Vocab` object\n    \"\"\"\n\n    freq = Counter(p for o in tokens for p in o)\n    itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n    for o in reversed(defaults.text_spec_tok):\n        if o in itos:\n            itos.remove(o)\n        itos.insert(0, o)\n\n    if self.pad_idx is not None and self.pad_idx != 1:\n        itos.remove(PAD)\n        itos.insert(self.pad_idx, PAD)\n        # get the new 'xxunk' index\n        xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n    else:\n        xxunk_idx = 0\n\n    itos = itos[: self.max_vocab]\n    if (\n        len(itos) &lt; self.max_vocab\n    ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n        while len(itos) % 8 != 0:\n            itos.append(\"xxfake\")\n\n    self.itos = itos\n    self.stoi = defaultdict(\n        lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n    )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.fit","title":"fit","text":"<pre><code>fit(tokens)\n</code></pre> <p>Calls the <code>create</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a fit method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def fit(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    \"\"\"\n    Calls the `create` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including a fit method\n    \"\"\"\n    return self.create(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize","title":"numericalize","text":"<pre><code>numericalize(t)\n</code></pre> <p>Convert a list of tokens <code>t</code> to their ids.</p> <p>Returns:</p> <ul> <li> <code>List[int]</code>         \u2013          <p>List of 'numericalsed' tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def numericalize(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"Convert a list of tokens ``t`` to their ids.\n\n    Returns\n    -------\n    List[int]\n        List of '_numericalsed_' tokens\n    \"\"\"\n    return [self.stoi[w] for w in t]\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.transform","title":"transform","text":"<pre><code>transform(t)\n</code></pre> <p>Calls the <code>numericalize</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def transform(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"\n    Calls the `numericalize` method. I simply want to honor fast ai naming,\n    but for consistency with the rest of the library I am including a\n    transform method\n    \"\"\"\n    return self.numericalize(t)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.textify","title":"textify","text":"<pre><code>textify(nums, sep=' ')\n</code></pre> <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p> <p>Returns:</p> <ul> <li> <code>List[str]</code>         \u2013          <p>List of tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n    \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    return (\n        sep.join([self.itos[i] for i in nums])\n        if sep is not None\n        else [self.itos[i] for i in nums]\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(nums, sep=' ')\n</code></pre> <p>Calls the <code>textify</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including an inverse_transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def inverse_transform(\n    self, nums: Collection[int], sep=\" \"\n) -&gt; Union[str, List[str]]:\n    \"\"\"\n    Calls the `textify` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including an\n    inverse_transform method\n    \"\"\"\n    # I simply want to honor fast ai naming, but for consistency with the\n    # rest of the library I am including an inverse_transform method\n    return self.textify(nums, sep)\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html","title":"Image utils","text":"<p>SimplePreprocessor and AspectAwarePreprocessor are directly taked from the great series of Books `Deep Learning for Computer Vision by Adrian. Therefore, all credit for the code in the <code>image_utils</code> module goes to Adrian Rosebrock.</p>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor","title":"AspectAwarePreprocessor","text":"<pre><code>AspectAwarePreprocessor(\n    width, height, inter=cv2.INTER_AREA\n)\n</code></pre> <p>Class to resize an image to a certain width and height taking into account the image aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>width</code>             (<code>int</code>)         \u2013          <p>output width</p> </li> <li> <code>height</code>             (<code>int</code>)         \u2013          <p>output height</p> </li> <li> <code>inter</code>         \u2013          <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n    self.width = width\n    self.height = height\n    self.inter = inter\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image taking into account the image aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>image</code>             (<code>ndarray</code>)         \u2013          <p>Input image to be resized</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import cv2\n&gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n&gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n&gt;&gt;&gt; img.shape\n(694, 890, 3)\n&gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n&gt;&gt;&gt; resized_img = app.preprocess(img)\n&gt;&gt;&gt; resized_img.shape\n(224, 224, 3)\n</code></pre> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>Resized image according to its original image aspect ratio</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Examples\n    --------\n    &gt;&gt;&gt; import cv2\n    &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n    &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n    &gt;&gt;&gt; img.shape\n    (694, 890, 3)\n    &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n    &gt;&gt;&gt; resized_img = app.preprocess(img)\n    &gt;&gt;&gt; resized_img.shape\n    (224, 224, 3)\n\n    Returns\n    -------\n    np.ndarray\n        Resized image according to its original image aspect ratio\n    \"\"\"\n    (h, w) = image.shape[:2]\n    dW = 0\n    dH = 0\n\n    if w &lt; h:\n        image = imutils.resize(image, width=self.width, inter=self.inter)\n        dH = int((image.shape[0] - self.height) / 2.0)\n    else:\n        image = imutils.resize(image, height=self.height, inter=self.inter)\n        dW = int((image.shape[1] - self.width) / 2.0)\n\n    (h, w) = image.shape[:2]\n    image = image[dH : h - dH, dW : w - dW]\n\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor","title":"SimplePreprocessor","text":"<pre><code>SimplePreprocessor(width, height, inter=cv2.INTER_AREA)\n</code></pre> <p>Class to resize an image to a certain width and height</p> <p>Parameters:</p> <ul> <li> <code>width</code>             (<code>int</code>)         \u2013          <p>output width</p> </li> <li> <code>height</code>             (<code>int</code>)         \u2013          <p>output height</p> </li> <li> <code>inter</code>         \u2013          <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n    self.width = width\n    self.height = height\n    self.inter = inter\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image</p> <p>Parameters:</p> <ul> <li> <code>image</code>             (<code>ndarray</code>)         \u2013          <p>Input image to be resized</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>Resized image</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Returns\n    -------\n    np.ndarray\n        Resized image\n\n    \"\"\"\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html","title":"Text utils","text":"<p>Collection of helper function that facilitate processing text.</p>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.simple_preprocess","title":"simple_preprocess","text":"<pre><code>simple_preprocess(\n    doc, lower=False, deacc=False, min_len=2, max_len=15\n)\n</code></pre> <p>This is <code>Gensim</code>'s <code>simple_preprocess</code> with a <code>lower</code> param to indicate wether or not to lower case all the token in the doc</p> <p>For more information see: <code>Gensim</code> utils module</p> <p>Parameters:</p> <ul> <li> <code>doc</code>             (<code>str</code>)         \u2013          <p>Input document.</p> </li> <li> <code>lower</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Lower case tokens in the input doc</p> </li> <li> <code>deacc</code>             (<code>bool</code>, default:                 <code>False</code> )         \u2013          <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p> </li> <li> <code>min_len</code>             (<code>int</code>, default:                 <code>2</code> )         \u2013          <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p> </li> <li> <code>max_len</code>             (<code>int</code>, default:                 <code>15</code> )         \u2013          <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n&gt;&gt;&gt; simple_preprocess('Machine learning is great')\n['Machine', 'learning', 'is', 'great']\n</code></pre> <p>Returns:</p> <ul> <li> <code>List[str]</code>         \u2013          <p>List with the processed tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def simple_preprocess(\n    doc: str,\n    lower: bool = False,\n    deacc: bool = False,\n    min_len: int = 2,\n    max_len: int = 15,\n) -&gt; List[str]:\n    r\"\"\"\n    This is `Gensim`'s `simple_preprocess` with a `lower` param to\n    indicate wether or not to lower case all the token in the doc\n\n    For more information see: `Gensim` [utils module](https://radimrehurek.com/gensim/utils.html)\n\n    Parameters\n    ----------\n    doc: str\n        Input document.\n    lower: bool, default = False\n        Lower case tokens in the input doc\n    deacc: bool, default = False\n        Remove accent marks from tokens using `Gensim`'s `deaccent`\n    min_len: int, default = 2\n        Minimum length of token (inclusive). Shorter tokens are discarded.\n    max_len: int, default = 15\n        Maximum length of token in result (inclusive). Longer tokens are discarded.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n    &gt;&gt;&gt; simple_preprocess('Machine learning is great')\n    ['Machine', 'learning', 'is', 'great']\n\n    Returns\n    -------\n    List[str]\n        List with the processed tokens\n    \"\"\"\n    tokens = [\n        token\n        for token in tokenize(doc, lower=lower, deacc=deacc, errors=\"ignore\")\n        if min_len &lt;= len(token) &lt;= max_len and not token.startswith(\"_\")\n    ]\n    return tokens\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.get_texts","title":"get_texts","text":"<pre><code>get_texts(texts, already_processed=False, n_cpus=None)\n</code></pre> <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a series of very convenients things during the tokenization process</p> <p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p> <p>Parameters:</p> <ul> <li> <code>texts</code>             (<code>List[str]</code>)         \u2013          <p>List of str with the texts (or documents). One str per document</p> </li> <li> <code>already_processed</code>             (<code>Optional[bool]</code>, default:                 <code>False</code> )         \u2013          <p>Boolean indicating if the text is already processed and we simply want to tokenize it. This parameter is thought for those cases where the input sequences might not be text (but IDs, or anything else) and we just want to tokenize it</p> </li> <li> <code>n_cpus</code>             (<code>Optional[int]</code>, default:                 <code>None</code> )         \u2013          <p>number of CPUs to used during the tokenization process</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; get_texts(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p>Returns:</p> <ul> <li> <code>List[List[str]]</code>         \u2013          <p>List of lists, one list per 'document' containing its corresponding tokens</p> </li> <li> <code>information_source: **NOTE**:</code>         \u2013          </li> <li> <code>`get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</code>         \u2013          </li> <li> <code>Such tokenizer uses a series of convenient processing steps, including</code>         \u2013          </li> <li> <code>the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</code>         \u2013          </li> <li> <code>indicate the next word begins with a capital in the original text. For more</code>         \u2013          </li> <li> <code>details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</code>         \u2013          </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def get_texts(\n    texts: List[str],\n    already_processed: Optional[bool] = False,\n    n_cpus: Optional[int] = None,\n) -&gt; List[List[str]]:\n    r\"\"\"Tokenization using `Fastai`'s `Tokenizer` because it does a\n    series of very convenients things during the tokenization process\n\n    See `pytorch_widedeep.utils.fastai_utils.Tokenizer`\n\n    Parameters\n    ----------\n    texts: List\n        List of str with the texts (or documents). One str per document\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the text is already processed and we simply want\n        to tokenize it. This parameter is thought for those cases where the\n        input sequences might not be text (but IDs, or anything else) and we\n        just want to tokenize it\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; get_texts(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    Returns\n    -------\n    List[List[str]]\n        List of lists, one list per '_document_' containing its corresponding tokens\n\n    :information_source: **NOTE**:\n    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.\n    Such tokenizer uses a series of convenient processing steps, including\n    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to\n    indicate the next word begins with a capital in the original text. For more\n    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)\n    \"\"\"\n\n    num_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    if not already_processed:\n        processed_texts = [\" \".join(simple_preprocess(t)) for t in texts]\n    else:\n        processed_texts = texts\n    tok = Tokenizer(n_cpus=num_cpus).process_all(processed_texts)\n    return tok\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.pad_sequences","title":"pad_sequences","text":"<pre><code>pad_sequences(seq, maxlen, pad_first=True, pad_idx=1)\n</code></pre> <p>Given a List of tokenized and <code>numericalised</code> sequences it will return padded sequences according to the input parameters.</p> <p>Parameters:</p> <ul> <li> <code>seq</code>             (<code>List[int]</code>)         \u2013          <p>List of int with the <code>numericalised</code> tokens</p> </li> <li> <code>maxlen</code>             (<code>int</code>)         \u2013          <p>Maximum length of the padded sequences</p> </li> <li> <code>pad_first</code>             (<code>bool</code>, default:                 <code>True</code> )         \u2013          <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n&gt;&gt;&gt; seq = [1,2,3]\n&gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\narray([0, 0, 1, 2, 3], dtype=int32)\n</code></pre> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>numpy array with the padded sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def pad_sequences(\n    seq: List[int], maxlen: int, pad_first: bool = True, pad_idx: int = 1\n) -&gt; np.ndarray:\n    r\"\"\"\n    Given a List of tokenized and `numericalised` sequences it will return\n    padded sequences according to the input parameters.\n\n    Parameters\n    ----------\n    seq: List\n        List of int with the `numericalised` tokens\n    maxlen: int\n        Maximum length of the padded sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n    &gt;&gt;&gt; seq = [1,2,3]\n    &gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\n    array([0, 0, 1, 2, 3], dtype=int32)\n\n    Returns\n    -------\n    np.ndarray\n        numpy array with the padded sequences\n    \"\"\"\n    if len(seq) == 0:\n        return np.zeros(maxlen, dtype=\"int32\") + pad_idx\n    elif len(seq) &gt;= maxlen:\n        res = np.array(seq[-maxlen:]).astype(\"int32\")\n        return res\n    else:\n        res = np.zeros(maxlen, dtype=\"int32\") + pad_idx\n        if pad_first:\n            res[-len(seq) :] = seq\n        else:\n            res[: len(seq) :] = seq\n        return res\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.build_embeddings_matrix","title":"build_embeddings_matrix","text":"<pre><code>build_embeddings_matrix(\n    vocab, word_vectors_path, min_freq, verbose=1\n)\n</code></pre> <p>Build the embedding matrix using pretrained word vectors.</p> <p>Returns pretrained word embeddings. If a word in our vocabulary is not among the pretrained embeddings it will be assigned the mean pretrained word-embeddings vector</p> <p>Parameters:</p> <ul> <li> <code>vocab</code>             (<code>Union[Vocab, ChunkVocab]</code>)         \u2013          <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p> </li> <li> <code>word_vectors_path</code>             (<code>str</code>)         \u2013          <p>path to the pretrained word embeddings</p> </li> <li> <code>min_freq</code>             (<code>int</code>)         \u2013          <p>minimum frequency required for a word to be in the vocabulary</p> </li> <li> <code>verbose</code>             (<code>int</code>, default:                 <code>1</code> )         \u2013          <p>level of verbosity. Set to 0 for no verbosity</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>         \u2013          <p>Pretrained word embeddings</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def build_embeddings_matrix(\n    vocab: Union[Vocab, ChunkVocab],\n    word_vectors_path: str,\n    min_freq: int,\n    verbose: int = 1,\n) -&gt; np.ndarray:  # pragma: no cover\n    r\"\"\"Build the embedding matrix using pretrained word vectors.\n\n    Returns pretrained word embeddings. If a word in our vocabulary is not\n    among the pretrained embeddings it will be assigned the mean pretrained\n    word-embeddings vector\n\n    Parameters\n    ----------\n    vocab: Vocab\n        see `pytorch_widedeep.utils.fastai_utils.Vocab`\n    word_vectors_path: str\n        path to the pretrained word embeddings\n    min_freq: int\n        minimum frequency required for a word to be in the vocabulary\n    verbose: int,  default=1\n        level of verbosity. Set to 0 for no verbosity\n\n    Returns\n    -------\n    np.ndarray\n        Pretrained word embeddings\n    \"\"\"\n    if not os.path.isfile(word_vectors_path):\n        raise FileNotFoundError(\"{} not found\".format(word_vectors_path))\n    if verbose:\n        print(\"Indexing word vectors...\")\n\n    embeddings_index = {}\n    f = open(word_vectors_path)\n    for line in f:\n        values = line.split()\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype=\"float32\")\n        embeddings_index[word] = coefs\n    f.close()\n\n    if verbose:\n        print(\"Loaded {} word vectors\".format(len(embeddings_index)))\n        print(\"Preparing embeddings matrix...\")\n\n    mean_word_vector = np.mean(list(embeddings_index.values()), axis=0)  # type: ignore[arg-type]\n    embedding_dim = len(list(embeddings_index.values())[0])\n    num_words = len(vocab.itos)\n    embedding_matrix = np.zeros((num_words, embedding_dim))\n    found_words = 0\n    for i, word in enumerate(vocab.itos):\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n            found_words += 1\n        else:\n            embedding_matrix[i] = mean_word_vector\n\n    if verbose:\n        print(\n            \"{} words in the vocabulary had {} vectors and appear more than {} times\".format(\n                found_words, word_vectors_path, min_freq\n            )\n        )\n\n    return embedding_matrix.astype(\"float32\")\n</code></pre>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"index.html","title":"Home","text":""},{"location":"index.html#pytorch-widedeep","title":"pytorch-widedeep","text":"<p>A flexible package for multimodal-deep-learning to combine tabular data with text and images using Wide and Deep models in Pytorch</p> <p>Documentation: https://pytorch-widedeep.readthedocs.io</p> <p>Companion posts and tutorials: infinitoml</p> <p>Experiments and comparison with <code>LightGBM</code>: TabularDL vs LightGBM</p> <p>Slack: if you want to contribute or just want to chat with us, join slack</p> <p>The content of this document is organized as follows:</p> <ul> <li>pytorch-widedeep<ul> <li>Introduction</li> <li>Architectures</li> <li>The <code>deeptabular</code> component</li> <li>Text and Images</li> <li>Acknowledgments</li> <li>License</li> <li>Cite<ul> <li>BibTex</li> <li>APA</li> </ul> </li> </ul> </li> </ul>"},{"location":"index.html#introduction","title":"Introduction","text":"<p><code>pytorch-widedeep</code> is based on Google's Wide and Deep Algorithm, adjusted for multi-modal datasets.</p> <p>In general terms, <code>pytorch-widedeep</code> is a package to use deep learning with tabular data. In particular, is intended to facilitate the combination of text and images with corresponding tabular data using wide and deep models. With that in mind there are a number of architectures that can be implemented with the library. The main components of those architectures are shown in the Figure below:</p> <p> </p> <p>In math terms, and following the notation in the paper, the expression for the architecture without a <code>deephead</code> component can be formulated as:</p> <p> </p> <p>Where \u03c3 is the sigmoid function, 'W' are the weight matrices applied to the wide model and to the final activations of the deep models, 'a' are these final activations, \u03c6(x) are the cross product transformations of the original features 'x', and , and 'b' is the bias term. In case you are wondering what are \"cross product transformations\", here is a quote taken directly from the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>It is perfectly possible to use custom models (and not necessarily those in the library) as long as the the custom models have a property called <code>output_dim</code> with the size of the last layer of activations, so that <code>WideDeep</code> can be constructed. Examples on how to use custom components can be found in the Examples folder and the section below.</p>"},{"location":"index.html#architectures","title":"Architectures","text":"<p>The <code>pytorch-widedeep</code> library offers a number of different architectures. In this section we will show some of them in their simplest form (i.e. with default param values in most cases) with their corresponding code snippets. Note that all the snippets below shoud run locally. For a more detailed explanation of the different components and their parameters, please refer to the documentation.</p> <p>For the examples below we will be using a toy dataset generated as follows:</p> <pre><code>import os\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom faker import Faker\n\n\ndef create_and_save_random_image(image_number, size=(32, 32)):\n\n    if not os.path.exists(\"images\"):\n        os.makedirs(\"images\")\n\n    array = np.random.randint(0, 256, (size[0], size[1], 3), dtype=np.uint8)\n\n    image = Image.fromarray(array)\n\n    image_name = f\"image_{image_number}.png\"\n    image.save(os.path.join(\"images\", image_name))\n\n    return image_name\n\n\nfake = Faker()\n\ncities = [\"New York\", \"Los Angeles\", \"Chicago\", \"Houston\"]\nnames = [\"Alice\", \"Bob\", \"Charlie\", \"David\", \"Eva\"]\n\ndata = {\n    \"city\": [random.choice(cities) for _ in range(100)],\n    \"name\": [random.choice(names) for _ in range(100)],\n    \"age\": [random.uniform(18, 70) for _ in range(100)],\n    \"height\": [random.uniform(150, 200) for _ in range(100)],\n    \"sentence\": [fake.sentence() for _ in range(100)],\n    \"other_sentence\": [fake.sentence() for _ in range(100)],\n    \"image_name\": [create_and_save_random_image(i) for i in range(100)],\n    \"target\": [random.choice([0, 1]) for _ in range(100)],\n}\n\ndf = pd.DataFrame(data)\n</code></pre> <p>This will create a 100 rows dataframe and a dir in your local folder, called <code>images</code> with 100 random images (or images with just noise).</p> <p>Perhaps the simplest architecture would be just one component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> or <code>deepimage</code> on their own, which is also possible, but let's start the examples with a standard Wide and Deep architecture. From there, how to build a model comprised only of one component will be straightforward.</p> <p>Note that the examples shown below would be almost identical using any of the models available in the library. For example, <code>TabMlp</code> can be replaced by <code>TabResnet</code>, <code>TabNet</code>, <code>TabTransformer</code>, etc. Similarly, <code>BasicRNN</code> can be replaced by <code>AttentiveRNN</code>, <code>StackedAttentiveRNN</code>, or <code>HFModel</code> with their corresponding parameters and preprocessor in the case of the Hugging Face models.</p> <p>1. Wide and Tabular component (aka deeptabular)</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Wide\nwide_cols = [\"city\"]\ncrossed_cols = [(\"city\", \"name\")]\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\nwide = Wide(input_dim=np.unique(X_wide).shape[0])\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# WideDeep\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>2. Tabular and Text data</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>3. Tabular and text with a FC head on top via the <code>head_hidden_dims</code> param   in <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn, head_hidden_dims=[32, 16])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>4. Tabular and multiple text columns that are passed directly to   <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=[rnn_1, rnn_2])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>5. Tabular data and multiple text columns that are fused via a the library's   <code>ModelFuser</code> class</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\n\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\nmodels_fuser = ModelFuser(models=[rnn_1, rnn_2], fusion_method=\"mult\")\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=models_fuser)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>6. Tabular and multiple text columns, with an image column. The text columns   are fused via the library's <code>ModelFuser</code> and then all fused via the   deephead paramenter in <code>WideDeep</code> which is a custom <code>ModelFuser</code> coded by   the user</p> <p>This is perhaps the less elegant solution as it involves a custom component by the user and slicing the 'incoming' tensor. In the future, we will include a <code>TextAndImageModelFuser</code> to make this process more straightforward. Still, is not really complicated and it is a good example of how to use custom components in <code>pytorch-widedeep</code>.</p> <p>Note that the only requirement for the custom component is that it has a property called <code>output_dim</code> that returns the size of the last layer of activations. In other words, it does not need to inherit from <code>BaseWDModelComponent</code>. This base class simply checks the existence of such property and avoids some typing errors internally.</p> <p> </p> <pre><code>import torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[16, 8],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nmodels_fuser = ModelFuser(\n    models=[rnn_1, rnn_2],\n    fusion_method=\"mult\",\n)\n\n# Image\nimage_preprocessor = ImagePreprocessor(img_col=\"image_name\", img_path=\"images\")\nX_img = image_preprocessor.fit_transform(df)\nvision = Vision(pretrained_model_setup=\"resnet18\", head_hidden_dims=[16, 8])\n\n# deephead (custom model fuser)\nclass MyModelFuser(BaseWDModelComponent):\n    \"\"\"\n    Simply a Linear + Relu sequence on top of the text + images followed by a\n    Linear -&gt; Relu -&gt; Linear for the concatenation of tabular slice of the\n    tensor and the output of the text and image sequential model\n    \"\"\"\n    def __init__(\n        self,\n        tab_incoming_dim: int,\n        text_incoming_dim: int,\n        image_incoming_dim: int,\n        output_units: int,\n    ):\n\n        super(MyModelFuser, self).__init__()\n\n        self.tab_incoming_dim = tab_incoming_dim\n        self.text_incoming_dim = text_incoming_dim\n        self.image_incoming_dim = image_incoming_dim\n        self.output_units = output_units\n        self.text_and_image_fuser = torch.nn.Sequential(\n            torch.nn.Linear(text_incoming_dim + image_incoming_dim, output_units),\n            torch.nn.ReLU(),\n        )\n        self.out = torch.nn.Sequential(\n            torch.nn.Linear(output_units + tab_incoming_dim, output_units * 4),\n            torch.nn.ReLU(),\n            torch.nn.Linear(output_units * 4, output_units),\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        tab_slice = slice(0, self.tab_incoming_dim)\n        text_slice = slice(\n            self.tab_incoming_dim, self.tab_incoming_dim + self.text_incoming_dim\n        )\n        image_slice = slice(\n            self.tab_incoming_dim + self.text_incoming_dim,\n            self.tab_incoming_dim + self.text_incoming_dim + self.image_incoming_dim,\n        )\n        X_tab = X[:, tab_slice]\n        X_text = X[:, text_slice]\n        X_img = X[:, image_slice]\n        X_text_and_image = self.text_and_image_fuser(torch.cat([X_text, X_img], dim=1))\n        return self.out(torch.cat([X_tab, X_text_and_image], dim=1))\n\n    @property\n    def output_dim(self):\n        return self.output_units\n\ndeephead = MyModelFuser(\n    tab_incoming_dim=tab_mlp.output_dim,\n    text_incoming_dim=models_fuser.output_dim,\n    image_incoming_dim=vision.output_dim,\n    output_units=8,\n)\n\n# WideDeep\nmodel = WideDeep(\n    deeptabular=tab_mlp,\n    deeptext=models_fuser,\n    deepimage=vision,\n    deephead=deephead,\n)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    X_img=X_img,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>7. Tabular with a multi-target loss</p> <p>This one is \"a bonus\" to illustrate the use of multi-target losses, more than actually a different architecture.</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# let's add a second target to the dataframe\ndf[\"target2\"] = [random.choice([0, 1]) for _ in range(100)]\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# 'pred_dim=2' because we have two binary targets. For other types of targets,\n#  please, see the documentation\nmodel = WideDeep(deeptabular=tab_mlp, pred_dim=2).\n\nloss = MultiTargetClassificationLoss(binary_config=[0, 1], reduction=\"mean\")\n\n# When a multi-target loss is used, 'custom_loss_function' must not be None.\n# See the docs\ntrainer = Trainer(model, objective=\"multitarget\", custom_loss_function=loss)\n\ntrainer.fit(\n    X_tab=X_tab,\n    target=df[[\"target\", \"target2\"]].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre>"},{"location":"index.html#the-deeptabular-component","title":"The <code>deeptabular</code> component","text":"<p>It is important to emphasize again that each individual component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code>, can be used independently and in isolation. For example, one could use only <code>wide</code>, which is in simply a linear model. In fact, one of the most interesting functionalities in<code>pytorch-widedeep</code> would be the use of the <code>deeptabular</code> component on its own, i.e. what one might normally refer as Deep Learning for Tabular Data. Currently, <code>pytorch-widedeep</code> offers the following different models for that component:</p> <ol> <li>Wide: a simple linear model where the nonlinearities are captured via cross-product transformations, as explained before.</li> <li>TabMlp: a simple MLP that receives embeddings representing the categorical features, concatenated with the continuous features, which can also be embedded.</li> <li>TabResnet: similar to the previous model but the embeddings are passed through a series of ResNet blocks built with dense layers.</li> <li>TabNet: details on TabNet can be found in TabNet: Attentive Interpretable Tabular Learning</li> </ol> <p>Two simpler attention based models that we call:</p> <ol> <li>ContextAttentionMLP: MLP with at attention mechanism \"on top\" that is based on     Hierarchical Attention Networks for Document Classification</li> <li>SelfAttentionMLP: MLP with an attention mechanism that is a simplified     version of a transformer block that we refer as \"query-key self-attention\".</li> </ol> <p>The <code>Tabformer</code> family, i.e. Transformers for Tabular data:</p> <ol> <li>TabTransformer: details on the TabTransformer can be found in TabTransformer: Tabular Data Modeling Using Contextual Embeddings.</li> <li>SAINT: Details on SAINT can be found in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training.</li> <li>FT-Transformer: details on the FT-Transformer can be found in Revisiting Deep Learning Models for Tabular Data.</li> <li>TabFastFormer: adaptation of the FastFormer for tabular data. Details on the Fasformer can be found in FastFormers: Highly Efficient Transformer Models for Natural Language Understanding</li> <li>TabPerceiver: adaptation of the Perceiver for tabular data. Details on the Perceiver can be found in Perceiver: General Perception with Iterative Attention</li> </ol> <p>And probabilistic DL models for tabular data based on Weight Uncertainty in Neural Networks:</p> <ol> <li>BayesianWide: Probabilistic adaptation of the <code>Wide</code> model.</li> <li>BayesianTabMlp: Probabilistic adaptation of the <code>TabMlp</code> model</li> </ol> <p>Note that while there are scientific publications for the TabTransformer, SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own adaptation of those algorithms for tabular data.</p> <p>In addition, Self-Supervised pre-training can be used for all <code>deeptabular</code> models, with the exception of the <code>TabPerceiver</code>. Self-Supervised pre-training can be used via two methods or routines which we refer as: encoder-decoder method and constrastive-denoising method. Please, see the documentation and the examples for details on this functionality, and all other options in the library.</p>"},{"location":"index.html#text-and-images","title":"Text and Images","text":"<p>For the text component, <code>deeptext</code>, the library offers the following models:</p> <ol> <li>BasicRNN: a simple RNN 2. AttentiveRNN: a RNN with an attention mechanism based on the Hierarchical Attention Networks for DocumentClassification</li> <li>StackedAttentiveRNN: a stack of AttentiveRNNs</li> <li>HFModel: a wrapper around Hugging Face Transfomer-based models. At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks. If there is demand for other models, they will be included in the future.</li> </ol> <p>For the image component, <code>deepimage</code>, the library supports models from the following families: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3',  'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'.  These are  offered via <code>torchvision</code> and wrapped up in the <code>Vision</code> class.</p>"},{"location":"index.html#acknowledgments","title":"Acknowledgments","text":"<p>This library takes from a series of other libraries, so I think it is just fair to mention them here in the README (specific mentions are also included in the code).</p> <p>The <code>Callbacks</code> and <code>Initializers</code> structure and code is inspired by the <code>torchsample</code> library, which in itself partially inspired by <code>Keras</code>.</p> <p>The <code>TextProcessor</code> class in this library uses the <code>fastai</code>'s <code>Tokenizer</code> and <code>Vocab</code>. The code at <code>utils.fastai_transforms</code> is a minor adaptation of their code so it functions within this library. To my experience their <code>Tokenizer</code> is the best in class.</p> <p>The <code>ImageProcessor</code> class in this library uses code from the fantastic Deep Learning for Computer Vision (DL4CV) book by Adrian Rosebrock.</p>"},{"location":"index.html#license","title":"License","text":"<p>This work is dual-licensed under Apache 2.0 and MIT (or any later version). You can choose between one of them if you use this work.</p> <p><code>SPDX-License-Identifier: Apache-2.0 AND MIT</code></p>"},{"location":"index.html#cite","title":"Cite","text":""},{"location":"index.html#bibtex","title":"BibTex","text":"<pre><code>@article{Zaurin_pytorch-widedeep_A_flexible_2023,\nauthor = {Zaurin, Javier Rodriguez and Mulinka, Pavol},\ndoi = {10.21105/joss.05027},\njournal = {Journal of Open Source Software},\nmonth = jun,\nnumber = {86},\npages = {5027},\ntitle = {{pytorch-widedeep: A flexible package for multimodal deep learning}},\nurl = {https://joss.theoj.org/papers/10.21105/joss.05027},\nvolume = {8},\nyear = {2023}\n}\n</code></pre>"},{"location":"index.html#apa","title":"APA","text":"<pre><code>Zaurin, J. R., &amp; Mulinka, P. (2023). pytorch-widedeep: A flexible package for\nmultimodal deep learning. Journal of Open Source Software, 8(86), 5027.\nhttps://doi.org/10.21105/joss.05027\n</code></pre>"},{"location":"contributing.html","title":"Contributing","text":"<p>Pytorch-widedeep is being developed and used by many active community members. Your help is very valuable to make it better for everyone.</p> <ul> <li>Check for the Roadmap or Open an issue to report problems or recommend new features and submit a draft pull requests, which will be changed to pull request after intial review</li> <li>Contribute to the tests to make it more reliable.</li> <li>Contribute to the documentation to make it clearer for everyone.</li> <li>Contribute to the examples to share your experience with other users.</li> <li>Join the dicussion on slack</li> </ul>"},{"location":"installation.html","title":"Installation","text":"<p>This section explains how to install <code>pytorch-widedeep</code>.</p> <p>For the latest stable release, execute:</p> <pre><code>pip install pytorch-widedeep\n</code></pre> <p>For the bleeding-edge version, execute:</p> <pre><code>pip install git+https://github.com/jrzaurin/pytorch-widedeep.git\n</code></pre> <p>For developer install</p> <pre><code># Clone the repository\ngit clone https://github.com/jrzaurin/pytorch-widedeep\ncd pytorch-widedeep\n\n# Install in dev mode\npip install -e .\n</code></pre>"},{"location":"installation.html#dependencies","title":"Dependencies","text":"<ul> <li>pandas&gt;=1.3.5</li> <li>numpy&gt;=1.21.6</li> <li>scipy&gt;=1.7.3,&lt;=1.12.0</li> <li>scikit-learn&gt;=1.0.2</li> <li>gensim</li> <li>spacy</li> <li>opencv-contrib-python</li> <li>imutils</li> <li>tqdm</li> <li>torch &gt;= 2.0.0</li> <li>torchvision &gt;= 0.15.0</li> <li>einops</li> <li>wrapt</li> <li>torchmetrics</li> <li>pyarrow</li> <li>fastparquet&gt;=0.8.1</li> <li>transformers</li> <li>sentence-transformers</li> <li>sentencepiece</li> </ul>"},{"location":"quick_start.html","title":"Quick Start","text":"<p>This is an example of a binary classification with the adult census dataset using a combination of a wide and deep model (in this case a so called <code>deeptabular</code> model) with defaults settings.</p> <pre><code>import numpy as np\nimport torch\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n\n\ndf = load_adult(as_frame=True)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\n# Define the 'column set up'\nwide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital-status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital-gain\",\n    \"capital-loss\",\n    \"native-country\",\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\ntarget = \"income_label\"\ntarget = df_train[target].values\n\n# prepare the data\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df_train)\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]\n)\nX_tab = tab_preprocessor.fit_transform(df_train)\n\n# build the model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# train and validate\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=5,\n    batch_size=256,\n)\n\n# predict on test\nX_wide_te = wide_preprocessor.transform(df_test)\nX_tab_te = tab_preprocessor.transform(df_test)\npreds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)\n\n# Save and load\n\n# Option 1: this will also save training history and lr history if the\n# LRHistory callback is used\ntrainer.save(path=\"model_weights\", save_state_dict=True)\n\n# Option 2: save as any other torch model\ntorch.save(model.state_dict(), \"model_weights/wd_model.pt\")\n\n# From here in advance, Option 1 or 2 are the same. I assume the user has\n# prepared the data and defined the new model components:\n# 1. Build the model\nmodel_new = WideDeep(wide=wide, deeptabular=tab_mlp)\nmodel_new.load_state_dict(torch.load(\"model_weights/wd_model.pt\"))\n\n# 2. Instantiate the trainer\ntrainer_new = Trainer(model_new, objective=\"binary\")\n\n# 3. Either start the fit or directly predict\npreds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab, batch_size=32)\n</code></pre>"},{"location":"examples/01_preprocessors_and_utils.html","title":"01_preprocessors_and_utils","text":"<p>For example</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport pytorch_widedeep as wd\n\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import WidePreprocessor\n</pre> import numpy as np import pandas as pd import pytorch_widedeep as wd  from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import WidePreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre>wide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n</pre> wide_cols = [     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native-country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")] In\u00a0[4]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_wide = wide_preprocessor.transform(new_df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_wide = wide_preprocessor.transform(new_df) In\u00a0[5]: Copied! <pre>X_wide\n</pre> X_wide Out[5]: <pre>array([[  1,  17,  23, ...,  89,  91, 316],\n       [  2,  18,  23, ...,  89,  92, 317],\n       [  3,  18,  24, ...,  89,  93, 318],\n       ...,\n       [  2,  20,  23, ...,  90, 103, 323],\n       [  2,  17,  23, ...,  89, 103, 323],\n       [  2,  21,  29, ...,  90, 115, 324]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[6]: Copied! <pre>X_wide[0]\n</pre> X_wide[0] Out[6]: <pre>array([  1,  17,  23,  32,  47,  89,  91, 316])</pre> In\u00a0[7]: Copied! <pre>wide_preprocessor.inverse_transform(X_wide[:1])\n</pre> wide_preprocessor.inverse_transform(X_wide[:1]) Out[7]: education relationship workclass occupation native-country gender education_occupation native-country_occupation 0 11th Own-child Private Machine-op-inspct United-States Male 11th-Machine-op-inspct United-States-Machine-op-inspct <p>As we can see, <code>wide_preprocessor</code> numerically encodes the <code>wide_cols</code> and the <code>crossed_cols</code>, which can be recovered using the method <code>inverse_transform</code>.</p> In\u00a0[8]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[9]: Copied! <pre># cat_embed_cols = [(column_name, embed_dim), ...]\ncat_embed_cols = [\n    (\"education\", 10),\n    (\"relationship\", 8),\n    (\"workclass\", 10),\n    (\"occupation\", 10),\n    (\"native-country\", 10),\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\n</pre> # cat_embed_cols = [(column_name, embed_dim), ...] cat_embed_cols = [     (\"education\", 10),     (\"relationship\", 8),     (\"workclass\", 10),     (\"occupation\", 10),     (\"native-country\", 10), ] continuous_cols = [\"age\", \"hours-per-week\"] In\u00a0[10]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_deep = deep_preprocessor.transform(new_df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_deep = deep_preprocessor.transform(new_df) In\u00a0[11]: Copied! <pre>X_tab\n</pre> X_tab Out[11]: <pre>array([[ 1.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -9.95128932e-01,  4.00000000e+01],\n       [ 2.00000000e+00,  2.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -4.69415091e-02,  5.00000000e+01],\n       [ 3.00000000e+00,  2.00000000e+00,  2.00000000e+00, ...,\n         1.00000000e+00, -7.76316450e-01,  4.00000000e+01],\n       ...,\n       [ 2.00000000e+00,  4.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00,  1.41180837e+00,  4.00000000e+01],\n       [ 2.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -1.21394141e+00,  2.00000000e+01],\n       [ 2.00000000e+00,  5.00000000e+00,  7.00000000e+00, ...,\n         1.00000000e+00,  9.74183408e-01,  4.00000000e+01]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[12]: Copied! <pre>X_tab[0]\n</pre> X_tab[0] Out[12]: <pre>array([ 1.        ,  1.        ,  1.        ,  1.        ,  1.        ,\n       -0.99512893, 40.        ])</pre> In\u00a0[13]: Copied! <pre>tab_preprocessor.inverse_transform(X_tab[:1])\n</pre> tab_preprocessor.inverse_transform(X_tab[:1]) Out[13]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 25.0 40.0 <p>The <code>TabPreprocessor</code> will have a series of useful attributes that can later be used when instantiating the different Tabular Models, such us for example, the column indexes (used to slice the tensors, internally in the models) or the categorical embeddings set up</p> In\u00a0[14]: Copied! <pre>tab_preprocessor.column_idx\n</pre> tab_preprocessor.column_idx Out[14]: <pre>{'education': 0,\n 'relationship': 1,\n 'workclass': 2,\n 'occupation': 3,\n 'native-country': 4,\n 'age': 5,\n 'hours-per-week': 6}</pre> In\u00a0[15]: Copied! <pre># column name, num unique, embedding dim\ntab_preprocessor.cat_embed_input\n</pre> # column name, num unique, embedding dim tab_preprocessor.cat_embed_input Out[15]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10)]</pre> <p>As I mentioned, there is more one can do, such as for example, quantize (or bucketize) the continuous cols. For this we could use the <code>quantization_setup</code> param. This parameter accepts a number of different inputs and uses <code>pd.cut</code> under the hood to quantize the continuous cols. For more info, please, read the docs. Let's use it here to quantize \"age\" and \"hours-per-week\" in 4 and 5 \"buckets\" respectively</p> In\u00a0[16]: Copied! <pre>quantization_setup = {\n    \"age\": 4,\n    \"hours-per-week\": 5,\n}  # you can also pass a list of floats with the boundaries if you wanted\nquant_tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    quantization_setup=quantization_setup,\n)\nqX_tab = quant_tab_preprocessor.fit_transform(df)\n</pre> quantization_setup = {     \"age\": 4,     \"hours-per-week\": 5, }  # you can also pass a list of floats with the boundaries if you wanted quant_tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     quantization_setup=quantization_setup, ) qX_tab = quant_tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>qX_tab\n</pre> qX_tab Out[17]: <pre>array([[1, 1, 1, ..., 1, 1, 2],\n       [2, 2, 1, ..., 1, 2, 3],\n       [3, 2, 2, ..., 1, 1, 2],\n       ...,\n       [2, 4, 1, ..., 1, 3, 2],\n       [2, 1, 1, ..., 1, 1, 1],\n       [2, 5, 7, ..., 1, 2, 2]])</pre> <p>Note that the continuous columns that have been bucketised into quantiles are treated as any other categorical column</p> In\u00a0[18]: Copied! <pre>quant_tab_preprocessor.cat_embed_input\n</pre> quant_tab_preprocessor.cat_embed_input Out[18]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10),\n ('age', 4, 4),\n ('hours-per-week', 5, 4)]</pre> <p>Where the column 'age' has now 4 categories, which will be encoded using embeddings of 4 dims. Note that, as any other categorical columns, the categorical \"counter\" starts with 1. This is because all incoming values that are lower/higher than the existing lowest/highest value in the train (or already seen) dataset, will be encoded as 0.</p> In\u00a0[19]: Copied! <pre>np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]])\n</pre> np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]]) Out[19]: <pre>array([1, 2, 3, 4])</pre> <p>Finally, if we now wanted to <code>inverse_transform</code> the transformed array into the original dataframe, we could still do it, but the continuous, bucketized columns will be transformed back to the middle of their quantile/bucket range</p> In\u00a0[20]: Copied! <pre>df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab)\n</pre> df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> In\u00a0[21]: Copied! <pre>df.head(2)\n</pre> df.head(2) Out[21]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K In\u00a0[22]: Copied! <pre>df_decoded.head(2)\n</pre> df_decoded.head(2) Out[22]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 26.0885 30.4 1 HS-grad Husband Private Farming-fishing United-States 44.3750 50.0 <p>there is one final comment to make regarding to the <code>inverse_transform</code> functionality. As we mentioned before, the encoding <code>0</code> is reserved for values that fall outside the range covered by the data we used to run the <code>fit</code> method. For example</p> In\u00a0[23]: Copied! <pre>df.age.min(), df.age.max()\n</pre> df.age.min(), df.age.max() Out[23]: <pre>(17, 90)</pre> <p>All future age values outside that range will be encoded as 0 and decoded as <code>NaN</code></p> In\u00a0[24]: Copied! <pre>tmp_df = df.head(1).copy()\ntmp_df.loc[:, \"age\"] = 5\n</pre> tmp_df = df.head(1).copy() tmp_df.loc[:, \"age\"] = 5 In\u00a0[25]: Copied! <pre>tmp_df\n</pre> tmp_df Out[25]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 5 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K In\u00a0[26]: Copied! <pre># quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90\ntmp_qX_tab = quant_tab_preprocessor.transform(tmp_df)\n</pre> # quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90 tmp_qX_tab = quant_tab_preprocessor.transform(tmp_df) In\u00a0[27]: Copied! <pre>tmp_qX_tab\n</pre> tmp_qX_tab Out[27]: <pre>array([[1, 1, 1, 1, 1, 0, 2]])</pre> In\u00a0[28]: Copied! <pre>quant_tab_preprocessor.inverse_transform(tmp_qX_tab)\n</pre> quant_tab_preprocessor.inverse_transform(tmp_qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> Out[28]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States NaN 30.4 In\u00a0[29]: Copied! <pre>from pytorch_widedeep.preprocessing import TextPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TextPreprocessor In\u00a0[30]: Copied! <pre># The airbnb dataset, which you could get from here:\n# http://insideairbnb.com/get-the-data.html, is too big to be included in\n# our datasets module (when including images). Therefore, go there,\n# download it, and use the download_images.py script to get the images\n# and the airbnb_data_processing.py to process the data. We'll find\n# better datasets in the future ;). Note that here we are only using a\n# small sample to illustrate the use, so PLEASE ignore the results, just\n# focus on usage\ndf = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\n</pre> # The airbnb dataset, which you could get from here: # http://insideairbnb.com/get-the-data.html, is too big to be included in # our datasets module (when including images). Therefore, go there, # download it, and use the download_images.py script to get the images # and the airbnb_data_processing.py to process the data. We'll find # better datasets in the future ;). Note that here we are only using a # small sample to illustrate the use, so PLEASE ignore the results, just # focus on usage df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") In\u00a0[31]: Copied! <pre>texts = df.description.tolist()\ntexts[:2]\n</pre> texts = df.description.tolist() texts[:2] Out[31]: <pre>[\"My bright double bedroom with a large window has a relaxed feeling! It comfortably fits one or two and is centrally located just two blocks from Finsbury Park. Enjoy great restaurants in the area and easy access to easy transport tubes, trains and buses. Babies and children of all ages are welcome. Hello Everyone, I'm offering my lovely double bedroom in Finsbury Park area (zone 2) for let in a shared apartment.  You will share the apartment with me and it is fully furnished with a self catering kitchen. Two people can easily sleep well as the room has a queen size bed. I also have a travel cot for a baby for guest with small children.  I will require a deposit up front as a security gesture on both our parts and will be given back to you when you return the keys.  I trust anyone who will be responding to this add would treat my home with care and respect .  Best Wishes  Alina Guest will have access to the self catering kitchen and bathroom. There is the flat is equipped wifi internet,\",\n \"Lots of windows and light.  St Luke's Gardens are at the end of the block, and the river not too far the other way. Ten minutes walk if you go slowly. Buses to everywhere round the corner and shops, restaurants, pubs, the cinema and Waitrose . Bright Chelsea Apartment  This is a bright one bedroom ground floor apartment in an interesting listed building. There is one double bedroom and a living room/kitchen The apartment has a full  bathroom and the kitchen is fully equipped. Two wardrobes are available exclusively for guests and bedside tables and two long drawers. This sunny convenient compact flat is just around the corner from the Waitrose supermarket and all sorts of shops, cinemas, restaurants and pubs.  This is a lovely part of London. There is a fun farmers market in the King's Road at the weekend.  Buses to everywhere are just round the corner, and two underground stations are within ten minutes walk. There is a very nice pub round by St. Luke's gardens, 4 mins slow walk, the \"]</pre> In\u00a0[32]: Copied! <pre>text_preprocessor = TextPreprocessor(text_col=\"description\")\nX_text = text_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_text = text_preprocessor.transform(new_df)\n</pre> text_preprocessor = TextPreprocessor(text_col=\"description\") X_text = text_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_text = text_preprocessor.transform(new_df) <pre>The vocabulary contains 2192 tokens\n</pre> In\u00a0[33]: Copied! <pre>print(X_text[0])\n</pre> print(X_text[0]) <pre>[  29   48   37  367  818   17  910   17  177   15  122  349   53  879\n 1174  126  393   40  911    0   23  228   71  819    9   53   55 1380\n  225   11   18  308   18 1564   10  755    0  942  239   53   55    0\n   11   36 1013  277 1974   70   62   15 1475    9  943    5  251    5\n    0    5    0    5  177   53   37   75   11   10  294  726   32    9\n   42    5   25   12   10   22   12  136  100  145]\n</pre> In\u00a0[34]: Copied! <pre>from pytorch_widedeep.preprocessing import ImagePreprocessor\n</pre> from pytorch_widedeep.preprocessing import ImagePreprocessor In\u00a0[35]: Copied! <pre>image_preprocessor = wd.preprocessing.ImagePreprocessor(\n    img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\"\n)\nX_images = image_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_images = image_preprocessor.transform(new_df)\n</pre> image_preprocessor = wd.preprocessing.ImagePreprocessor(     img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\" ) X_images = image_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_images = image_preprocessor.transform(new_df) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 667.89it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[36]: Copied! <pre>X_images[0].shape\n</pre> X_images[0].shape Out[36]: <pre>(224, 224, 3)</pre>"},{"location":"examples/01_preprocessors_and_utils.html#processors-and-utils","title":"Processors and Utils\u00b6","text":"<p>Description of the main tools and utilities that one needs to prepare the data for a <code>WideDeep</code> model constructor.</p>"},{"location":"examples/01_preprocessors_and_utils.html#the-preprocessing-module","title":"The <code>preprocessing</code>  module\u00b6","text":"<p>There are 4 preprocessors, corresponding to 4 main components of the <code>WideDeep</code> model. These are</p> <ul> <li><code>WidePreprocessor</code></li> <li><code>TabPreprocessor</code></li> <li><code>TextPreprocessor</code></li> <li><code>ImagePreprocessor</code></li> </ul> <p>Behind the scenes, these preprocessors use a series of helper funcions and classes that are in the <code>utils</code> module. If you were interested please go and have a look to the documentation</p>"},{"location":"examples/01_preprocessors_and_utils.html#1-widepreprocessor","title":"1. WidePreprocessor\u00b6","text":"<p>The <code>wide</code> component of the model is a linear model that in principle, could be implemented as a linear layer receiving the result of on one-hot encoding categorical columns. However, this is not memory efficient. Therefore, we implement a liner layer as an Embedding layer plus a bias. I will explain in a bit more detail later.</p> <p>With that in mind, <code>WidePreprocessor</code> simply encodes the categories numerically so that they are the indexes of the lookup table that is an Embedding layer.</p>"},{"location":"examples/01_preprocessors_and_utils.html#2-tabpreprocessor","title":"2. TabPreprocessor\u00b6","text":"<p>The <code>TabPreprocessor</code> has a lot of different functionalities. Let's explore some of them in detail. In its basic use, the <code>TabPreprocessor</code> simply label encodes the categorical columns and normalises the numerical ones (unless otherwised specified).</p>"},{"location":"examples/01_preprocessors_and_utils.html#3-textpreprocessor","title":"3. TextPreprocessor\u00b6","text":"<p>This preprocessor returns the tokenised, padded sequences that will be directly fed to the stack of LSTMs.</p>"},{"location":"examples/01_preprocessors_and_utils.html#4-imagepreprocessor","title":"4. ImagePreprocessor\u00b6","text":"<p><code>ImagePreprocessor</code> simply resizes the images, being aware of the aspect ratio.</p>"},{"location":"examples/02_model_components.html","title":"02_model_components","text":"In\u00a0[1]: Copied! <pre>import torch\nimport pandas as pd\nimport numpy as np\n\nfrom torch import nn\n</pre> import torch import pandas as pd import numpy as np  from torch import nn In\u00a0[2]: Copied! <pre>df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]})\ndf.head()\n</pre> df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]}) df.head() Out[2]: color size 0 r s 1 b n 2 g l <p>one hot encoded, the first observation would be</p> In\u00a0[3]: Copied! <pre>obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\")\n</pre> obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\") <p>if we simply numerically encode (label encode or <code>le</code>) the values:</p> In\u00a0[4]: Copied! <pre>obs_0_le = (np.array([0, 3])).astype(\"int64\")\n</pre> obs_0_le = (np.array([0, 3])).astype(\"int64\") <p>Note that in the functioning implementation of the package we start from 1, saving 0 for padding, i.e. unseen values.</p> <p>Now, let's see if the two implementations are equivalent</p> In\u00a0[5]: Copied! <pre># we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1\nlin = nn.Linear(6, 1)\n</pre> # we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1 lin = nn.Linear(6, 1) In\u00a0[6]: Copied! <pre>emb = nn.Embedding(6, 1)\nemb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight))\n</pre> emb = nn.Embedding(6, 1) emb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight)) In\u00a0[7]: Copied! <pre>lin(torch.tensor(obs_0_oh))\n</pre> lin(torch.tensor(obs_0_oh)) Out[7]: <pre>tensor([-0.5181], grad_fn=&lt;ViewBackward0&gt;)</pre> In\u00a0[8]: Copied! <pre>emb(torch.tensor(obs_0_le)).sum() + lin.bias\n</pre> emb(torch.tensor(obs_0_le)).sum() + lin.bias Out[8]: <pre>tensor([-0.5181], grad_fn=&lt;AddBackward0&gt;)</pre> <p>And this is precisely how the linear model <code>Wide</code> is implemented</p> In\u00a0[9]: Copied! <pre>from pytorch_widedeep.models import Wide\n</pre> from pytorch_widedeep.models import Wide <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[10]: Copied! <pre># ?Wide\n</pre> # ?Wide In\u00a0[11]: Copied! <pre>wide = Wide(input_dim=10, pred_dim=1)\nwide\n</pre> wide = Wide(input_dim=10, pred_dim=1) wide Out[11]: <pre>Wide(\n  (wide_linear): Embedding(11, 1, padding_idx=0)\n)</pre> <p>Note that even though the input dim is 10, the Embedding layer has 11 weights. Again, this is because we save <code>0</code> for padding, which is used for unseen values during the encoding process.</p> <p>As I mentioned, <code>deeptabular</code> has enough complexity on its own and it will be described in a separated notebook. Let's then jump to <code>deeptext</code>.</p> In\u00a0[12]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp In\u00a0[13]: Copied! <pre>data = {\n    \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),\n    \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),\n    \"cont1\": np.random.rand(20),\n    \"cont2\": np.random.rand(20),\n}\n\ndf = pd.DataFrame(data)\n</pre> data = {     \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),     \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),     \"cont1\": np.random.rand(20),     \"cont2\": np.random.rand(20), }  df = pd.DataFrame(data) In\u00a0[14]: Copied! <pre>df.head()\n</pre> df.head() Out[14]: cat1 cat2 cont1 cont2 0 A Y 0.789347 0.561789 1 C X 0.050822 0.061538 2 A Y 0.863784 0.241967 3 C X 0.917848 0.644658 4 C Y 0.042328 0.417303 In\u00a0[15]: Copied! <pre># see the docs for details on all params/options\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"cat1\", \"cat2\"],\n    continuous_cols=[\"cont1\", \"cont2\"],\n    embedding_rule=\"fastai\",\n)\n</pre> # see the docs for details on all params/options tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"cat1\", \"cat2\"],     continuous_cols=[\"cont1\", \"cont2\"],     embedding_rule=\"fastai\", ) In\u00a0[16]: Copied! <pre>X_tab = tab_preprocessor.fit_transform(df)\n</pre> X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre># toy example just to build a model.\ntabmlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=4,\n    mlp_hidden_dims=[8, 4],\n    mlp_linear_first=True,\n)\ntabmlp\n</pre> # toy example just to build a model. tabmlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=4,     mlp_hidden_dims=[8, 4],     mlp_linear_first=True, ) tabmlp Out[17]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_cat1): Embedding(4, 3, padding_idx=0)\n      (emb_layer_cat2): Embedding(3, 2, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=4, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=13, out_features=8, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=8, out_features=4, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Lets describe a bit the model: first we have what we call a <code>DiffSizeCatEmbeddings</code>, where categorical columns with different number of unique categories will be encoded with embeddings of different dimensions. Then the continuous columns will not be normalised (the normalised layer is just the identity) and they will be embedded via a \"standard\" method, using a so-called <code>ContLinear</code> layer. This layer displays some <code>INFO</code> that tells us what it is (<code>ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]</code>). There are two other options available to embed the continuous cols based on the paper On Embeddings for Numerical Features in Tabular Deep Learning. These are <code>PieceWise</code> and <code>Periodic</code> and all available via the <code>embed_continuous_method</code> param, which can adopt values <code>\"standard\", \"piecewise\"</code> and <code>\"periodic\"</code>. The embedded categorical and continuous columns will be then concatenated ($3 + 2 + (4 * 2) = 13$ input dims) and passed to an MLP.</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import BasicRNN\n</pre> from pytorch_widedeep.models import BasicRNN In\u00a0[19]: Copied! <pre>basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4)\n</pre> basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/torch/nn/modules/rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n  warnings.warn(\"dropout option adds dropout after all but last \"\n</pre> In\u00a0[20]: Copied! <pre>basic_rnn\n</pre> basic_rnn Out[20]: <pre>BasicRNN(\n  (word_embed): Embedding(4, 4, padding_idx=0)\n  (rnn): LSTM(4, 4, batch_first=True, dropout=0.1)\n  (rnn_mlp): Identity()\n)</pre> <p>You could, if you wanted, add a Fully Connected Head (FC-Head) on top of it</p> In\u00a0[21]: Copied! <pre>from pytorch_widedeep.models import Vision\n</pre> from pytorch_widedeep.models import Vision In\u00a0[22]: Copied! <pre>resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0)\n</pre> resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0) In\u00a0[23]: Copied! <pre>resnet\n</pre> resnet Out[23]: <pre>Vision(\n  (features): Sequential(\n    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (2): ReLU(inplace=True)\n    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n    (4): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (5): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (6): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (7): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (8): AdaptiveAvgPool2d(output_size=(1, 1))\n  )\n)</pre>"},{"location":"examples/02_model_components.html#model-components","title":"Model Components\u00b6","text":"<p>The main components of a <code>WideDeep</code> (i.e. Multimodal) model are tabular data, text and images, which are feed into the model via so called <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> model components</p>"},{"location":"examples/02_model_components.html#1-wide","title":"1. <code>wide</code>\u00b6","text":"<p>The <code>wide</code> component is a Linear layer \"plugged\" into the output neuron(s). Here, the non-linearities are captured via crossed columns. Crossed columns are, quoting directly the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>The only particularity of our implementation is that we have implemented the linear layer via an Embedding layer plus a bias. While the implementations are equivalent, the latter is faster and far more memory efficient, since we do not need to one hot encode the categorical features.</p> <p>Let's assume we the following dataset:</p>"},{"location":"examples/02_model_components.html#2-deeptabular","title":"2. <code>deeptabular</code>\u00b6","text":"<p>The <code>deeptabular</code> model alone is what normally would be referred as Deep Learning for tabular data. As mentioned a number of times throughout the library, each component can be used independently. Therefore, if you wanted to use any of the models below alone, it is perfectly possible. There are just a couple of simple requirement that will be covered in a later notebook.</p> <p>By the time of writing, there are a number of models available in <code>pytorch-widedeep</code> to do DL for tabular data. These are:</p> <ol> <li><code>TabMlp</code></li> <li><code>ContextAttentionMLP</code></li> <li><code>SelfAttentionMLP</code></li> <li><code>TabResnet</code></li> <li><code>Tabnet</code></li> <li><code>TabTransformer</code></li> <li><code>FT-Tabransformer</code></li> <li><code>SAINT</code></li> <li><code>TabFastFormer</code></li> <li><code>TabPerceiver</code></li> </ol> <p>Let's have a look to one of them. For more information on each of these models, please, have a look to the documentation</p>"},{"location":"examples/02_model_components.html#3-deeptext","title":"3. <code>deeptext</code>\u00b6","text":"<p>At the time of writing, <code>pytorch-widedeep</code> offers three models that can be passed to <code>WideDeep</code> as the <code>deeptext</code> component. These are:</p> <ol> <li>BasicRNN</li> <li>AttentiveRNN</li> <li>StackedAttentiveRNN</li> </ol> <p>For details on each of these models, please, have a look to the documentation of the package.</p> <p>We will soon integrate with Hugginface, but let me insist. It is perfectly possible to use custom models for each component, please, have a look to the corresponding notebook. In general, simply, build them and pass them as the corresponding parameters. Note that the custom models MUST return a last layer of activations (i.e. not the final prediction) so that  these activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with the size of these last layers of activations.</p> <p>Let's have a look to the <code>BasicRNN</code> model</p>"},{"location":"examples/02_model_components.html#4-deepimage","title":"4. <code>deepimage</code>\u00b6","text":"<p>At the time of writing <code>pytorch-widedeep</code> is integrated with torchvision via the <code>Vision</code> class. This means that the it is possible to use a variant of the following architectures:</p> <ol> <li>resnet</li> <li>shufflenet</li> <li>resnext</li> <li>wide_resnet</li> <li>regnet</li> <li>densenet</li> <li>mobilenet</li> <li>mnasnet</li> <li>efficientnet</li> <li>squeezenet</li> </ol> <p>The user can choose which layers will be trainable. Alternatively, in none of these architectures is useful, one could use a simple, fully trained CNN (please see the package documentation) or pass a custom model.</p> <p>let's have a look</p>"},{"location":"examples/03_binary_classification_with_defaults.html","title":"03_binary_classification_with_defaults","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy, Precision\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import Wide, TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy, Precision from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\n</pre> df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True) In\u00a0[5]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre># TARGET\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # TARGET target_col = \"income_label\" target = df[target_col].values <p>let's see what the preprocessors do</p> In\u00a0[8]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[9]: Copied! <pre># # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary\n# wide_preprocessor.encoding_dict\n</pre> # # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary # wide_preprocessor.encoding_dict In\u00a0[10]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) In\u00a0[11]: Copied! <pre># check the docs to understand the useful attributes that the tab_preprocessor has. For example,\n# as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input\n# that specifies the categortical columns that will be represented as embeddings, the number\n# of different categories per feature, and the dimension of the embeddings as defined by some\n# of the internal rules of thumb that the preprocessor has (have a look to the docs)\ntab_preprocessor.cat_embed_input\n</pre> # check the docs to understand the useful attributes that the tab_preprocessor has. For example, # as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input # that specifies the categortical columns that will be represented as embeddings, the number # of different categories per feature, and the dimension of the embeddings as defined by some # of the internal rules of thumb that the preprocessor has (have a look to the docs) tab_preprocessor.cat_embed_input Out[11]: <pre>[('workclass', 9, 5),\n ('education', 16, 8),\n ('marital_status', 7, 5),\n ('occupation', 15, 7),\n ('relationship', 6, 4),\n ('race', 5, 4),\n ('gender', 2, 2),\n ('capital_gain', 123, 24),\n ('capital_loss', 99, 21),\n ('native_country', 42, 13)]</pre> In\u00a0[12]: Copied! <pre>print(X_wide)\nprint(X_wide.shape)\n</pre> print(X_wide) print(X_wide.shape) <pre>[[  1  10  26 ...  61 103 328]\n [  1  11  27 ...  61 104 329]\n [  2  12  27 ...  61 105 330]\n ...\n [  1  11  28 ...  61 115 335]\n [  1  11  26 ...  61 115 335]\n [  7  11  27 ...  61 127 336]]\n(48842, 10)\n</pre> In\u00a0[13]: Copied! <pre>print(X_tab)\nprint(X_tab.shape)\n</pre> print(X_tab) print(X_tab.shape) <pre>[[ 1.          1.          1.         ...  1.         -0.99512893\n  -0.03408696]\n [ 1.          2.          2.         ...  1.         -0.04694151\n   0.77292975]\n [ 2.          3.          2.         ...  1.         -0.77631645\n  -0.03408696]\n ...\n [ 1.          2.          3.         ...  1.          1.41180837\n  -0.03408696]\n [ 1.          2.          1.         ...  1.         -1.21394141\n  -1.64812038]\n [ 7.          2.          2.         ...  1.          0.97418341\n  -0.03408696]]\n(48842, 12)\n</pre> In\u00a0[14]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) <p>Let's first find out how a linear model performs</p> In\u00a0[15]: Copied! <pre>wide\n</pre> wide Out[15]: <pre>Wide(\n  (wide_linear): Embedding(809, 1, padding_idx=0)\n)</pre> <p>Before being passed to the Trainer, the models need to be \"constructed\" with the <code>WideDeep</code> constructor class. For the particular case of the wide/linear model, not much really happens</p> In\u00a0[16]: Copied! <pre>lin_model = WideDeep(wide=wide)\n</pre> lin_model = WideDeep(wide=wide) In\u00a0[17]: Copied! <pre>lin_model\n</pre> lin_model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n)</pre> In\u00a0[18]: Copied! <pre>lin_trainer = Trainer(\n    model=lin_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),\n    metrics=[Accuracy, Precision],\n)\n</pre> lin_trainer = Trainer(     model=lin_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),     metrics=[Accuracy, Precision], ) In\u00a0[19]: Copied! <pre>lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 109.04it/s, loss=0.426, metrics={'acc': 0.7983, 'prec': 0.6152}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 102.46it/s, loss=0.366, metrics={'acc': 0.832, 'prec': 0.6916}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.27it/s, loss=0.364, metrics={'acc': 0.8305, 'prec': 0.6933}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 150.46it/s, loss=0.361, metrics={'acc': 0.8357, 'prec': 0.6982}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 133.19it/s, loss=0.359, metrics={'acc': 0.8329, 'prec': 0.6994}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 145.75it/s, loss=0.361, metrics={'acc': 0.836, 'prec': 0.7009}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.91it/s, loss=0.358, metrics={'acc': 0.8333, 'prec': 0.7005}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 155.08it/s, loss=0.361, metrics={'acc': 0.8364, 'prec': 0.702}]\n</pre> <p>Bear in mind that <code>wide</code> is a linear model where the non-linearities are captured via the crossed columns. For the crossed-columns to be effective one needs proper business knowledge. There is no magic formula to produce them</p> <p>Let's have a look to the tabular model by itself</p> In\u00a0[20]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_model = WideDeep(deeptabular=tab_mlp) In\u00a0[21]: Copied! <pre>tab_model\n</pre> tab_model Out[21]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=95, out_features=400, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=400, out_features=200, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>You can see how the <code>WideDeep</code> class has added a final prediction layer that collects the activations from the last layer of the model and plugs them into the output neuron. If this was a multiclass classification problem, the prediction dimension (i.e. the size of that final layer) needs to be specified via the <code>pred_dim</code> when instantiating the <code>WideDeep</code> class, as we will see later</p> In\u00a0[22]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[23]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 97.00it/s, loss=0.37, metrics={'acc': 0.8267, 'prec': 0.7037}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 134.91it/s, loss=0.313, metrics={'acc': 0.8588, 'prec': 0.7577}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 86.86it/s, loss=0.319, metrics={'acc': 0.8514, 'prec': 0.761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 73.13it/s, loss=0.296, metrics={'acc': 0.8675, 'prec': 0.7685}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 79.07it/s, loss=0.305, metrics={'acc': 0.8574, 'prec': 0.7646}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 130.11it/s, loss=0.289, metrics={'acc': 0.8696, 'prec': 0.7765}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 87.39it/s, loss=0.296, metrics={'acc': 0.8622, 'prec': 0.7769}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 90.63it/s, loss=0.285, metrics={'acc': 0.8697, 'prec': 0.7741}]\n</pre> <p>The best result I ever obtained with <code>LightGBM</code> on this dataset is 0.8782...so we are pretty close.</p> <p>Let's combine the <code>wide</code> and <code>tab_mlp</code> components see if it helps</p> In\u00a0[24]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\nwd_model = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) wd_model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[25]: Copied! <pre>wd_trainer = Trainer(\n    model=wd_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> wd_trainer = Trainer(     model=wd_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[26]: Copied! <pre>wd_trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2\n)\n</pre> wd_trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 77.48it/s, loss=0.418, metrics={'acc': 0.8047, 'prec': 0.6154}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 110.51it/s, loss=0.321, metrics={'acc': 0.8521, 'prec': 0.7059}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 82.70it/s, loss=0.333, metrics={'acc': 0.8428, 'prec': 0.7141}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 112.52it/s, loss=0.299, metrics={'acc': 0.866, 'prec': 0.7447}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 74.34it/s, loss=0.312, metrics={'acc': 0.8533, 'prec': 0.7404}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 89.86it/s, loss=0.29, metrics={'acc': 0.8683, 'prec': 0.7496}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 65.32it/s, loss=0.301, metrics={'acc': 0.8591, 'prec': 0.7542}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 86.81it/s, loss=0.286, metrics={'acc': 0.8712, 'prec': 0.7552}]\n</pre> <p>For this particular case, the combination of both did not lead to better results that using just the tab_mlp model, when using only 4 epochs.</p> <p>Note that we have use a <code>TabMlp</code> model, but we could use any other model in the library using the same syntax</p> In\u00a0[27]: Copied! <pre>from pytorch_widedeep.models import TabTransformer\n</pre> from pytorch_widedeep.models import TabTransformer <p>The parameters for the <code>TabTransformer</code> are this</p> <pre><code>column_idx: Dict[str, int],\ncat_embed_input: Optional[List[Tuple[str, int]]] = None,\ncat_embed_dropout: Optional[float] = None,\nuse_cat_bias: Optional[bool] = None,\ncat_embed_activation: Optional[str] = None,\nshared_embed: Optional[bool] = None,\nadd_shared_embed: Optional[bool] = None,\nfrac_shared_embed: Optional[float] = None,\ncontinuous_cols: Optional[List[str]] = None,\ncont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\nembed_continuous: Optional[bool] = None,\nembed_continuous_method: Optional[Literal[\"standard\", \"piecewise\", \"periodic\"]] = None,\ncont_embed_dropout: Optional[float] = None,\ncont_embed_activation: Optional[str] = None,\nquantization_setup: Optional[Dict[str, List[float]]] = None,\nn_frequencies: Optional[int] = None,\nsigma: Optional[float] = None,\nshare_last_layer: Optional[bool] = None,\nfull_embed_dropout: Optional[bool] = None,\ninput_dim: int = 32,\nn_heads: int = 8,\nuse_qkv_bias: bool = False,\nn_blocks: int = 4,\nattn_dropout: float = 0.2,\nff_dropout: float = 0.1,\nff_factor: int = 4,\ntransformer_activation: str = \"gelu\",\nuse_linear_attention: bool = False,\nuse_flash_attention: bool = False,\nmlp_hidden_dims: Optional[List[int]] = None,\nmlp_activation: str = \"relu\",\nmlp_dropout: float = 0.1,\nmlp_batchnorm: bool = False,\nmlp_batchnorm_last: bool = False,\nmlp_linear_first: bool = True,\n</code></pre> <p>Please, see the documentation for details on each one of them, for now let's see how one could use a <code>TabTransformer</code> model in a few lines of code</p> In\u00a0[28]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_norm_layer=\"layernorm\",\n    cont_embed_dropout=0.2,\n    cont_embed_activation=\"leaky_relu\",\n    n_heads=4,\n    ff_dropout=0.2,\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n    mlp_linear_first=\"True\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_norm_layer=\"layernorm\",     cont_embed_dropout=0.2,     cont_embed_activation=\"leaky_relu\",     n_heads=4,     ff_dropout=0.2,     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\",     mlp_linear_first=\"True\", ) In\u00a0[29]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_transformer)\n</pre> tab_model = WideDeep(deeptabular=tab_transformer) In\u00a0[30]: Copied! <pre>tab_model\n</pre> tab_model Out[30]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabTransformer(\n      (cat_embed): SameSizeCatEmbeddings(\n        (embed): Embedding(325, 32, padding_idx=0)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.2)\n        (activation_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dropout): Dropout(p=0.2, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block2): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block3): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=384, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[31]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[32]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:11&lt;00:00, 27.57it/s, loss=0.359, metrics={'acc': 0.8334, 'prec': 0.7082}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 57.89it/s, loss=0.33, metrics={'acc': 0.8536, 'prec': 0.7152}]\n</pre>"},{"location":"examples/03_binary_classification_with_defaults.html#simple-binary-classification-with-defaults","title":"Simple Binary Classification with defaults\u00b6","text":"<p>In this notebook we will train a Wide and Deep model and simply a \"Deep\" model using the well known adult dataset</p>"},{"location":"examples/03_binary_classification_with_defaults.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/03_binary_classification_with_defaults.html#defining-the-model","title":"Defining the model\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html","title":"04_regression_with_images_and_text","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\nfrom torchvision.transforms import ToTensor, Normalize\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\n</pre> import numpy as np import pandas as pd import os import torch from torchvision.transforms import ToTensor, Normalize  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n</pre> text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df) <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\n</pre> In\u00a0[8]: Copied! <pre>image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>Reading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 638.00it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[9]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# DeepText: a stack of 2 LSTMs\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_matrix=text_preprocessor.embedding_matrix,\n    n_layers=2,\n    hidden_dim=64,\n    rnn_dropout=0.5,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # DeepText: a stack of 2 LSTMs basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_matrix=text_preprocessor.embedding_matrix,     n_layers=2,     hidden_dim=64,     rnn_dropout=0.5, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4) <p>Combine them all with the \"collector\" class <code>WideDeep</code></p> In\u00a0[10]: Copied! <pre>model = WideDeep(\n    wide=wide,\n    deeptabular=tab_mlp,\n    deeptext=basic_rnn,\n    deepimage=resnet,\n    head_hidden_dims=[256, 128],\n)\n</pre> model = WideDeep(     wide=wide,     deeptabular=tab_mlp,     deeptext=basic_rnn,     deepimage=resnet,     head_hidden_dims=[256, 128], ) In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"rmse\")\n</pre> trainer = Trainer(model, objective=\"rmse\") In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.28it/s, loss=115]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=94.1]\n</pre> <p>Both, the Text and Image components allow FC-heads on their own (have a look to the documentation).</p> <p>Now let's go \"kaggle crazy\". Let's use different optimizers, initializers and schedulers for different components. Moreover, let's use a different learning rate for different parameter groups, for the <code>deeptabular</code> component</p> In\u00a0[13]: Copied! <pre>deep_params = []\nfor childname, child in model.named_children():\n    if childname == \"deeptabular\":\n        for n, p in child.named_parameters():\n            if \"embed_layer\" in n:\n                deep_params.append({\"params\": p, \"lr\": 1e-4})\n            else:\n                deep_params.append({\"params\": p, \"lr\": 1e-3})\n</pre> deep_params = [] for childname, child in model.named_children():     if childname == \"deeptabular\":         for n, p in child.named_parameters():             if \"embed_layer\" in n:                 deep_params.append({\"params\": p, \"lr\": 1e-4})             else:                 deep_params.append({\"params\": p, \"lr\": 1e-3}) In\u00a0[14]: Copied! <pre>wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03)\ndeep_opt = torch.optim.Adam(deep_params)\ntext_opt = torch.optim.AdamW(model.deeptext.parameters())\nimg_opt = torch.optim.AdamW(model.deepimage.parameters())\nhead_opt = torch.optim.Adam(model.deephead.parameters())\n</pre> wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03) deep_opt = torch.optim.Adam(deep_params) text_opt = torch.optim.AdamW(model.deeptext.parameters()) img_opt = torch.optim.AdamW(model.deepimage.parameters()) head_opt = torch.optim.Adam(model.deephead.parameters()) In\u00a0[15]: Copied! <pre>wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\ndeep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\ntext_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\nimg_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\nhead_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5)\n</pre> wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5) deep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5) img_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) head_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5) In\u00a0[16]: Copied! <pre># remember, one optimizer per model components, for lr_schedures and initializers is not neccesary\noptimizers = {\n    \"wide\": wide_opt,\n    \"deeptabular\": deep_opt,\n    \"deeptext\": text_opt,\n    \"deepimage\": img_opt,\n    \"deephead\": head_opt,\n}\nschedulers = {\n    \"wide\": wide_sch,\n    \"deeptabular\": deep_sch,\n    \"deeptext\": text_sch,\n    \"deepimage\": img_sch,\n    \"deephead\": head_sch,\n}\n\n# Now...we have used pretrained word embeddings, so you do not want to\n# initialise these  embeddings. However you might still want to initialise the\n# other layers in the DeepText component. No probs, you can do that with the\n# parameter pattern and your knowledge on regular  expressions. Here we are\n# telling to the KaimingNormal initializer to NOT touch the  parameters whose\n# name contains the string word_embed.\ninitializers = {\n    \"wide\": KaimingNormal,\n    \"deeptabular\": KaimingNormal,\n    \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),\n    \"deepimage\": KaimingNormal,\n}\n\nmean = [0.406, 0.456, 0.485]  # BGR\nstd = [0.225, 0.224, 0.229]  # BGR\ntransforms = [ToTensor, Normalize(mean=mean, std=std)]\ncallbacks = [\n    LRHistory(n_epochs=10),\n    EarlyStopping,\n    ModelCheckpoint(filepath=\"model_weights/wd_out\"),\n]\n</pre> # remember, one optimizer per model components, for lr_schedures and initializers is not neccesary optimizers = {     \"wide\": wide_opt,     \"deeptabular\": deep_opt,     \"deeptext\": text_opt,     \"deepimage\": img_opt,     \"deephead\": head_opt, } schedulers = {     \"wide\": wide_sch,     \"deeptabular\": deep_sch,     \"deeptext\": text_sch,     \"deepimage\": img_sch,     \"deephead\": head_sch, }  # Now...we have used pretrained word embeddings, so you do not want to # initialise these  embeddings. However you might still want to initialise the # other layers in the DeepText component. No probs, you can do that with the # parameter pattern and your knowledge on regular  expressions. Here we are # telling to the KaimingNormal initializer to NOT touch the  parameters whose # name contains the string word_embed. initializers = {     \"wide\": KaimingNormal,     \"deeptabular\": KaimingNormal,     \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),     \"deepimage\": KaimingNormal, }  mean = [0.406, 0.456, 0.485]  # BGR std = [0.225, 0.224, 0.229]  # BGR transforms = [ToTensor, Normalize(mean=mean, std=std)] callbacks = [     LRHistory(n_epochs=10),     EarlyStopping,     ModelCheckpoint(filepath=\"model_weights/wd_out\"), ] In\u00a0[17]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"rmse\",\n    initializers=initializers,\n    optimizers=optimizers,\n    lr_schedulers=schedulers,\n    callbacks=callbacks,\n    transforms=transforms,\n)\n</pre> trainer = Trainer(     model,     objective=\"rmse\",     initializers=initializers,     optimizers=optimizers,     lr_schedulers=schedulers,     callbacks=callbacks,     transforms=transforms, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/initializers.py:34: UserWarning: No initializer found for deephead\n  warnings.warn(\n</pre> In\u00a0[18]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.25it/s, loss=101]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=90.6]\n</pre> <pre>Model weights after training corresponds to the those of the final epoch which might not be the best performing weights. Use the 'ModelCheckpoint' Callback to restore the best epoch weights.\n</pre> <p>we have only run one epoch, but let's check that the LRHistory callback records the lr values for each group</p> In\u00a0[19]: Copied! <pre>trainer.lr_history\n</pre> trainer.lr_history Out[19]: <pre>{'lr_wide_0': [0.03, 0.03],\n 'lr_deeptabular_0': [0.0001, 0.0001],\n 'lr_deeptabular_1': [0.0001, 0.0001],\n 'lr_deeptabular_2': [0.0001, 0.0001],\n 'lr_deeptabular_3': [0.0001, 0.0001],\n 'lr_deeptabular_4': [0.0001, 0.0001],\n 'lr_deeptabular_5': [0.0001, 0.0001],\n 'lr_deeptabular_6': [0.0001, 0.0001],\n 'lr_deeptabular_7': [0.0001, 0.0001],\n 'lr_deeptabular_8': [0.0001, 0.0001],\n 'lr_deeptabular_9': [0.001, 0.001],\n 'lr_deeptabular_10': [0.001, 0.001],\n 'lr_deeptabular_11': [0.001, 0.001],\n 'lr_deeptabular_12': [0.001, 0.001],\n 'lr_deeptext_0': [0.001, 0.001],\n 'lr_deepimage_0': [0.001, 0.001],\n 'lr_deephead_0': [0.001, 0.001]}</pre>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-images-and-text","title":"Regression with Images and Text\u00b6","text":"<p>In this notebook we will go through a series of examples on how to combine all Wide &amp; Deep components.</p> <p>To that aim I will use the Airbnb listings dataset for London, which you can download from here. I use this dataset simply because it contains tabular data, images and text.</p> <p>I have taken a sample of 1000 listings to keep the data tractable in this notebook. Also, I have preprocessed the data and prepared it for this exercise. All preprocessing steps can be found in the notebook <code>airbnb_data_preprocessing.ipynb</code> in this <code>examples</code> folder.</p>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-the-defaults","title":"Regression with the defaults\u00b6","text":"<p>The set up</p>"},{"location":"examples/04_regression_with_images_and_text.html#prepare-the-data","title":"Prepare the data\u00b6","text":"<p>I will focus here on how to prepare the data and run the model. Check notebooks 1 and 2 to see what's going on behind the scences</p> <p>Preparing the data is rather simple</p>"},{"location":"examples/04_regression_with_images_and_text.html#build-the-model-components","title":"Build the model components\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html#build-the-trainer-and-fit","title":"Build the trainer and fit\u00b6","text":""},{"location":"examples/05_save_and_load_model_and_artifacts.html","title":"05_save_and_load_model_and_artifacts","text":"In\u00a0[1]: Copied! <pre>import pickle\nimport numpy as np\nimport pandas as pd\nimport torch\nimport shutil\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom sklearn.model_selection import train_test_split\n</pre> import pickle import numpy as np import pandas as pd import torch import shutil  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from sklearn.model_selection import train_test_split <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>train, valid = train_test_split(df, test_size=0.2, stratify=df.target)\n# the test data will be used lately as if it was \"fresh\", new data coming after some time...\nvalid, test = train_test_split(valid, test_size=0.5, stratify=valid.target)\n</pre> train, valid = train_test_split(df, test_size=0.2, stratify=df.target) # the test data will be used lately as if it was \"fresh\", new data coming after some time... valid, test = train_test_split(valid, test_size=0.5, stratify=valid.target) In\u00a0[5]: Copied! <pre>print(f\"train shape: {train.shape}\")\nprint(f\"valid shape: {valid.shape}\")\nprint(f\"test shape: {test.shape}\")\n</pre> print(f\"train shape: {train.shape}\") print(f\"valid shape: {valid.shape}\") print(f\"test shape: {test.shape}\") <pre>train shape: (39073, 15)\nvalid shape: (4884, 15)\ntest shape: (4885, 15)\n</pre> In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab_train = tab_preprocessor.fit_transform(train)\ny_train = train.target.values\nX_tab_valid = tab_preprocessor.transform(valid)\ny_valid = valid.target.values\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab_train = tab_preprocessor.fit_transform(train) y_train = train.target.values X_tab_valid = tab_preprocessor.transform(valid) y_valid = valid.target.values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[8]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[9]: Copied! <pre>model\n</pre> model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(122, 23, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(97, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=8, embed_dropout=0.0)\n        (dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=108, out_features=64, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre>early_stopping = EarlyStopping()\nmodel_checkpoint = ModelCheckpoint(\n    filepath=\"tmp_dir/adult_tabmlp_model\",\n    save_best_only=True,\n    verbose=1,\n    max_save=1,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    callbacks=[early_stopping, model_checkpoint],\n    metrics=[Accuracy],\n)\n\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=4,\n    batch_size=256,\n)\n</pre> early_stopping = EarlyStopping() model_checkpoint = ModelCheckpoint(     filepath=\"tmp_dir/adult_tabmlp_model\",     save_best_only=True,     verbose=1,     max_save=1, )  trainer = Trainer(     model,     objective=\"binary\",     callbacks=[early_stopping, model_checkpoint],     metrics=[Accuracy], )  trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=4,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 76.25it/s, loss=0.452, metrics={'acc': 0.7867}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 125.36it/s, loss=0.335, metrics={'acc': 0.8532}]\n</pre> <pre>\nEpoch 1: val_loss improved from inf to 0.33532 Saving model to tmp_dir/adult_tabmlp_model_1.p\n</pre> <pre>epoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 76.98it/s, loss=0.355, metrics={'acc': 0.8401}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 106.51it/s, loss=0.303, metrics={'acc': 0.8665}]\n</pre> <pre>\nEpoch 2: val_loss improved from 0.33532 to 0.30273 Saving model to tmp_dir/adult_tabmlp_model_2.p\n</pre> <pre>epoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.71it/s, loss=0.332, metrics={'acc': 0.849}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 107.80it/s, loss=0.288, metrics={'acc': 0.8757}]\n</pre> <pre>\nEpoch 3: val_loss improved from 0.30273 to 0.28791 Saving model to tmp_dir/adult_tabmlp_model_3.p\n</pre> <pre>epoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 79.02it/s, loss=0.32, metrics={'acc': 0.8541}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 127.07it/s, loss=0.282, metrics={'acc': 0.8763}]</pre> <pre>\nEpoch 4: val_loss improved from 0.28791 to 0.28238 Saving model to tmp_dir/adult_tabmlp_model_4.p\nModel weights restored to best epoch: 4\n</pre> <pre>\n</pre> In\u00a0[11]: Copied! <pre>torch.save(model, \"tmp_dir/model_saved_option_1.pt\")\n</pre> torch.save(model, \"tmp_dir/model_saved_option_1.pt\") In\u00a0[12]: Copied! <pre>torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\")\n</pre> torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\") In\u00a0[13]: Copied! <pre>trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\")\n</pre> trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\") <p>or the state dict</p> In\u00a0[14]: Copied! <pre>trainer.save(\n    path=\"tmp_dir/\",\n    model_filename=\"model_state_dict_saved_option_2.pt\",\n    save_state_dict=True,\n)\n</pre> trainer.save(     path=\"tmp_dir/\",     model_filename=\"model_state_dict_saved_option_2.pt\",     save_state_dict=True, ) In\u00a0[15]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\n</pre> In\u00a0[16]: Copied! <pre>%%bash\n\nls tmp_dir/history/\n</pre> %%bash  ls tmp_dir/history/ <pre>train_eval_history.json\n</pre> <p>Note that since we have used the <code>ModelCheckpoint</code> Callback, <code>adult_tabmlp_model_2.p</code> is the model state dict of the model at epoch 2, i.e. same as <code>model_state_dict_saved_option_1.p</code> or <code>model_state_dict_saved_option_2.p</code>.</p> In\u00a0[17]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:\n    pickle.dump(tab_preprocessor, dp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:     pickle.dump(tab_preprocessor, dp) In\u00a0[18]: Copied! <pre>with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:\n    pickle.dump(early_stopping, es)\n</pre> with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:     pickle.dump(early_stopping, es) In\u00a0[19]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\neary_stop.pkl\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\ntab_preproc.pkl\n</pre> <p>And that is pretty much all you need to resume training or directly predict, let's see</p> In\u00a0[20]: Copied! <pre>test.head()\n</pre> test.head() Out[20]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 10103 43 Private 198282 HS-grad 9 Married-civ-spouse Craft-repair Husband White Male 0 0 40 United-States 1 31799 20 Private 228686 11th 7 Married-civ-spouse Other-service Husband White Male 0 0 40 United-States 0 19971 26 Private 291968 HS-grad 9 Married-civ-spouse Transport-moving Husband White Male 0 0 44 United-States 0 3039 48 Private 175958 Bachelors 13 Divorced Prof-specialty Not-in-family White Male 0 0 30 United-States 0 20725 18 Private 232024 11th 7 Never-married Machine-op-inspct Own-child White Male 0 0 55 United-States 0 In\u00a0[21]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:\n    tab_preprocessor_new = pickle.load(tp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:     tab_preprocessor_new = pickle.load(tp) In\u00a0[22]: Copied! <pre>X_test_tab = tab_preprocessor_new.transform(test)\ny_test = test.target\n</pre> X_test_tab = tab_preprocessor_new.transform(test) y_test = test.target In\u00a0[23]: Copied! <pre>tab_mlp_new = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nnew_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp_new = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) new_model = WideDeep(deeptabular=tab_mlp) In\u00a0[24]: Copied! <pre>new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\"))\n</pre> new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\")) Out[24]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[25]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\", ) In\u00a0[26]: Copied! <pre>preds = trainer.predict(X_tab=X_test_tab, batch_size=32)\n</pre> preds = trainer.predict(X_tab=X_test_tab, batch_size=32) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:00&lt;00:00, 309.83it/s]\n</pre> In\u00a0[27]: Copied! <pre>from sklearn.metrics import accuracy_score\n</pre> from sklearn.metrics import accuracy_score In\u00a0[28]: Copied! <pre>accuracy_score(y_test, preds)\n</pre> accuracy_score(y_test, preds) Out[28]: <pre>0.8595701125895598</pre> In\u00a0[29]: Copied! <pre>shutil.rmtree(\"tmp_dir/\")\n</pre> shutil.rmtree(\"tmp_dir/\")"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-and-load-model-and-artifacts","title":"Save and load model and artifacts\u00b6","text":"<p>In this notebook I will show the different options to save and load a model, as well as some additional objects produced during training.</p> <p>On a given day, you train a model...</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-1","title":"Save model: option 1\u00b6","text":"<p>save (and load) a model as you woud do with any other torch model</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-2","title":"Save model: option 2\u00b6","text":"<p>use the <code>trainer</code>. The <code>trainer</code> will also save the training history and the learning rate history (if learning rate schedulers are used)</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-preprocessors-and-callbacks","title":"Save preprocessors and callbacks\u00b6","text":"<p>...just pickle them</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#run-new-experiment-prepare-new-dataset-load-model-and-predict","title":"Run New experiment: prepare new dataset, load model, and predict\u00b6","text":""},{"location":"examples/06_finetune_and_warmup.html","title":"06_finetune_and_warmup","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\n# For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[3]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[4]: Copied! <pre># TARGET\ntarget = df[target_col].values\n\n# WIDE\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# DEEP\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # TARGET target = df[target_col].values  # WIDE wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # DEEP tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[5]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[6]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[7]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 74.26it/s, loss=0.399, metrics={'acc': 0.8163}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 91.03it/s, loss=0.296, metrics={'acc': 0.8677}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 81.31it/s, loss=0.3, metrics={'acc': 0.8614}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 106.45it/s, loss=0.285, metrics={'acc': 0.8721}]\n</pre> In\u00a0[8]: Copied! <pre>trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\")\n</pre> trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\") <p>Now time goes by...and we want to fine-tune the model to another, new dataset (for example, a dataset that is identical to the one you used to train the previous model but for another country).</p> <p>Here I will use the same dataset just for illustration purposes, but the flow would be identical to that new dataset</p> In\u00a0[9]: Copied! <pre>wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp_1 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_1 = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp_1 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_1 = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[10]: Copied! <pre>model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\"))\n</pre> model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\")) Out[10]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[11]: Copied! <pre>trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy]) In\u00a0[12]: Copied! <pre>trainer_1.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=2,\n    batch_size=256,\n    finetune=True,\n    finetune_epochs=2,\n)\n</pre> trainer_1.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     n_epochs=2,     batch_size=256,     finetune=True,     finetune_epochs=2, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 97.37it/s, loss=0.39, metrics={'acc': 0.8152}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 104.04it/s, loss=0.359, metrics={'acc': 0.824}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 83.83it/s, loss=0.297, metrics={'acc': 0.8365}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 82.78it/s, loss=0.283, metrics={'acc': 0.8445}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 72.84it/s, loss=0.281, metrics={'acc': 0.8716}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 77.46it/s, loss=0.273, metrics={'acc': 0.8744}]\n</pre> <p>Note that, as I describe above, in scenario 2, we can just use this to warm up models before they joined training begins:</p> In\u00a0[13]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[14]: Copied! <pre>trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[15]: Copied! <pre>trainer_2.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    warmup=True,\n    warmup_epochs=2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_2.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     warmup=True,     warmup_epochs=2,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 102.49it/s, loss=0.52, metrics={'acc': 0.7519}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 98.15it/s, loss=0.381, metrics={'acc': 0.7891}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 82.97it/s, loss=0.356, metrics={'acc': 0.8043}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 80.27it/s, loss=0.295, metrics={'acc': 0.8195}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 77.27it/s, loss=0.291, metrics={'acc': 0.8667}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 89.57it/s, loss=0.289, metrics={'acc': 0.8665}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.69it/s, loss=0.283, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 91.81it/s, loss=0.284, metrics={'acc': 0.869}]\n</pre> <p>We need to explicitly indicate</p> <ol> <li><p>That we want fine-tune</p> </li> <li><p>The components that we want to individually fine-tune</p> </li> <li><p>In case of gradual fine-tuning, the routine (\"felbo\" or \"howard\")</p> </li> <li><p>The layers we want to fine-tune.</p> </li> </ol> <p>For example</p> In\u00a0[16]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[17]: Copied! <pre>model\n</pre> model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>let's first train as usual</p> In\u00a0[18]: Copied! <pre>trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[19]: Copied! <pre>trainer_3.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256\n)\n</pre> trainer_3.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.23it/s, loss=0.382, metrics={'acc': 0.8239}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.72it/s, loss=0.331, metrics={'acc': 0.8526}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.35it/s, loss=0.33, metrics={'acc': 0.8465}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 68.15it/s, loss=0.312, metrics={'acc': 0.8604}]\n</pre> In\u00a0[20]: Copied! <pre>trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\")\n</pre> trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\") <p>Now we are going to fine-tune the model components, and in the case of the <code>deeptabular</code> component, we will fine-tune the resnet-blocks and the linear layer but NOT the embeddings.</p> <p>For this, we need to access the model component's children: <code>deeptabular</code> $\\rightarrow$ <code>tab_resnet</code> $\\rightarrow$ <code>dense_resnet</code> $\\rightarrow$ <code>blocks</code></p> In\u00a0[21]: Copied! <pre>wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet_3 = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel_3 = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet_3 = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model_3 = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[22]: Copied! <pre>model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\"))\n</pre> model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\")) Out[22]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[23]: Copied! <pre>model_3\n</pre> model_3 Out[23]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre>tab_lin_layer = list(model_3.deeptabular.children())[1]\n</pre> tab_lin_layer = list(model_3.deeptabular.children())[1] In\u00a0[25]: Copied! <pre>tab_lin_layer\n</pre> tab_lin_layer Out[25]: <pre>Linear(in_features=200, out_features=1, bias=True)</pre> In\u00a0[26]: Copied! <pre>tab_deep_layers = []\nfor n1, c1 in model_3.deeptabular.named_children():\n    if (\n        n1 == \"0\"\n    ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class\n        for n2, c2 in c1.named_children():\n            if n2 == \"encoder\":  # TabResnet\n                for _, c3 in c2.named_children():\n                    for n4, c4 in c3.named_children():  # dense_resnet\n                        if \"block\" in n4:\n                            tab_deep_layers.append((n4, c4))\n</pre> tab_deep_layers = [] for n1, c1 in model_3.deeptabular.named_children():     if (         n1 == \"0\"     ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class         for n2, c2 in c1.named_children():             if n2 == \"encoder\":  # TabResnet                 for _, c3 in c2.named_children():                     for n4, c4 in c3.named_children():  # dense_resnet                         if \"block\" in n4:                             tab_deep_layers.append((n4, c4)) In\u00a0[27]: Copied! <pre>tab_deep_layers\n</pre> tab_deep_layers Out[27]: <pre>[('block_0',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  )),\n ('block_1',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  ))]</pre> <p>Now remember, we need to pass ONLY LAYERS (before I included the name for clarity) the layers in WARM UP ORDER, therefore:</p> In\u00a0[28]: Copied! <pre>tab_deep_layers = [el[1] for el in tab_deep_layers][::-1]\n</pre> tab_deep_layers = [el[1] for el in tab_deep_layers][::-1] In\u00a0[29]: Copied! <pre>tab_layers = [tab_lin_layer] + tab_deep_layers[::-1]\n</pre> tab_layers = [tab_lin_layer] + tab_deep_layers[::-1] In\u00a0[30]: Copied! <pre>tab_layers\n</pre> tab_layers Out[30]: <pre>[Linear(in_features=200, out_features=1, bias=True),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n ),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )]</pre> <p>And now simply</p> In\u00a0[31]: Copied! <pre>trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy]) In\u00a0[32]: Copied! <pre>trainer_4.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    deeptabular_gradual=True,\n    deeptabular_layers=tab_layers,\n    deeptabular_max_lr=0.01,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_4.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     deeptabular_gradual=True,     deeptabular_layers=tab_layers,     deeptabular_max_lr=0.01,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 95.17it/s, loss=0.504, metrics={'acc': 0.7523}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 99.83it/s, loss=0.384, metrics={'acc': 0.789}]\n</pre> <pre>Training deeptabular, layer 1 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.31it/s, loss=0.317, metrics={'acc': 0.8098}]\n</pre> <pre>Training deeptabular, layer 2 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 65.97it/s, loss=0.312, metrics={'acc': 0.8214}]\n</pre> <pre>Training deeptabular, layer 3 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 63.92it/s, loss=0.306, metrics={'acc': 0.8284}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 57.26it/s, loss=0.292, metrics={'acc': 0.8664}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.56it/s, loss=0.292, metrics={'acc': 0.8696}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 53.61it/s, loss=0.282, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 80.59it/s, loss=0.289, metrics={'acc': 0.8719}]\n</pre> <p>Finally, there is one more use case I would like to consider. The case where we train only one component and we just want to fine-tune and stop the training afterwards, since there is no joined training. This is a simple as</p> In\u00a0[33]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[34]: Copied! <pre>trainer_5 = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_5 = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[35]: Copied! <pre>trainer_5.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256\n)\n</pre> trainer_5.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.69it/s, loss=0.365, metrics={'acc': 0.8331}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 92.56it/s, loss=0.299, metrics={'acc': 0.8673}]\n</pre> In\u00a0[36]: Copied! <pre>trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\")\n</pre> trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\") In\u00a0[37]: Copied! <pre>tab_mlp_5 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_5 = WideDeep(deeptabular=tab_mlp_5)\n</pre> tab_mlp_5 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_5 = WideDeep(deeptabular=tab_mlp_5) In\u00a0[38]: Copied! <pre>model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\"))\n</pre> model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\")) Out[38]: <pre>&lt;All keys matched successfully&gt;</pre> <p>...times go by...</p> In\u00a0[39]: Copied! <pre>trainer_6 = Trainer(\n    model_5,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_6 = Trainer(     model_5,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[40]: Copied! <pre>trainer_6.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    finetune_max_lr=0.01,\n    stop_after_finetuning=True,\n    batch_size=256,\n)\n</pre> trainer_6.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     finetune_max_lr=0.01,     stop_after_finetuning=True,     batch_size=256, ) <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.86it/s, loss=0.298, metrics={'acc': 0.8652}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 75.45it/s, loss=0.286, metrics={'acc': 0.8669}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 1 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 76.29it/s, loss=0.282, metrics={'acc': 0.8698}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.93it/s, loss=0.281, metrics={'acc': 0.8749}]\n</pre> In\u00a0[42]: Copied! <pre>import shutil\n\nshutil.rmtree(\"models_dir/\")\nshutil.rmtree(\"model_weights/\")\n</pre> import shutil  shutil.rmtree(\"models_dir/\") shutil.rmtree(\"model_weights/\") In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/06_finetune_and_warmup.html#the-finetunewarm-up-option","title":"The FineTune/Warm Up option\u00b6","text":"<p>Let's place ourselves in two possible scenarios.</p> <ol> <li><p>Let's assume we have run a model and we want to just transfer the learnings (you know...transfer-learning) to another dataset, or simply we have received new data and we do not want to start the training of each component from scratch. Simply, we want to load the pre-trained weights and fine-tune.</p> </li> <li><p>We just want to \"warm up\" individual model components individually before the joined training begins.</p> </li> </ol> <p>This can be done with the <code>finetune</code> set of parameters. There are 3 fine-tuning routines:</p> <ol> <li>Fine-tune all trainable layers at once with a triangular one-cycle learning rate (referred as slanted triangular learning rates in Howard &amp; Ruder 2018)</li> <li>Gradual fine-tuning inspired by the work of Felbo et al., 2017</li> <li>Gradual fine-tuning based on the work of Howard &amp; Ruder 2018</li> </ol> <p>Currently fine-tunning is only supported without a fully connected head, i.e. if <code>deephead=None</code>. In addition, <code>Felbo</code> and <code>Howard</code> routines only applied, of course, to the <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> models. The <code>wide</code> component can also be fine-tuned, but only in an \"all at once\" mode.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-or-warm-up-all-at-once","title":"Fine-tune or warm-up all at once\u00b6","text":"<p>Here, the model components will be trained for <code>finetune_epochs</code> using a triangular one-cycle learning rate (slanted triangular learning rate) ranging from <code>finetune_max_lr/10</code> to <code>finetune_max_lr</code> (default is 0.01). 10% of the training steps are used to increase the learning rate which then decreases for the remaining 90%.</p> <p>Here all trainable layers are fine-tuned.</p> <p>Let's have a look to one example.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-gradually-the-felbo-and-the-howard-routines","title":"Fine-tune Gradually: The \"felbo\"  and the \"howard\" routines\u00b6","text":"<p>The Felbo routine can be illustrated as follows:</p> <p> </p> <p>Figure 1. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while freezing the already warmed up layer(s). Repeat untill all individual layers are warmed. Then warm one last epoch with all warmed layers trainable. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that this is not identical to the Fine-Tunning routine described in Felbo et al, 2017, this is why I used the word 'inspired'.</p> <p>The Howard routine can be illustrated as follows:</p> <p> </p> <p>Figure 2. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while keeping the already warmed up layer(s) trainable. Repeat. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that I write \"fine-tune (or train) the last layer for one epoch [...]\". However, in practice the user will have to specify the order of the layers to be fine-tuned. This is another reason why I wrote that the fine-tune routines I have implemented are inspired by the work of Felbo and Howard and not identical to their implemenations.</p> <p>The <code>felbo</code> and <code>howard</code> routines can be accessed with via the <code>fine-tune</code> parameters.</p>"},{"location":"examples/07_custom_components.html","title":"07_custom_components","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\n\nfrom torch import Tensor\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import os import torch  from torch import Tensor from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n# it does not make sense to standarised Latitude and Longitude\nalready_standard = [\"latitude\", \"longitude\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"] # it does not make sense to standarised Latitude and Longitude already_standard = [\"latitude\", \"longitude\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n\ntext_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n\nimage_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df)  text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df)  image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\nReading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:02&lt;00:00, 497.80it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> <p>Now we are ready to build a wide and deep model. Three of the four components we will use are included in this package, and they will be combined with a custom <code>deeptext</code> component. Then the fit process will run with a custom loss function.</p> <p>Let's have a look</p> In\u00a0[6]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0) In\u00a0[7]: Copied! <pre>class MyDeepText(nn.Module):\n    def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):\n        super(MyDeepText, self).__init__()\n\n        # word/token embeddings\n        self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)\n\n        # stack of RNNs\n        self.rnn = nn.GRU(\n            embed_dim,\n            hidden_dim,\n            num_layers=2,\n            bidirectional=True,\n            batch_first=True,\n        )\n\n        # Remember, this MUST be defined. If not WideDeep will through an error\n        self.output_dim = hidden_dim * 2\n\n    def forward(self, X):\n        embed = self.word_embed(X.long())\n        o, h = self.rnn(embed)\n        return torch.cat((h[-2], h[-1]), dim=1)\n</pre> class MyDeepText(nn.Module):     def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):         super(MyDeepText, self).__init__()          # word/token embeddings         self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)          # stack of RNNs         self.rnn = nn.GRU(             embed_dim,             hidden_dim,             num_layers=2,             bidirectional=True,             batch_first=True,         )          # Remember, this MUST be defined. If not WideDeep will through an error         self.output_dim = hidden_dim * 2      def forward(self, X):         embed = self.word_embed(X.long())         o, h = self.rnn(embed)         return torch.cat((h[-2], h[-1]), dim=1) In\u00a0[8]: Copied! <pre>mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos))\n</pre> mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos)) In\u00a0[9]: Copied! <pre>model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet)\n</pre> model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet) In\u00a0[10]: Copied! <pre>class RMSELoss(nn.Module):\n    def __init__(self):\n        \"\"\"root mean squared error\"\"\"\n        super().__init__()\n        self.mse = nn.MSELoss()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        return torch.sqrt(self.mse(input, target))\n</pre> class RMSELoss(nn.Module):     def __init__(self):         \"\"\"root mean squared error\"\"\"         super().__init__()         self.mse = nn.MSELoss()      def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:         return torch.sqrt(self.mse(input, target)) <p>and now we just instantiate the <code>Trainer</code> as usual. Needless to say, but this runs with 1000 random observations, so loss and metric values are meaningless. This is just an example</p> In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss())\n</pre> trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss()) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:23&lt;00:00,  1.07it/s, loss=126]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:05&lt;00:00,  1.24it/s, loss=97.4]\n</pre> <p>In addition to model components and loss functions, we can also use custom callbacks or custom metrics. The former need to be of type <code>Callback</code> and the latter need to be of type <code>Metric</code>. See:</p> <pre>pytorch-widedeep.callbacks\n</pre> <p>and</p> <pre>pytorch-widedeep.metrics\n</pre> <p>For this example let me use the adult dataset. Again, we first prepare the data as usual</p> In\u00a0[13]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[13]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[14]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[14]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[15]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[16]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[18]: Copied! <pre>from pytorch_widedeep.metrics import Metric\n</pre> from pytorch_widedeep.metrics import Metric In\u00a0[19]: Copied! <pre>class Accuracy(Metric):\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n\n        # \u00a0metric name needs to be defined\n        self._name = \"acc\"\n\n    def reset(self):\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</pre> class Accuracy(Metric):     def __init__(self, top_k: int = 1):         super(Accuracy, self).__init__()          self.top_k = top_k         self.correct_count = 0         self.total_count = 0          # \u00a0metric name needs to be defined         self._name = \"acc\"      def reset(self):         self.correct_count = 0         self.total_count = 0      def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:         num_classes = y_pred.size(1)          if num_classes == 1:             y_pred = y_pred.round()             y_true = y_true         elif num_classes &gt; 1:             y_pred = y_pred.topk(self.top_k, 1)[1]             y_true = y_true.view(-1, 1).expand_as(y_pred)          self.correct_count += y_pred.eq(y_true).sum().item()         self.total_count += len(y_pred)         accuracy = float(self.correct_count) / float(self.total_count)         return np.array(accuracy) In\u00a0[20]: Copied! <pre># have a look to the class\nfrom pytorch_widedeep.callbacks import Callback\n</pre> # have a look to the class from pytorch_widedeep.callbacks import Callback In\u00a0[21]: Copied! <pre>class SillyCallback(Callback):\n    def on_train_begin(self, logs=None):\n        # recordings will be the trainer object attributes\n        self.trainer.silly_callback = {}\n\n        self.trainer.silly_callback[\"beginning\"] = []\n        self.trainer.silly_callback[\"end\"] = []\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.trainer.silly_callback[\"beginning\"].append(epoch + 1)\n\n    def on_epoch_end(self, epoch, logs=None, metric=None):\n        self.trainer.silly_callback[\"end\"].append(epoch + 1)\n</pre> class SillyCallback(Callback):     def on_train_begin(self, logs=None):         # recordings will be the trainer object attributes         self.trainer.silly_callback = {}          self.trainer.silly_callback[\"beginning\"] = []         self.trainer.silly_callback[\"end\"] = []      def on_epoch_begin(self, epoch, logs=None):         self.trainer.silly_callback[\"beginning\"].append(epoch + 1)      def on_epoch_end(self, epoch, logs=None, metric=None):         self.trainer.silly_callback[\"end\"].append(epoch + 1) <p>and now, as usual:</p> In\u00a0[22]: Copied! <pre>trainer = Trainer(\n    model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback]\n)\n</pre> trainer = Trainer(     model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback] ) In\u00a0[23]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:06&lt;00:00, 94.39it/s, loss=0.411, metrics={'acc': 0.814}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 121.91it/s, loss=0.327, metrics={'acc': 0.8449}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:07&lt;00:00, 85.39it/s, loss=0.324, metrics={'acc': 0.8495}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.68it/s, loss=0.298, metrics={'acc': 0.8612}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 74.35it/s, loss=0.302, metrics={'acc': 0.8593}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 100.51it/s, loss=0.29, metrics={'acc': 0.8665}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 73.83it/s, loss=0.292, metrics={'acc': 0.8637}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 105.98it/s, loss=0.286, metrics={'acc': 0.8695}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 72.15it/s, loss=0.286, metrics={'acc': 0.866}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.27it/s, loss=0.284, metrics={'acc': 0.8698}]\n</pre> In\u00a0[24]: Copied! <pre>trainer.silly_callback\n</pre> trainer.silly_callback Out[24]: <pre>{'beginning': [1, 2, 3, 4, 5], 'end': [1, 2, 3, 4, 5]}</pre>"},{"location":"examples/07_custom_components.html#custom-components","title":"Custom components\u00b6","text":"<p>As I mentioned earlier in the example notebooks, and also in the <code>README</code>, it is possible to customise almost every component in <code>pytorch-widedeep</code>.</p> <p>Let's now go through a couple of simple examples to illustrate how that could be done.</p> <p>First let's load and process the data \"as usual\", let's start with a regression and the airbnb dataset.</p>"},{"location":"examples/07_custom_components.html#custom-deeptext","title":"Custom <code>deeptext</code>\u00b6","text":"<p>Standard Pytorch model</p>"},{"location":"examples/07_custom_components.html#custom-loss-function","title":"Custom loss function\u00b6","text":"<p>Loss functions must simply inherit pytorch's <code>nn.Module</code>. For example, let's say we want to use <code>RMSE</code> (note that this is already available in the package, but I will pass it here as a custom loss for illustration purposes)</p>"},{"location":"examples/07_custom_components.html#custom-metric","title":"Custom metric\u00b6","text":"<p>Let's say we want to use our own accuracy metric (again, this is already available in the package, but I will pass it here as a custom loss for illustration purposes).</p> <p>This could be done as:</p>"},{"location":"examples/07_custom_components.html#custom-callback","title":"Custom Callback\u00b6","text":"<p>Let's code a callback that records the current epoch at the beginning and the end of each epoch (silly, but you know, this is just an example)</p>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html","title":"08_custom_dataLoader_imbalanced_dataset","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\n# drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n\ndf_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n\ncontinuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> df = load_bio_kdd04(as_frame=True) # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)  df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 )  continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[3]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[4]: Copied! <pre># Define the model\ninput_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n\ndeeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=hidden_layers,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> # Define the model input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist()  deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols,     mlp_hidden_dims=hidden_layers, ) model = WideDeep(deeptabular=deeptabular) model Out[4]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[5]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\n\n# # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision],\n    verbose=1,\n)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False)  # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1)  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)  # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision],     verbose=1, ) In\u00a0[6]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=1,\n    batch_size=32,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n\npd.DataFrame(trainer.history)\n\ndf_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=1,     batch_size=32,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) )  pd.DataFrame(trainer.history)  df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 325/325 [00:02&lt;00:00, 153.99it/s, loss=0.163, metrics={'acc': 0.9363, 'prec': [0.9358]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:02&lt;00:00, 205.93it/s, loss=0.1, metrics={'acc': 0.9501, 'prec': [0.1447]}]\n</pre> <pre>Training time[s]: 0:00:04\n</pre> <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:01&lt;00:00, 368.16it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.95      0.97     14446\n           1       0.15      0.95      0.25       130\n\n    accuracy                           0.95     14576\n   macro avg       0.57      0.95      0.61     14576\nweighted avg       0.99      0.95      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13736,   840]))\n</pre>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#custom-dataloader-for-imbalanced-dataset","title":"Custom DataLoader for Imbalanced dataset\u00b6","text":""},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/09_extracting_embeddings.html","title":"09_extracting_embeddings","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import FTTransformer, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep import Tab2Vec\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import FTTransformer, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep import Tab2Vec from pytorch_widedeep.datasets import load_adult In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"educational_num\"], axis=1, inplace=True)\n\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"educational_num\"], axis=1, inplace=True)  df.head() Out[3]: age workclass fnlwgt education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_cols, cont_cols = [], []\nfor col in df.columns:\n    # 50 is just a random number I choose here for this example\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":\n        cat_cols.append(col)\n    elif col != \"target\":\n        cont_cols.append(col)\ntarget_col = \"target\"\n</pre> cat_cols, cont_cols = [], [] for col in df.columns:     # 50 is just a random number I choose here for this example     if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":         cat_cols.append(col)     elif col != \"target\":         cont_cols.append(col) target_col = \"target\" In\u00a0[5]: Copied! <pre>target = df[target_col].values\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> target = df[target_col].values  tab_preprocessor = TabPreprocessor(     embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[6]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    n_blocks=2,\n    n_heads=4,\n    input_dim=16,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     n_blocks=2,     n_heads=4,     input_dim=16, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/utils/general_utils.py:12: DeprecationWarning: The 'embed_continuous' parameter is deprecated and will be removed in the next release. Please use 'embed_continuous_method' instead See the documentation for more details.\n  return func(*args, **kwargs)\n</pre> In\u00a0[7]: Copied! <pre>model = WideDeep(deeptabular=ft_transformer)\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2)\n</pre> model = WideDeep(deeptabular=ft_transformer) trainer = Trainer(model, objective=\"binary\", metrics=[Accuracy]) trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 41.47it/s, loss=221, metrics={'acc': 0.686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 86.24it/s, loss=9.28, metrics={'acc': 0.76}]\n</pre> In\u00a0[8]: Copied! <pre>t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor)\n</pre> t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor) In\u00a0[9]: Copied! <pre># assuming is a test set with target col\nX_vec, y = t2v.transform(df.sample(100), target_col=\"target\")\n</pre> # assuming is a test set with target col X_vec, y = t2v.transform(df.sample(100), target_col=\"target\") In\u00a0[10]: Copied! <pre># X vec is the dataframe turned into the embeddings\nX_vec.shape\n</pre> # X vec is the dataframe turned into the embeddings X_vec.shape Out[10]: <pre>(100, 208)</pre> <p><code>208 = input_dim (16) * n_cols (13)</code></p> In\u00a0[11]: Copied! <pre># ...or if we don't have target col\nX_vec = t2v.transform(df.sample(100))\n</pre> # ...or if we don't have target col X_vec = t2v.transform(df.sample(100))"},{"location":"examples/09_extracting_embeddings.html#extracting-embeddings","title":"Extracting embeddings\u00b6","text":"<p>This notebook is a simple guide to extracting learned feature embeddings using Tab2Vec</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html","title":"10_3rd_party_integration-RayTune_WnB","text":"In\u00a0[11]: Copied! <pre>from typing import Optional, Dict\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport wandb\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.callbacks import (\n    EarlyStopping,\n    ModelCheckpoint,\n    Callback,\n)\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\nfrom ray import tune\nfrom ray.tune.schedulers import AsyncHyperBandScheduler\nfrom ray.tune import JupyterNotebookReporter\nfrom ray.air.integrations.wandb import WandbLoggerCallback\n\n# from ray.tune.integration.wandb import wandb_mixin\n\nimport tracemalloc\n\ntracemalloc.start()\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> from typing import Optional, Dict import os  import numpy as np import pandas as pd import torch import wandb from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.callbacks import (     EarlyStopping,     ModelCheckpoint,     Callback, ) from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  from ray import tune from ray.tune.schedulers import AsyncHyperBandScheduler from ray.tune import JupyterNotebookReporter from ray.air.integrations.wandb import WandbLoggerCallback  # from ray.tune.integration.wandb import wandb_mixin  import tracemalloc  tracemalloc.start()  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[12]: Copied! <pre>class RayTuneReporter(Callback):\n    r\"\"\"Callback that allows reporting history and lr_history values to RayTune\n    during Hyperparameter tuning\n\n    Callbacks are passed as input parameters to the ``Trainer`` class. See\n    :class:`pytorch_widedeep.trainer.Trainer`\n\n    For examples see the examples folder at:\n\n        .. code-block:: bash\n\n            /examples/12_HyperParameter_tuning_w_RayTune.ipynb\n    \"\"\"\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        report_dict = {}\n        for k, v in self.trainer.history.items():\n            report_dict.update({k: v[-1]})\n        if hasattr(self.trainer, \"lr_history\"):\n            for k, v in self.trainer.lr_history.items():\n                report_dict.update({k: v[-1]})\n        tune.report(report_dict)\n\n\nclass WnBReportBest(Callback):\n    r\"\"\"Callback that allows reporting best performance of a run to WnB\n    during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint\n    with added WnB and removed checkpoint saving.\n\n    Callbacks are passed as input parameters to the ``Trainer`` class.\n\n    Parameters\n    ----------\n    wb: obj\n        Weights&amp;Biases API interface to report single best result usable for\n        comparisson of multiple paramater combinations by, for example,\n        `parallel coordinates\n        &lt;https://docs.wandb.ai/ref/app/features/panels/parallel-coordinates&gt;`_.\n        E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically `'val_loss'` or metric name\n        (e.g. `'val_acc'`)\n    mode: str, default=\"auto\"\n        If ``save_best_only=True``, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For `'acc'`, this should be `'max'`, for\n        `'loss'` this should be `'min'`, etc. In `'auto'` mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        wb: object,\n        monitor: str = \"val_loss\",\n        mode: str = \"auto\",\n    ):\n        super(WnBReportBest, self).__init__()\n\n        self.monitor = monitor\n        self.mode = mode\n        self.wb = wb\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"WnBReportBest mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if self._is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is not None:\n            if self.monitor_op(current, self.best):\n                self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]\n                self.best = current\n                self.best_epoch = epoch\n\n    @staticmethod\n    def _is_metric(monitor: str):\n        \"copied from pytorch_widedeep.callbacks\"\n        if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):\n            return True\n        else:\n            return False\n</pre> class RayTuneReporter(Callback):     r\"\"\"Callback that allows reporting history and lr_history values to RayTune     during Hyperparameter tuning      Callbacks are passed as input parameters to the ``Trainer`` class. See     :class:`pytorch_widedeep.trainer.Trainer`      For examples see the examples folder at:          .. code-block:: bash              /examples/12_HyperParameter_tuning_w_RayTune.ipynb     \"\"\"      def on_epoch_end(         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         report_dict = {}         for k, v in self.trainer.history.items():             report_dict.update({k: v[-1]})         if hasattr(self.trainer, \"lr_history\"):             for k, v in self.trainer.lr_history.items():                 report_dict.update({k: v[-1]})         tune.report(report_dict)   class WnBReportBest(Callback):     r\"\"\"Callback that allows reporting best performance of a run to WnB     during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint     with added WnB and removed checkpoint saving.      Callbacks are passed as input parameters to the ``Trainer`` class.      Parameters     ----------     wb: obj         Weights&amp;Biases API interface to report single best result usable for         comparisson of multiple paramater combinations by, for example,         `parallel coordinates         `_.         E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.     monitor: str, default=\"loss\"         quantity to monitor. Typically `'val_loss'` or metric name         (e.g. `'val_acc'`)     mode: str, default=\"auto\"         If ``save_best_only=True``, the decision to overwrite the current save         file is made based on either the maximization or the minimization of         the monitored quantity. For `'acc'`, this should be `'max'`, for         `'loss'` this should be `'min'`, etc. In `'auto'` mode, the         direction is automatically inferred from the name of the monitored         quantity.      \"\"\"      def __init__(         self,         wb: object,         monitor: str = \"val_loss\",         mode: str = \"auto\",     ):         super(WnBReportBest, self).__init__()          self.monitor = monitor         self.mode = mode         self.wb = wb          if self.mode not in [\"auto\", \"min\", \"max\"]:             warnings.warn(                 \"WnBReportBest mode %s is unknown, \"                 \"fallback to auto mode.\" % (self.mode),                 RuntimeWarning,             )             self.mode = \"auto\"         if self.mode == \"min\":             self.monitor_op = np.less             self.best = np.Inf         elif self.mode == \"max\":             self.monitor_op = np.greater  # type: ignore[assignment]             self.best = -np.Inf         else:             if self._is_metric(self.monitor):                 self.monitor_op = np.greater  # type: ignore[assignment]                 self.best = -np.Inf             else:                 self.monitor_op = np.less                 self.best = np.Inf      def on_epoch_end(  # noqa: C901         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         logs = logs or {}         current = logs.get(self.monitor)         if current is not None:             if self.monitor_op(current, self.best):                 self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]                 self.best = current                 self.best_epoch = epoch      @staticmethod     def _is_metric(monitor: str):         \"copied from pytorch_widedeep.callbacks\"         if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):             return True         else:             return False In\u00a0[13]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[13]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[14]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[14]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[15]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[16]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[17]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[18]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[19]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[20]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular) model Out[20]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre># Metrics from torchmetrics\naccuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\")\nprecision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\")\nf1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\")\nrecall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\")\n</pre> # Metrics from torchmetrics accuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\") precision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\") f1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\") recall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\") <p>Note:</p> <p>Following cells includes usage of both <code>RayTuneReporter</code> and <code>WnBReportBest</code> callbacks. In case you want to use just <code>RayTuneReporter</code>, remove following:</p> <ul> <li>wandb from config</li> <li><code>WandbLoggerCallback</code></li> <li><code>WnBReportBest</code></li> <li><code>@wandb_mixin</code> decorator</li> </ul> <p>We do not see strong reason to use WnB without RayTune for a single paramater combination run, but it is possible:</p> <ul> <li>option01: define paramaters in config only for a single value <code>tune.grid_search([1000])</code> (single value RayTune run)</li> <li>option02: define WnB callback that reports currnet validation/training loss, metrics, etc. at the end of batch, ie. do not report to WnB at <code>epoch_end</code> as in <code>WnBReportBest</code> but at the <code>on_batch_end</code>, see <code>pytorch_widedeep.callbacks.Callback</code></li> </ul> In\u00a0[26]: Copied! <pre>config = {\n    \"batch_size\": tune.grid_search([1000, 5000]),\n    \"wandb\": {\n        \"project\": \"test\",\n        # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",\n        \"api_key\": \"WNB_API_KEY\",\n    },\n}\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\n\n@wandb_mixin\ndef training_function(config, X_train, X_val):\n    early_stopping = EarlyStopping()\n    model_checkpoint = ModelCheckpoint(save_best_only=True)\n    # Hyperparameters\n    batch_size = config[\"batch_size\"]\n    trainer = Trainer(\n        model,\n        objective=\"binary_focal_loss\",\n        callbacks=[\n            RayTuneReporter,\n            WnBReportBest(wb=wandb),\n            early_stopping,\n            model_checkpoint,\n        ],\n        lr_schedulers={\"deeptabular\": deep_sch},\n        initializers={\"deeptabular\": XavierNormal},\n        optimizers={\"deeptabular\": deep_opt},\n        metrics=[accuracy, precision, recall, f1],\n        verbose=0,\n    )\n\n    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)\n\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n\nasha_scheduler = AsyncHyperBandScheduler(\n    time_attr=\"training_iteration\",\n    metric=\"_metric/val_loss\",\n    mode=\"min\",\n    max_t=100,\n    grace_period=10,\n    reduction_factor=3,\n    brackets=1,\n)\n\nanalysis = tune.run(\n    tune.with_parameters(training_function, X_train=X_train, X_val=X_val),\n    resources_per_trial={\"cpu\": 1, \"gpu\": 0},\n    progress_reporter=JupyterNotebookReporter(overwrite=True),\n    scheduler=asha_scheduler,\n    config=config,\n    callbacks=[\n        WandbLoggerCallback(\n            project=config[\"wandb\"][\"project\"],\n            # api_key_file=config[\"wandb\"][\"api_key_file\"],\n            api_key=config[\"wandb\"][\"api_key\"],\n            log_config=True,\n        )\n    ],\n)\n</pre> config = {     \"batch_size\": tune.grid_search([1000, 5000]),     \"wandb\": {         \"project\": \"test\",         # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",         \"api_key\": \"WNB_API_KEY\",     }, }  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)   @wandb_mixin def training_function(config, X_train, X_val):     early_stopping = EarlyStopping()     model_checkpoint = ModelCheckpoint(save_best_only=True)     # Hyperparameters     batch_size = config[\"batch_size\"]     trainer = Trainer(         model,         objective=\"binary_focal_loss\",         callbacks=[             RayTuneReporter,             WnBReportBest(wb=wandb),             early_stopping,             model_checkpoint,         ],         lr_schedulers={\"deeptabular\": deep_sch},         initializers={\"deeptabular\": XavierNormal},         optimizers={\"deeptabular\": deep_opt},         metrics=[accuracy, precision, recall, f1],         verbose=0,     )      trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)   X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}  asha_scheduler = AsyncHyperBandScheduler(     time_attr=\"training_iteration\",     metric=\"_metric/val_loss\",     mode=\"min\",     max_t=100,     grace_period=10,     reduction_factor=3,     brackets=1, )  analysis = tune.run(     tune.with_parameters(training_function, X_train=X_train, X_val=X_val),     resources_per_trial={\"cpu\": 1, \"gpu\": 0},     progress_reporter=JupyterNotebookReporter(overwrite=True),     scheduler=asha_scheduler,     config=config,     callbacks=[         WandbLoggerCallback(             project=config[\"wandb\"][\"project\"],             # api_key_file=config[\"wandb\"][\"api_key_file\"],             api_key=config[\"wandb\"][\"api_key\"],             log_config=True,         )     ], ) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmp60pfyl1kwandb'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpnjv2rg1wwandb-artifacts'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpgebu5k1kwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpxy9y2yriwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n</pre> In\u00a0[14]: Copied! <pre>analysis.results\n</pre> analysis.results Out[14]: <pre>{'fc9a8_00000': {'_metric': {'train_loss': 0.006297602537127896,\n   'train_Accuracy': 0.9925042986869812,\n   'train_Precision': 0.9939393997192383,\n   'train_Recall': 0.15814851224422455,\n   'train_F1Score': 0.2728785574436188,\n   'val_loss': 0.005045663565397263,\n   'val_Accuracy': 0.9946483969688416,\n   'val_Precision': 1.0,\n   'val_Recall': 0.39534884691238403,\n   'val_F1Score': 0.5666667222976685},\n  'time_this_iter_s': 2.388202428817749,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00000',\n  'experiment_id': 'baad1d4f3d924b48b9ece1b9f26c80cc',\n  'date': '2022-07-31_14-06-51',\n  'timestamp': 1659276411,\n  'time_total_s': 12.656474113464355,\n  'pid': 1813,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 1000},\n  'time_since_restore': 12.656474113464355,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.8006253242492676,\n  'experiment_tag': '0_batch_size=1000'},\n 'fc9a8_00001': {'_metric': {'train_loss': 0.02519632239515583,\n   'train_Accuracy': 0.9910891652107239,\n   'train_Precision': 0.25,\n   'train_Recall': 0.0009643201483413577,\n   'train_F1Score': 0.0019212296465411782,\n   'val_loss': 0.02578434906899929,\n   'val_Accuracy': 0.9911492466926575,\n   'val_Precision': 0.0,\n   'val_Recall': 0.0,\n   'val_F1Score': 0.0},\n  'time_this_iter_s': 4.113586902618408,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00001',\n  'experiment_id': 'f2e54a6a5780429fbf0db0746853347e',\n  'date': '2022-07-31_14-06-56',\n  'timestamp': 1659276416,\n  'time_total_s': 12.926990509033203,\n  'pid': 1962,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 5000},\n  'time_since_restore': 12.926990509033203,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.9253025054931641,\n  'experiment_tag': '1_batch_size=5000'}}</pre> <p>Using Weights and Biases logging you can create parallel coordinates graphs that map parametr combinations to the best(lowest) loss achieved during the training of the networks</p> <p></p> <p>local visualization of raytune reults using tensorboard</p> In\u00a0[23]: Copied! <pre>%load_ext tensorboard\n%tensorboard --logdir ~/ray_results\n</pre> %load_ext tensorboard %tensorboard --logdir ~/ray_results"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#3rd-party-integration-raytune-weights-biases","title":"3rd party integration - RayTune, Weights &amp; Biases\u00b6","text":"<p>This notebook provides guideline for integration of external library functions in the model training process through <code>Callback</code> objects, a popular concept of using objects as arguments for other objects.</p> <p>[DISCLAIMER]</p> <p>We show integration of RayTune (a hyperparameter tuning framework) and Weights &amp; Biases (ML projects experiment tracking and versioning solution) in the <code>pytorch_widedeep</code> model training process. We did not include <code>RayTuneReporter</code> and <code>WnBReportBest</code> in the library code to minimize the dependencies on other libraries that are not directly included in the model design and training.</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/11_auc_multiclass.html","title":"11_auc_multiclass","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import AUROC\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_ecoli\nfrom pytorch_widedeep.utils import LabelEncoder\n\nfrom sklearn.model_selection import train_test_split\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import AUROC from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_ecoli from pytorch_widedeep.utils import LabelEncoder  from sklearn.model_selection import train_test_split  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_ecoli(as_frame=True)\ndf.head()\n</pre> df = load_ecoli(as_frame=True) df.head() Out[2]: SequenceName mcg gvh lip chg aac alm1 alm2 class 0 AAT_ECOLI 0.49 0.29 0.48 0.5 0.56 0.24 0.35 cp 1 ACEA_ECOLI 0.07 0.40 0.48 0.5 0.54 0.35 0.44 cp 2 ACEK_ECOLI 0.56 0.40 0.48 0.5 0.49 0.37 0.46 cp 3 ACKA_ECOLI 0.59 0.49 0.48 0.5 0.52 0.45 0.36 cp 4 ADI_ECOLI 0.23 0.32 0.48 0.5 0.55 0.25 0.35 cp In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"class\"].value_counts()\n</pre> # imbalance of the classes df[\"class\"].value_counts() Out[3]: <pre>class\ncp     143\nim      77\npp      52\nimU     35\nom      20\nomL      5\nimS      2\nimL      2\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre>df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])]\ndf.reset_index(inplace=True, drop=True)\n</pre> df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])] df.reset_index(inplace=True, drop=True) In\u00a0[5]: Copied! <pre>encoder = LabelEncoder([\"class\"])\ndf_enc = encoder.fit_transform(df)\ndf_enc[\"class\"] = df_enc[\"class\"] - 1\n</pre> encoder = LabelEncoder([\"class\"]) df_enc = encoder.fit_transform(df) df_enc[\"class\"] = df_enc[\"class\"] - 1 In\u00a0[6]: Copied! <pre># drop columns we won't need in this example\ndf_enc = df_enc.drop(columns=[\"SequenceName\"])\n</pre> # drop columns we won't need in this example df_enc = df_enc.drop(columns=[\"SequenceName\"]) In\u00a0[7]: Copied! <pre>df_train, df_valid = train_test_split(\n    df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1 ) In\u00a0[8]: Copied! <pre>continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist()\n</pre> continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist() In\u00a0[9]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"class\"].values\ny_valid = df_valid[\"class\"].values\ny_test = df_test[\"class\"].values\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"class\"].values y_valid = df_valid[\"class\"].values y_test = df_test[\"class\"].values  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n  self._check_inputs(cat_embed_cols)\n</pre> In\u00a0[10]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique())\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique()) model Out[10]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=7, out_features=200, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=200, out_features=100, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=100, out_features=5, bias=True)\n  )\n)</pre> In\u00a0[11]: Copied! <pre>auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\")\n</pre> auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\") In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"multiclass_focal_loss\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[auroc],\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"multiclass_focal_loss\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[auroc], )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 54.59it/s, loss=0.109, metrics={'MulticlassAUROC': 0.314}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 98.35it/s, loss=0.105, metrics={'MulticlassAUROC': 0.2558}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 91.55it/s, loss=0.105, metrics={'MulticlassAUROC': 0.3546}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 111.68it/s, loss=0.101, metrics={'MulticlassAUROC': 0.2737}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 62.55it/s, loss=0.1, metrics={'MulticlassAUROC': 0.3795}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 108.51it/s, loss=0.0966, metrics={'MulticlassAUROC': 0.3053}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 99.35it/s, loss=0.0965, metrics={'MulticlassAUROC': 0.3809}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 117.73it/s, loss=0.0962, metrics={'MulticlassAUROC': 0.3089}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 110.56it/s, loss=0.0967, metrics={'MulticlassAUROC': 0.3509}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 127.35it/s, loss=0.0958, metrics={'MulticlassAUROC': 0.3089}]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/11_auc_multiclass.html#auc-multiclass-computation","title":"AUC multiclass computation\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html","title":"12_ZILNLoss_origkeras_vs_pytorch_widedeep","text":"In\u00a0[1]: Copied! <pre># @title Copyright 2019 The Lifetime Value Authors.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n</pre> # @title Copyright 2019 The Lifetime Value Authors. # Licensed under the Apache License, Version 2.0 (the \"License\"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an \"AS IS\" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ Run in Google Colab View source on GitHub In\u00a0[3]: Copied! <pre>import os\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nfrom typing import Sequence\n\n# install and import ltv\n!pip install -q git+https://github.com/google/lifetime_value\nimport lifetime_value as ltv\n</pre> import os  import numpy as np import pandas as pd from scipy import stats import matplotlib.pyplot as plt import seaborn as sns import tensorflow as tf import tensorflow_probability as tfp from typing import Sequence  # install and import ltv !pip install -q git+https://github.com/google/lifetime_value import lifetime_value as ltv In\u00a0[\u00a0]: Copied! <pre>tfd = tfp.distributions\n%config InlineBackend.figure_format='retina'\nsns.set_style(\"whitegrid\")\n</pre> tfd = tfp.distributions %config InlineBackend.figure_format='retina' sns.set_style(\"whitegrid\") In\u00a0[\u00a0]: Copied! <pre>MODEL = \"dnn\"\nLOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln']\nLEARNING_RATE = 0.001  # @param { isTemplate: true}\nVERSION = 0  # @param { isTemplate: true, type: 'integer'}\nOUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'}\n</pre> MODEL = \"dnn\" LOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln'] LEARNING_RATE = 0.001  # @param { isTemplate: true} VERSION = 0  # @param { isTemplate: true, type: 'integer'} OUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'} <p>Download kdd_cup_98 data to /tmp/lifetime-value/kdd_cup_98</p> In\u00a0[\u00a0]: Copied! <pre>%%bash\nmkdir -p /tmp/lifetime-value/kdd_cup_98\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/\ncd /tmp/lifetime-value/kdd_cup_98/\nunzip cup98lrn.zip\nunzip cup98val.zip\n</pre> %%bash mkdir -p /tmp/lifetime-value/kdd_cup_98 wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/ cd /tmp/lifetime-value/kdd_cup_98/ unzip cup98lrn.zip unzip cup98val.zip In\u00a0[\u00a0]: Copied! <pre>df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\")\nnum_train = df_train.shape[0]\ndf_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\")\ndf_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\")\ndf_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\")\n</pre> df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\") num_train = df_train.shape[0] df_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\") df_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\") df_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\") In\u00a0[\u00a0]: Copied! <pre>df = pd.concat([df_train, df_eval], axis=0, sort=True)\n</pre> df = pd.concat([df_train, df_eval], axis=0, sort=True) In\u00a0[\u00a0]: Copied! <pre>y = df[\"TARGET_D\"][:num_train]\n</pre> y = df[\"TARGET_D\"][:num_train] In\u00a0[\u00a0]: Copied! <pre>def plot_hist_log_scale(y):\n    max_val = y.max() + 1.0\n    ax = pd.Series(y).hist(\n        figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)\n    )\n\n    plt.xlabel(\"Donation ($)\")\n    plt.ylabel(\"Count\")\n    # plt.title('Histogram of LTV')\n    plt.xticks(rotation=\"horizontal\")\n    plt.legend(loc=\"upper left\")\n    ax.set_xscale(\"log\")\n    ax.grid(False)\n    # Hide the right and top spines\n    ax.spines[\"right\"].set_visible(False)\n    ax.spines[\"top\"].set_visible(False)\n    # Only show ticks on the left and bottom spines\n    ax.yaxis.set_ticks_position(\"left\")\n    ax.xaxis.set_ticks_position(\"bottom\")\n    plt.show()\n\n    fig = ax.get_figure()\n    output_file = tf.io.gfile.GFile(\n        \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"\n    )\n    fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\")\n</pre> def plot_hist_log_scale(y):     max_val = y.max() + 1.0     ax = pd.Series(y).hist(         figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)     )      plt.xlabel(\"Donation ($)\")     plt.ylabel(\"Count\")     # plt.title('Histogram of LTV')     plt.xticks(rotation=\"horizontal\")     plt.legend(loc=\"upper left\")     ax.set_xscale(\"log\")     ax.grid(False)     # Hide the right and top spines     ax.spines[\"right\"].set_visible(False)     ax.spines[\"top\"].set_visible(False)     # Only show ticks on the left and bottom spines     ax.yaxis.set_ticks_position(\"left\")     ax.xaxis.set_ticks_position(\"bottom\")     plt.show()      fig = ax.get_figure()     output_file = tf.io.gfile.GFile(         \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"     )     fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\") In\u00a0[\u00a0]: Copied! <pre>plot_hist_log_scale(y[y &gt; 0])\n</pre> plot_hist_log_scale(y[y &gt; 0]) In\u00a0[\u00a0]: Copied! <pre>VOCAB_FEATURES = [\n    \"ODATEDW\",  # date of donor's first gift (YYMM)\n    \"OSOURCE\",  # donor acquisition mailing list\n    \"TCODE\",  # donor title code\n    \"STATE\",\n    \"ZIP\",\n    \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood\n    \"CLUSTER\",  # socio-economic status\n    \"GENDER\",\n    \"MAXADATE\",  # date of the most recent promotion received\n    \"MINRDATE\",\n    \"LASTDATE\",\n    \"FISTDATE\",\n    \"RFA_2A\",\n]\n</pre> VOCAB_FEATURES = [     \"ODATEDW\",  # date of donor's first gift (YYMM)     \"OSOURCE\",  # donor acquisition mailing list     \"TCODE\",  # donor title code     \"STATE\",     \"ZIP\",     \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood     \"CLUSTER\",  # socio-economic status     \"GENDER\",     \"MAXADATE\",  # date of the most recent promotion received     \"MINRDATE\",     \"LASTDATE\",     \"FISTDATE\",     \"RFA_2A\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\")\ndf[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x))\ndf[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5)\ndf[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\")\ndf[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\")\ndf[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\")\ndf[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\")\n</pre> df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\") df[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x)) df[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5) df[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\") df[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\") df[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\") df[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\") In\u00a0[\u00a0]: Copied! <pre>def label_encoding(y, frequency_threshold=100):\n    value_counts = pd.value_counts(y)\n    categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()\n    # 0 indicates the unknown category.\n    return pd.Categorical(y, categories=categories).codes + 1\n</pre> def label_encoding(y, frequency_threshold=100):     value_counts = pd.value_counts(y)     categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()     # 0 indicates the unknown category.     return pd.Categorical(y, categories=categories).codes + 1 In\u00a0[\u00a0]: Copied! <pre>for key in VOCAB_FEATURES:\n    df[key] = label_encoding(df[key])\n</pre> for key in VOCAB_FEATURES:     df[key] = label_encoding(df[key]) In\u00a0[\u00a0]: Copied! <pre>MAIL_ORDER_RESPONSES = [\n    \"MBCRAFT\",\n    \"MBGARDEN\",\n    \"MBBOOKS\",\n    \"MBCOLECT\",\n    \"MAGFAML\",\n    \"MAGFEM\",\n    \"MAGMALE\",\n    \"PUBGARDN\",\n    \"PUBCULIN\",\n    \"PUBHLTH\",\n    \"PUBDOITY\",\n    \"PUBNEWFN\",\n    \"PUBPHOTO\",\n    \"PUBOPP\",\n    \"RFA_2F\",\n]\n</pre> MAIL_ORDER_RESPONSES = [     \"MBCRAFT\",     \"MBGARDEN\",     \"MBBOOKS\",     \"MBCOLECT\",     \"MAGFAML\",     \"MAGFEM\",     \"MAGMALE\",     \"PUBGARDN\",     \"PUBCULIN\",     \"PUBHLTH\",     \"PUBDOITY\",     \"PUBNEWFN\",     \"PUBPHOTO\",     \"PUBOPP\",     \"RFA_2F\", ] In\u00a0[\u00a0]: Copied! <pre>INDICATOR_FEATURES = [\n    \"AGE\",  # age decile, 0 indicates unknown\n    \"NUMCHLD\",\n    \"INCOME\",\n    \"WEALTH1\",\n    \"HIT\",\n] + MAIL_ORDER_RESPONSES\n</pre> INDICATOR_FEATURES = [     \"AGE\",  # age decile, 0 indicates unknown     \"NUMCHLD\",     \"INCOME\",     \"WEALTH1\",     \"HIT\", ] + MAIL_ORDER_RESPONSES In\u00a0[\u00a0]: Copied! <pre>df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1\ndf[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1)\ndf[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes\n\nfor col in MAIL_ORDER_RESPONSES:\n    df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1\n</pre> df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1 df[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1) df[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes  for col in MAIL_ORDER_RESPONSES:     df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1 In\u00a0[\u00a0]: Copied! <pre>NUMERIC_FEATURES = [\n    # binary\n    \"MAILCODE\",  # bad address\n    \"NOEXCH\",  # do not exchange\n    \"RECINHSE\",  # donor has given to PVA's in house program\n    \"RECP3\",  # donor has given to PVA's P3 program\n    \"RECPGVG\",  # planned giving record\n    \"RECSWEEP\",  # sweepstakes record\n    \"HOMEOWNR\",  # home owner\n    \"CHILD03\",\n    \"CHILD07\",\n    \"CHILD12\",\n    \"CHILD18\",\n    # continuous\n    \"CARDPROM\",\n    \"NUMPROM\",\n    \"CARDPM12\",\n    \"NUMPRM12\",\n    \"RAMNTALL\",\n    \"NGIFTALL\",\n    \"MINRAMNT\",\n    \"MAXRAMNT\",\n    \"LASTGIFT\",\n    \"AVGGIFT\",\n]\n</pre> NUMERIC_FEATURES = [     # binary     \"MAILCODE\",  # bad address     \"NOEXCH\",  # do not exchange     \"RECINHSE\",  # donor has given to PVA's in house program     \"RECP3\",  # donor has given to PVA's P3 program     \"RECPGVG\",  # planned giving record     \"RECSWEEP\",  # sweepstakes record     \"HOMEOWNR\",  # home owner     \"CHILD03\",     \"CHILD07\",     \"CHILD12\",     \"CHILD18\",     # continuous     \"CARDPROM\",     \"NUMPROM\",     \"CARDPM12\",     \"NUMPRM12\",     \"RAMNTALL\",     \"NGIFTALL\",     \"MINRAMNT\",     \"MAXRAMNT\",     \"LASTGIFT\",     \"AVGGIFT\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\")\ndf[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\")\ndf[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\")\ndf[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\")\ndf[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\")\ndf[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\")\ndf[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\")\ndf[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\")\ndf[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\n\ndf[\"CARDPROM\"] = df[\"CARDPROM\"] / 100\ndf[\"NUMPROM\"] = df[\"NUMPROM\"] / 100\ndf[\"CARDPM12\"] = df[\"CARDPM12\"] / 100\ndf[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100\ndf[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"])\ndf[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"])\ndf[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"])\ndf[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"])\ndf[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"])\ndf[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"])\n</pre> df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\") df[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\") df[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\") df[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\") df[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\") df[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\") df[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\") df[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\") df[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")  df[\"CARDPROM\"] = df[\"CARDPROM\"] / 100 df[\"NUMPROM\"] = df[\"NUMPROM\"] / 100 df[\"CARDPM12\"] = df[\"CARDPM12\"] / 100 df[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100 df[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"]) df[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"]) df[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"]) df[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"]) df[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"]) df[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"]) In\u00a0[\u00a0]: Copied! <pre>CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES\nALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES\n</pre> CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES ALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES In\u00a0[\u00a0]: Copied! <pre>def dnn_split(df):\n    df_train = df.iloc[:num_train]\n    df_eval = df.iloc[num_train:]\n\n    def feature_dict(df):\n        features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}\n        features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values\n        return features\n\n    x_train, y_train = (\n        feature_dict(df_train),\n        df_train[\"TARGET_D\"].astype(\"float32\").values,\n    )\n    x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values\n\n    return x_train, x_eval, y_train, y_eval\n</pre> def dnn_split(df):     df_train = df.iloc[:num_train]     df_eval = df.iloc[num_train:]      def feature_dict(df):         features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}         features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values         return features      x_train, y_train = (         feature_dict(df_train),         df_train[\"TARGET_D\"].astype(\"float32\").values,     )     x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values      return x_train, x_eval, y_train, y_eval In\u00a0[\u00a0]: Copied! <pre>def embedding_dim(x):\n    return int(x**0.25) + 1\n\n\ndef embedding_layer(vocab_size):\n    return tf.keras.Sequential(\n        [\n            tf.keras.layers.Embedding(\n                input_dim=vocab_size,\n                output_dim=embedding_dim(vocab_size),\n                input_length=1,\n            ),\n            tf.keras.layers.Flatten(),\n        ]\n    )\n\n\ndef dnn_model(output_units):\n    numeric_input = tf.keras.layers.Input(\n        shape=(len(NUMERIC_FEATURES),), name=\"numeric\"\n    )\n\n    embedding_inputs = [\n        tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)\n        for key in CATEGORICAL_FEATURES\n    ]\n\n    embedding_outputs = [\n        embedding_layer(vocab_size=df[key].max() + 1)(input)\n        for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)\n    ]\n\n    deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)\n    deep_model = tf.keras.Sequential(\n        [\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(units=output_units),\n        ]\n    )\n    return tf.keras.Model(\n        inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)\n    )\n</pre> def embedding_dim(x):     return int(x**0.25) + 1   def embedding_layer(vocab_size):     return tf.keras.Sequential(         [             tf.keras.layers.Embedding(                 input_dim=vocab_size,                 output_dim=embedding_dim(vocab_size),                 input_length=1,             ),             tf.keras.layers.Flatten(),         ]     )   def dnn_model(output_units):     numeric_input = tf.keras.layers.Input(         shape=(len(NUMERIC_FEATURES),), name=\"numeric\"     )      embedding_inputs = [         tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)         for key in CATEGORICAL_FEATURES     ]      embedding_outputs = [         embedding_layer(vocab_size=df[key].max() + 1)(input)         for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)     ]      deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)     deep_model = tf.keras.Sequential(         [             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(units=output_units),         ]     )     return tf.keras.Model(         inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)     ) In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    loss = tf.keras.losses.MeanSquaredError()\n    output_units = 1\n\nif LOSS == \"ziln\":\n    loss = ltv.zero_inflated_lognormal_loss\n    output_units = 3\n</pre> if LOSS == \"mse\":     loss = tf.keras.losses.MeanSquaredError()     output_units = 1  if LOSS == \"ziln\":     loss = ltv.zero_inflated_lognormal_loss     output_units = 3 In\u00a0[\u00a0]: Copied! <pre>x_train, x_eval, y_train, y_eval = dnn_split(df)\nmodel = dnn_model(output_units)\n</pre> x_train, x_eval, y_train, y_eval = dnn_split(df) model = dnn_model(output_units) In\u00a0[\u00a0]: Copied! <pre>model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss)\n</pre> model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss) In\u00a0[\u00a0]: Copied! <pre>callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),\n    tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10),\n]\n</pre> callbacks = [     tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),     tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10), ] In\u00a0[\u00a0]: Copied! <pre>history = model.fit(\n    x=x_train,\n    y=y_train,\n    batch_size=2048,\n    epochs=200,\n    verbose=2,\n    callbacks=callbacks,\n    validation_data=(x_eval, y_eval),\n).history\n</pre> history = model.fit(     x=x_train,     y=y_train,     batch_size=2048,     epochs=200,     verbose=2,     callbacks=callbacks,     validation_data=(x_eval, y_eval), ).history In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    y_pred = model.predict(x=x_eval, batch_size=1024).flatten()\n\nif LOSS == \"ziln\":\n    logits = model.predict(x=x_eval, batch_size=1024)\n    y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten()\n</pre> if LOSS == \"mse\":     y_pred = model.predict(x=x_eval, batch_size=1024).flatten()  if LOSS == \"ziln\":     logits = model.predict(x=x_eval, batch_size=1024)     y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten() In\u00a0[\u00a0]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom torch.optim import NAdam\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from torch.optim.lr_scheduler import ReduceLROnPlateau from pytorch_widedeep.callbacks import EarlyStopping from torch.optim import NAdam In\u00a0[\u00a0]: Copied! <pre># CATEGORICAL_FEATURES\nNUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)]\nx_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"])\nx_train_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_train.items() if key not in [\"numeric\"]}\n)\n\nx_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"])\nx_eval_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_eval.items() if key not in [\"numeric\"]}\n)\n</pre> # CATEGORICAL_FEATURES NUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)] x_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"]) x_train_pyt_cat = pd.DataFrame(     {key: value for key, value in x_train.items() if key not in [\"numeric\"]} )  x_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"]) x_eval_pyt_cat = pd.DataFrame(     {key: value for key, value in x_eval.items() if key not in [\"numeric\"]} ) In\u00a0[\u00a0]: Copied! <pre>x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1)\nx_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1)\n</pre> x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1) x_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1) In\u00a0[\u00a0]: Copied! <pre>embed_input = [\n    (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES\n]\n</pre> embed_input = [     (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES ] In\u00a0[\u00a0]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=embed_input,\n    continuous_cols=NUMERICAL_FEATURES,\n    shared_embed=False,\n    scale=False,\n)\nX_tab_train = tab_preprocessor.fit_transform(x_train_pyt)\nX_tab_valid = tab_preprocessor.transform(x_eval_pyt)\nX_tab_test = tab_preprocessor.transform(x_eval_pyt)\n\n# target\ny_train = y_train\ny_valid = y_eval\ny_test = y_train\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\nX_test = {\"X_tab\": X_tab_test}\n\ndeeptabular = TabMlp(\n    mlp_hidden_dims=[128, 128, 64, 64],\n    column_idx=tab_preprocessor.column_idx,\n    embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=3)\n\ndeep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE)\ncallbacks = [EarlyStopping()]\ndeep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)\n\nobjective = \"ziln\"\n\ntrainer = Trainer(\n    model,\n    callbacks=callbacks,\n    lr_schedulers={\"deeptabular\": deep_sch},\n    objective=objective,\n    optimizers={\"deeptabular\": deep_opt},\n)\n\ntrainer.fit(\n    X_train=X_train,\n    X_val=X_val,\n    n_epochs=200,\n    batch_size=2048,\n)\n\ny_pred_pytorch = trainer.predict(X_test=X_test)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=embed_input,     continuous_cols=NUMERICAL_FEATURES,     shared_embed=False,     scale=False, ) X_tab_train = tab_preprocessor.fit_transform(x_train_pyt) X_tab_valid = tab_preprocessor.transform(x_eval_pyt) X_tab_test = tab_preprocessor.transform(x_eval_pyt)  # target y_train = y_train y_valid = y_eval y_test = y_train  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} X_test = {\"X_tab\": X_tab_test}  deeptabular = TabMlp(     mlp_hidden_dims=[128, 128, 64, 64],     column_idx=tab_preprocessor.column_idx,     embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, )  model = WideDeep(deeptabular=deeptabular, pred_dim=3)  deep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE) callbacks = [EarlyStopping()] deep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)  objective = \"ziln\"  trainer = Trainer(     model,     callbacks=callbacks,     lr_schedulers={\"deeptabular\": deep_sch},     objective=objective,     optimizers={\"deeptabular\": deep_opt}, )  trainer.fit(     X_train=X_train,     X_val=X_val,     n_epochs=200,     batch_size=2048, )  y_pred_pytorch = trainer.predict(X_test=X_test) In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>from sklearn.metrics import mean_squared_error\n\nmean_squared_error(y_pred, y_pred_pytorch)\n</pre> from sklearn.metrics import mean_squared_error  mean_squared_error(y_pred, y_pred_pytorch) In\u00a0[\u00a0]: Copied! <pre>unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0]\n</pre> unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0] In\u00a0[\u00a0]: Copied! <pre>num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs]\nnum_mailed\n</pre> num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs] num_mailed In\u00a0[\u00a0]: Copied! <pre>baseline_total_profit = np.sum(y_eval - 0.68)\nbaseline_total_profit\n</pre> baseline_total_profit = np.sum(y_eval - 0.68) baseline_total_profit In\u00a0[\u00a0]: Copied! <pre>total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs]\ntotal_profits\n</pre> total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs] total_profits In\u00a0[\u00a0]: Copied! <pre>gain = pd.DataFrame(\n    {\n        \"lorenz\": ltv.cumulative_true(y_eval, y_eval),\n        \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),\n        \"model\": ltv.cumulative_true(y_eval, y_pred),\n    }\n)\n</pre> gain = pd.DataFrame(     {         \"lorenz\": ltv.cumulative_true(y_eval, y_eval),         \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),         \"model\": ltv.cumulative_true(y_eval, y_pred),     } ) In\u00a0[\u00a0]: Copied! <pre>num_customers = np.float32(gain.shape[0])\ngain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers\n</pre> num_customers = np.float32(gain.shape[0]) gain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers In\u00a0[\u00a0]: Copied! <pre>ax = gain[\n    [\n        \"cumulative_customer\",\n        \"lorenz\",\n        \"baseline\",\n        \"model\",\n    ]\n].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)\n\nax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")\n\nax.set_xlabel(\"Cumulative Fraction of Customers\")\nax.set_xticks(np.arange(0, 1.1, 0.1))\nax.set_xlim((0, 1.0))\n\nax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\")\nax.set_yticks(np.arange(0, 1.1, 0.1))\nax.set_ylim((0, 1.05))\nax.set_title(\"Gain Chart\");\n</pre> ax = gain[     [         \"cumulative_customer\",         \"lorenz\",         \"baseline\",         \"model\",     ] ].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)  ax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")  ax.set_xlabel(\"Cumulative Fraction of Customers\") ax.set_xticks(np.arange(0, 1.1, 0.1)) ax.set_xlim((0, 1.0))  ax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\") ax.set_yticks(np.arange(0, 1.1, 0.1)) ax.set_ylim((0, 1.05)) ax.set_title(\"Gain Chart\"); In\u00a0[\u00a0]: Copied! <pre>gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]])\ngini\n</pre> gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]]) gini In\u00a0[\u00a0]: Copied! <pre>df_decile = ltv.decile_stats(y_eval, y_pred)\ndf_decile\n</pre> df_decile = ltv.decile_stats(y_eval, y_pred) df_decile In\u00a0[\u00a0]: Copied! <pre>ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)\n\nax.set_title(\"Decile Chart\")\nax.set_xlabel(\"Prediction bucket\")\nax.set_ylabel(\"Average bucket value\")\nax.legend([\"Label\", \"Prediction\"], loc=\"upper left\");\n</pre> ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)  ax.set_title(\"Decile Chart\") ax.set_xlabel(\"Prediction bucket\") ax.set_ylabel(\"Average bucket value\") ax.legend([\"Label\", \"Prediction\"], loc=\"upper left\"); In\u00a0[\u00a0]: Copied! <pre>def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:\n    \"\"\"Calculates spearmanr rank correlation coefficient.\n\n    See https://docs.scipy.org/doc/scipy/reference/stats.html.\n\n    Args:\n      x1: 1D array_like.\n      x2: 1D array_like.\n\n    Returns:\n      correlation: float.\n    \"\"\"\n    return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]\n\n\nspearman_corr = spearmanr(y_eval, y_pred)\nspearman_corr\n</pre> def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:     \"\"\"Calculates spearmanr rank correlation coefficient.      See https://docs.scipy.org/doc/scipy/reference/stats.html.      Args:       x1: 1D array_like.       x2: 1D array_like.      Returns:       correlation: float.     \"\"\"     return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]   spearman_corr = spearmanr(y_eval, y_pred) spearman_corr In\u00a0[\u00a0]: Copied! <pre>df_metrics = pd.DataFrame(\n    {\n        \"model\": MODEL,\n        \"loss_function\": LOSS,\n        \"train_loss\": history[\"loss\"][-1],\n        \"eval_loss\": history[\"val_loss\"][-1],\n        \"label_positive\": np.mean(y_eval &gt; 0),\n        \"label_mean\": y_eval.mean(),\n        \"pred_mean\": y_pred.mean(),\n        \"decile_mape\": df_decile[\"decile_mape\"].mean(),\n        \"baseline_gini\": gini[\"normalized\"][1],\n        \"gini\": gini[\"normalized\"][2],\n        \"spearman_corr\": spearman_corr,\n    },\n    index=[VERSION],\n)\n</pre> df_metrics = pd.DataFrame(     {         \"model\": MODEL,         \"loss_function\": LOSS,         \"train_loss\": history[\"loss\"][-1],         \"eval_loss\": history[\"val_loss\"][-1],         \"label_positive\": np.mean(y_eval &gt; 0),         \"label_mean\": y_eval.mean(),         \"pred_mean\": y_pred.mean(),         \"decile_mape\": df_decile[\"decile_mape\"].mean(),         \"baseline_gini\": gini[\"normalized\"][1],         \"gini\": gini[\"normalized\"][2],         \"spearman_corr\": spearman_corr,     },     index=[VERSION], ) In\u00a0[\u00a0]: Copied! <pre>for unit_cost, total_profit in zip(unit_costs, total_profits):\n    df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit\n</pre> for unit_cost, total_profit in zip(unit_costs, total_profits):     df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit In\u00a0[\u00a0]: Copied! <pre>df_metrics.T\n</pre> df_metrics.T In\u00a0[\u00a0]: Copied! <pre>output_path = OUTPUT_CSV_FOLDER\n</pre> output_path = OUTPUT_CSV_FOLDER In\u00a0[\u00a0]: Copied! <pre>if not os.path.isdir(output_path):\n    os.makedirs(output_path)\n</pre> if not os.path.isdir(output_path):     os.makedirs(output_path) In\u00a0[\u00a0]: Copied! <pre>output_file = os.path.join(\n    output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION)\n)\n</pre> output_file = os.path.join(     output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION) ) In\u00a0[\u00a0]: Copied! <pre>df_metrics.to_csv(output_file, index=False)\n</pre> df_metrics.to_csv(output_file, index=False)"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#zilnloss","title":"ZILNLoss\u00b6","text":"<p>[DISCLAIMER]</p> <p>Purpose of this notebook is to check if ZILNloss implemented originaly Keras give same results in pytorch-widedeep implemenatation</p>"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#kdd-cup-98-ltv-prediction","title":"KDD Cup 98 LTV Prediction\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#configs","title":"Configs\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#load-data","title":"Load data\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#label-distribution","title":"Label distribution\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#preprocess-features","title":"Preprocess features\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#vocab","title":"Vocab\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#indicator","title":"Indicator\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#numeric","title":"Numeric\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all","title":"All\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#traineval-split","title":"Train/eval split\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#model","title":"Model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#loss","title":"Loss\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#train","title":"Train\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#eval","title":"Eval\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#pytorch-widedeep-approach","title":"Pytorch-widedeep approach\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#appendix","title":"Appendix\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#total-profit","title":"Total Profit\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#gini-coefficient","title":"Gini Coefficient\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#calibration","title":"Calibration\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#rank-correlation","title":"Rank Correlation\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all-metrics-together","title":"All metrics together\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#save","title":"Save\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html","title":"13_model_uncertainty_prediction","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[2]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[3]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[5]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[6]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[7]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[8]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[9]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=1)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=1) model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre># # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n# f1 = F1_torchmetrics(average=None, num_classes=1)\n# recall = Recall_torchmetrics(average=None, num_classes=1)\n</pre> # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1) # f1 = F1_torchmetrics(average=None, num_classes=1) # recall = Recall_torchmetrics(average=None, num_classes=1) In\u00a0[11]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\nrecall = Recall(average=True)\nf1 = F1Score(average=False)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False) recall = Recall(average=True) f1 = F1Score(average=False) In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision, recall, f1],\n    verbose=1,\n)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision, recall, f1],     verbose=1, ) In\u00a0[13]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=3,\n    batch_size=50,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=3,     batch_size=50,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 130.75it/s, loss=0.187, metrics={'acc': 0.9214, 'prec': [0.9149], 'rec': 0.9318, 'f1': [0.9233]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 173.71it/s, loss=0.106, metrics={'acc': 0.9499, 'prec': [0.1435], 'rec': 0.938, 'f1': [0.249]}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 139.68it/s, loss=0.109, metrics={'acc': 0.9559, 'prec': [0.9537], 'rec': 0.9572, 'f1': [0.9554]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 177.53it/s, loss=0.0888, metrics={'acc': 0.9602, 'prec': [0.1755], 'rec': 0.9457, 'f1': [0.2961]}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 141.63it/s, loss=0.08, metrics={'acc': 0.9706, 'prec': [0.9648], 'rec': 0.9766, 'f1': [0.9707]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 165.36it/s, loss=0.0969, metrics={'acc': 0.9564, 'prec': [0.1636], 'rec': 0.9535, 'f1': [0.2792]}]</pre> <pre>Training time[s]: 0:00:10\n</pre> <pre>\n</pre> In\u00a0[14]: Copied! <pre>pd.DataFrame(trainer.history)\n</pre> pd.DataFrame(trainer.history) Out[14]: train_loss train_acc train_prec train_rec train_f1 val_loss val_acc val_prec val_rec val_f1 0 0.186707 0.921408 [0.9149412512779236] 0.931801 [0.9232940673828125] 0.106023 0.949914 [0.14353498816490173] 0.937984 [0.24897116422653198] 1 0.109498 0.955931 [0.9536514282226562] 0.957193 [0.9554190039634705] 0.088787 0.960206 [0.17553956806659698] 0.945736 [0.29611650109291077] 2 0.079979 0.970588 [0.9648183584213257] 0.976582 [0.9706646203994751] 0.096858 0.956432 [0.1635638326406479] 0.953488 [0.279228150844574] In\u00a0[15]: Copied! <pre>df_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:00&lt;00:00, 346.55it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13845,   731]))\n</pre> In\u00a0[16]: Copied! <pre>df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\n    \"Actual predicted values:\\n{}\".format(\n        np.unique(df_pred_unc[:, -1], return_counts=True)\n    )\n)\n</pre> df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(     \"Actual predicted values:\\n{}\".format(         np.unique(df_pred_unc[:, -1], return_counts=True)     ) ) <pre>predict_UncertaintyIter: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:03&lt;00:00,  3.25it/s]</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0.]), array([14576]))\n</pre> <pre>\n</pre> In\u00a0[17]: Copied! <pre>df_pred_unc\n</pre> df_pred_unc Out[17]: <pre>array([[9.98401165e-01, 1.59881881e-03, 0.00000000e+00],\n       [9.99941409e-01, 5.85634953e-05, 0.00000000e+00],\n       [9.97351170e-01, 2.64881272e-03, 0.00000000e+00],\n       ...,\n       [9.99494374e-01, 5.05603210e-04, 0.00000000e+00],\n       [9.99981642e-01, 1.83574630e-05, 0.00000000e+00],\n       [9.99996483e-01, 3.52600046e-06, 0.00000000e+00]])</pre>"},{"location":"examples/13_model_uncertainty_prediction.html#model-uncertainty-prediction","title":"Model Uncertainty prediction\u00b6","text":"<p>Note:</p> <p>This notebook extends the \"Custom DataLoader for Imbalanced dataset\" notebook</p>"},{"location":"examples/13_model_uncertainty_prediction.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#normal-prediction","title":"\"Normal\" prediction\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#prediction-using-uncertainty","title":"Prediction using uncertainty\u00b6","text":""},{"location":"examples/14_bayesian_models.html","title":"14_bayesian_models","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp\nfrom pytorch_widedeep.training.bayesian_trainer import BayesianTrainer\n</pre> import numpy as np import torch import pandas as pd  from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score  from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor from pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp from pytorch_widedeep.training.bayesian_trainer import BayesianTrainer <p>The first few things to do we know them very well, like with any other model described in any of the other notebooks</p> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"age_buckets\"] = pd.cut(\n    df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)\n)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"age_buckets\"] = pd.cut(     df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9) ) df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country age_buckets income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 3 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 4 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 0 In\u00a0[3]: Copied! <pre>train, test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> train, test = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>wide_cols = [\n    \"age_buckets\",\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native_country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n\ntarget = train[\"income_label\"].values\n</pre> wide_cols = [     \"age_buckets\",     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native_country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"]  target = train[\"income_label\"].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_tab = wide_preprocessor.fit_transform(train)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_tab = wide_preprocessor.fit_transform(train) In\u00a0[6]: Copied! <pre>model = BayesianWide(\n    input_dim=np.unique(X_tab).shape[0],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified\n)\n</pre> model = BayesianWide(     input_dim=np.unique(X_tab).shape[0],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified ) In\u00a0[7]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[8]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 124.32it/s, loss=163, metrics={'acc': 0.7813}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 238.67it/s, loss=141, metrics={'acc': 0.8219}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 132.81it/s, loss=140, metrics={'acc': 0.8285}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 190.16it/s, loss=140, metrics={'acc': 0.8298}]\n</pre> In\u00a0[9]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(train)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(train) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[10]: Copied! <pre>model = BayesianTabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    #     embed_continuous_method = \"standard\",\n    #     cont_embed_activation=\"leaky_relu\",\n    #     cont_embed_dim = 8,\n    mlp_hidden_dims=[128, 64],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,\n)\n</pre> model = BayesianTabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     #     embed_continuous_method = \"standard\",     #     cont_embed_activation=\"leaky_relu\",     #     cont_embed_dim = 8,     mlp_hidden_dims=[128, 64],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1, ) In\u00a0[11]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 28.74it/s, loss=2e+3, metrics={'acc': 0.8007}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 136.89it/s, loss=1.75e+3, metrics={'acc': 0.8418}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 29.41it/s, loss=1.73e+3, metrics={'acc': 0.8596}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 143.87it/s, loss=1.71e+3, metrics={'acc': 0.8569}]\n</pre> <p>These models are powerful beyond the success metrics because they give us a sense of uncertainty as we predict. Let's have a look</p> In\u00a0[13]: Copied! <pre>X_tab_test = tab_preprocessor.transform(test)\n</pre> X_tab_test = tab_preprocessor.transform(test) In\u00a0[14]: Copied! <pre>preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5)\n</pre> preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 33.92it/s]\n</pre> In\u00a0[15]: Copied! <pre>preds.shape\n</pre> preds.shape Out[15]: <pre>(5, 9769)</pre> <p>as we can see the prediction have shape <code>(5, 9769)</code>, one set of predictions each time we have internally run predict (i.e. sample the network and predict, defined by the parameter <code>n_samples</code>). This gives us an idea of how certain the model is about a certain prediction.</p> <p>Similarly, we could obtain the probabilities</p> In\u00a0[16]: Copied! <pre>probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5)\n</pre> probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 32.79it/s]\n</pre> In\u00a0[17]: Copied! <pre>probs.shape\n</pre> probs.shape Out[17]: <pre>(5, 9769, 2)</pre> <p>And we could see how the model performs each time we sampled the network</p> In\u00a0[18]: Copied! <pre>for p in preds:\n    print(accuracy_score(p, test[\"income_label\"].values))\n</pre> for p in preds:     print(accuracy_score(p, test[\"income_label\"].values)) <pre>0.8559729757395844\n0.8564847988535162\n0.8567918927218753\n0.8562800696079435\n0.8558706111167981\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/14_bayesian_models.html#the-bayesian-models","title":"The Bayesian Models\u00b6","text":"<p>Perhaps one of the most interesting functionality in the library is the access to full Bayesian models in almost exactly the same way one would use any of the other models in the library.</p> <p>Note however that the Bayesian models are ONLY available for tabular data and, at the moment, we do not support combining them to form a Wide and Deep model.</p> <p>The implementation in this library is based on the publication: Weight Uncertainty in Neural Networks, by Blundell et al., 2015. Code-wise, our implementation is inspired by a number of source:</p> <ol> <li>https://joshfeldman.net/WeightUncertainty/</li> <li>https://www.nitarshan.com/bayes-by-backprop/</li> <li>https://github.com/piEsposito/blitz-bayesian-deep-learning</li> <li>https://github.com/zackchase/mxnet-the-straight-dope/tree/master/chapter18_variational-methods-and-uncertainty</li> </ol> <p>The two Bayesian models available in the library are:</p> <ol> <li>BayesianWide: this is a linear model where the non-linearities are captured via crossed-columns</li> <li>BayesianMLP: this is a standard MLP that receives categorical embeddings and continuous cols (embedded or not) which are the passed through a series of dense layers. All parameters in the model are probabilistic.</li> </ol>"},{"location":"examples/14_bayesian_models.html#1-bayesianwide","title":"1. <code>BayesianWide</code>\u00b6","text":""},{"location":"examples/14_bayesian_models.html#2-bayesiantabmlp","title":"2. <code>BayesianTabMlp</code>\u00b6","text":""},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html","title":"15 Self Supervised Pretraning pt1","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) In\u00a0[3]: Copied! <pre># one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>df_tr.head(2)\n</pre> df_tr.head(2) Out[4]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 9042 26 Local-gov 250551 HS-grad 9 Married-civ-spouse Craft-repair Own-child Black Male 0 0 40 United-States 0 25322 50 Private 34832 Bachelors 13 Married-civ-spouse Tech-support Husband White Male 15024 0 40 United-States 1 In\u00a0[5]: Copied! <pre># As always, we need to define which cols will be represented as embeddings\n# and which one will be continuous features\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n</pre> # As always, we need to define which cols will be represented as embeddings # and which one will be continuous features cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" In\u00a0[6]: Copied! <pre># We prepare the data to be passed to the model\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> # We prepare the data to be passed to the model tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>X_tab[:5]\n</pre> X_tab[:5] Out[7]: <pre>array([[ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1, 26, 40],\n       [ 2,  2,  1,  2,  2,  2,  1,  2,  1,  1, 50, 40],\n       [ 2,  1,  1,  3,  2,  2,  1,  1,  2,  1, 39, 46],\n       [ 2,  3,  2,  4,  1,  2,  2,  1,  1,  1, 17, 10],\n       [ 3,  4,  2,  1,  1,  2,  1,  1,  1,  1, 32, 20]])</pre> In\u00a0[8]: Copied! <pre># We define a model that will act as the encoder in the encoder/decoder\n# architecture. This could be any of: TabMlp, TabResnet or TabNet\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n</pre> # We define a model that will act as the encoder in the encoder/decoder # architecture. This could be any of: TabMlp, TabResnet or TabNet tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, ) In\u00a0[9]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[9]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[10]: Copied! <pre># If we do not pass a custom decoder, which is perfectly possible via the\n# decoder param,  the EncoderDecoderTrainer will automatically build a\n# decoder which will be the 'mirror' image of the encoder\nencoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp)\n</pre> # If we do not pass a custom decoder, which is perfectly possible via the # decoder param,  the EncoderDecoderTrainer will automatically build a # decoder which will be the 'mirror' image of the encoder encoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp) In\u00a0[11]: Copied! <pre># let's have a look to the encoder_decoder_model (aka ed_model)\nencoder_decoder_trainer.ed_model\n</pre> # let's have a look to the encoder_decoder_model (aka ed_model) encoder_decoder_trainer.ed_model Out[11]: <pre>EncoderDecoderModel(\n  (encoder): TabMlp(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n        (emb_layer_education): Embedding(17, 8, padding_idx=0)\n        (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n        (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n        (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n        (emb_layer_race): Embedding(6, 4, padding_idx=0)\n        (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n        (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n        (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n        (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.0, inplace=False)\n    )\n    (cont_norm): Identity()\n    (encoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=95, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (decoder): TabMlpDecoder(\n    (decoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=95, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (masker): RandomObfuscator()\n)</pre> <p>Ignoring the <code>masker</code>, which just...well...masks, the <code>ed_model</code> consists of:</p> <ol> <li>An encoder model that is a <code>TabMlp</code> model that is in itself comprised by an Embedding layer (or rather a collection of them, referred as <code>cat_and_cont_embed</code>) and an encoder (a simple MLP, referred as <code>encoder</code>)</li> <li>A decoder which is just an \"inverted\" MLP (referred as <code>decoder</code>)</li> </ol> In\u00a0[12]: Copied! <pre># And we just...pretrain\nencoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # And we just...pretrain encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.90it/s, loss=4.07]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 89.87it/s, loss=3.09]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.86it/s, loss=2.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.24it/s, loss=2.09]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.38it/s, loss=1.78]\n</pre> <p>At this point we have two options, we could either save the model for later use or we could continue to supervised training. The latter is rather simple, after running:</p> <pre>encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> <p>you just have to</p> <pre>model = WideDeep(deeptabular=tab_mlp)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\n</pre> <p>Let's say that in any case, we are 'decent' scientists/people and we want to save the model:</p> In\u00a0[13]: Copied! <pre>encoder_decoder_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\"\n)\n</pre> encoder_decoder_trainer.save(     path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\" ) <p>some time has passed...</p> In\u00a0[14]: Copied! <pre>encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\")\n</pre> encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\") <p>Now, AND THIS IS IMPORTANT We have loaded the encoder AND the decoder. To proceed to the supervised training we ONLY need the encoder</p> In\u00a0[15]: Copied! <pre>pretrained_encoder = encoder_decoder_model.encoder\n</pre> pretrained_encoder = encoder_decoder_model.encoder In\u00a0[16]: Copied! <pre>pretrained_encoder\n</pre> pretrained_encoder Out[16]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[17]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.04it/s, loss=0.374, metrics={'acc': 0.8253}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 85.63it/s, loss=0.324, metrics={'acc': 0.8491}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 87.56it/s, loss=0.301, metrics={'acc': 0.8608}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 73.38it/s, loss=0.29, metrics={'acc': 0.8655}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 78.68it/s, loss=0.284, metrics={'acc': 0.8686}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 173.02it/s]\n</pre> <pre>0.8730678677449074\n</pre> <p>As we mentioned before, we can also use a <code>TabResNet</code> or <code>TabNet</code> model and a custom decoder. Let's have a look:</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder\n</pre> from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder In\u00a0[19]: Copied! <pre>resnet_encoder = TabResnetEncoder(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 100, 100],\n)\n</pre> resnet_encoder = TabResnetEncoder(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     blocks_dims=[200, 100, 100], ) <p>let's have a look to the model</p> In\u00a0[20]: Copied! <pre>resnet_encoder\n</pre> resnet_encoder Out[20]: <pre>TabResnet(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n      (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      (block_0): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=False)\n          (1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=200, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n)</pre> <p>As we can see, the tensor we are trying to reconstruct, the embeddings, is of size <code>94</code> (this number is stored in the attribute: <code>esnet_encoder.cat_and_cont_embed.output_dim</code>), with that information we could build or own decoder as:</p> In\u00a0[21]: Copied! <pre># for all possible params see the docs\nresnet_decoder = TabResnetDecoder(\n    embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,\n    blocks_dims=[100, 100, 200],\n)\n</pre> # for all possible params see the docs resnet_decoder = TabResnetDecoder(     embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,     blocks_dims=[100, 100, 200], ) In\u00a0[22]: Copied! <pre>resnet_decoder\n</pre> resnet_decoder Out[22]: <pre>TabResnetDecoder(\n  (decoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (block_0): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=False)\n          (1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=100, out_features=200, bias=False)\n        (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=200, out_features=200, bias=False)\n        (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n  (reconstruction_layer): Linear(in_features=200, out_features=95, bias=False)\n)</pre> <p>and now:</p> In\u00a0[23]: Copied! <pre>ec_trainer = EncoderDecoderTrainer(\n    encoder=resnet_encoder,\n    decoder=resnet_decoder,\n    masked_prob=0.2,\n)\nec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> ec_trainer = EncoderDecoderTrainer(     encoder=resnet_encoder,     decoder=resnet_decoder,     masked_prob=0.2, ) ec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.89it/s, loss=1.52]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.78it/s, loss=0.81]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 39.82it/s, loss=0.56]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.73it/s, loss=0.417]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.24it/s, loss=0.329]\n</pre> In\u00a0[24]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=resnet_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=resnet_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.63it/s, loss=0.335, metrics={'acc': 0.8442}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.02it/s, loss=0.296, metrics={'acc': 0.864}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.91it/s, loss=0.283, metrics={'acc': 0.8687}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.00it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 51.95it/s, loss=0.272, metrics={'acc': 0.8732}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 120.15it/s]\n</pre> <pre>0.8725560446309756\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 1st of the two approaches (the 'TabNet approach'). For details on the second approach please see <code>16_Self_Supervised_Pretraning_pt2</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervision-for-non-transformer-based-models","title":"Self Supervision for non-transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns can have all different dimensions. In this library, these are: <code>TabMlp</code>, <code>TabResNet</code> and <code>TabNet</code></p> <p>As shown in Figure, this is an encoder-encoder approach where we learn to predict values in the incoming data that have been masked. However, as I mentioned before, our implementation is a bit different, and the masking occurs in th embedding space.</p> <p>Nonetheless, the code below illustrates how to use this encoder-decoder approach with <code>pytorch-widedeep</code></p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html","title":"15 Self Supervised Pretraning pt2","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import WideDeep, FTTransformer\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import (\n    ContrastiveDenoisingTrainer,\n)\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import WideDeep, FTTransformer from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import (     ContrastiveDenoisingTrainer, ) In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n\n# one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    with_attention=True,\n    with_cls_token=True,  # this is optional\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True)  # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\"  tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     with_attention=True,     with_cls_token=True,  # this is optional ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[3]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    input_dim=32,\n    kv_compression_factor=0.5,\n    n_blocks=3,\n    n_heads=4,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     input_dim=32,     kv_compression_factor=0.5,     n_blocks=3,     n_heads=4, ) In\u00a0[4]: Copied! <pre># for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs.\n# Note that using these params involves some knowledge of the routine and the architecture of the model used\ncontrastive_denoising_trainer = ContrastiveDenoisingTrainer(\n    model=ft_transformer,\n    preprocessor=tab_preprocessor,\n)\ncontrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs. # Note that using these params involves some knowledge of the routine and the architecture of the model used contrastive_denoising_trainer = ContrastiveDenoisingTrainer(     model=ft_transformer,     preprocessor=tab_preprocessor, ) contrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:13&lt;00:00, 11.73it/s, loss=579]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.56it/s, loss=143]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.49it/s, loss=141]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 12.77it/s, loss=138]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 13.29it/s, loss=137]\n</pre> In\u00a0[5]: Copied! <pre>contrastive_denoising_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\"\n)\n</pre> contrastive_denoising_trainer.save(     path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\" ) <p>some time has passed</p> In\u00a0[6]: Copied! <pre># some time has passed, we load the model with torch as usual:\ncontrastive_denoising_model = torch.load(\n    \"pretrained_weights/contrastive_denoising_model.pt\"\n)\n</pre> # some time has passed, we load the model with torch as usual: contrastive_denoising_model = torch.load(     \"pretrained_weights/contrastive_denoising_model.pt\" ) <p>NOW, AND THIS IS IMPORTANT! We have loaded the entire contrastive, denoising model. To proceed to the supervised training we ONLY need the attention-based model, which is the 'model' attribute of the trainer, let's have a look</p> In\u00a0[7]: Copied! <pre>contrastive_denoising_model.model\n</pre> contrastive_denoising_model.model Out[7]: <pre>FTTransformer(\n  (cat_embed): SameSizeCatEmbeddings(\n    (embed): Embedding(323, 32, padding_idx=0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): Sequential(\n    (fttransformer_block0): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block1): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block2): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n  )\n)</pre> In\u00a0[8]: Copied! <pre>pretrained_model = contrastive_denoising_model.model\n</pre> pretrained_model = contrastive_denoising_model.model In\u00a0[9]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_model)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_model) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  # And, you know...we get a test metric X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.19it/s, loss=0.383, metrics={'acc': 0.8176}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.08it/s, loss=0.325, metrics={'acc': 0.8502}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.56it/s, loss=0.306, metrics={'acc': 0.8601}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.41it/s, loss=0.295, metrics={'acc': 0.8641}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:06&lt;00:00, 24.70it/s, loss=0.289, metrics={'acc': 0.8656}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 97.26it/s]</pre> <pre>0.8695874705701709\n</pre> <pre>\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 2nd of the two approaches (the 'SAINT approach'). For details on the 1st approach (the 'TabNet' approach) please see <code>16_Self_Supervised_Pretraning_pt1</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervision-transformer-based-models","title":"Self Supervision transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns have all the same dimensions. In this library, these are:</p> <ul> <li>TabTransformer</li> <li>FTTransformer</li> <li>SAINT</li> <li>TabFastFormer</li> </ul> <p>Note that there is one additional Transformer-based model, the <code>TabPerceiver</code>, however this is a \"particular\" model and at the moment we do not support self supervision for it, but it will come.</p> <p>Let see at one example using the <code>FTTransformer</code>.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html","title":"16 Usign a custom hugging face model","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport lightgbm as lgb\nfrom lightgbm import Dataset as lgbDataset\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.metrics import (\n    f1_score,\n    recall_score,\n    accuracy_score,\n    precision_score,\n    confusion_matrix,\n)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom torch import Tensor, nn\nfrom transformers import DistilBertModel, DistilBertTokenizer\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.utils import Tokenizer, LabelEncoder\nfrom pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.utils.fastai_transforms import (\n    fix_html,\n    spec_add_spaces,\n    rm_useless_spaces,\n)\n</pre> import numpy as np import torch import lightgbm as lgb from lightgbm import Dataset as lgbDataset from scipy.sparse import hstack, csr_matrix from sklearn.metrics import (     f1_score,     recall_score,     accuracy_score,     precision_score,     confusion_matrix, ) from sklearn.model_selection import train_test_split from sklearn.feature_extraction.text import TfidfVectorizer  from torch import Tensor, nn from transformers import DistilBertModel, DistilBertTokenizer from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.utils import Tokenizer, LabelEncoder from pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.utils.fastai_transforms import (     fix_html,     spec_add_spaces,     rm_useless_spaces, ) <p>Let's load the data and have a look:</p> In\u00a0[2]: Copied! <pre>df = load_womens_ecommerce(as_frame=True)\n\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> df = load_womens_ecommerce(as_frame=True)  df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 0 767 33 None Absolutely wonderful - silky and sexy and comf... 2 1 0 Initmates Intimate Intimates 1 1080 34 None Love this dress!  it's sooo pretty.  i happene... 3 1 4 General Dresses Dresses 2 1077 60 Some major design flaws I had such high hopes for this dress and reall... 1 0 0 General Dresses Dresses 3 1049 50 My favorite buy! I love, love, love this jumpsuit. it's fun, fl... 3 1 0 General Petite Bottoms Pants 4 847 47 Flattering shirt This shirt is very flattering to all due to th... 3 1 6 General Tops Blouses <p>So, we will use the <code>review_text</code> column to predict the <code>rating</code>. Later on, we will try to combine it with some other columns (like <code>division_name</code> and <code>age</code>) see if these help.</p> <p>Let's first have a look to the distribution of ratings</p> In\u00a0[4]: Copied! <pre>df.rating.value_counts()\n</pre> df.rating.value_counts() Out[4]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> <p>This shows that we could have perhaps grouped rating scores of 1, 2 and 3 into 1...but anyway, let's just move on with those 4 classes.</p> <p>We are not going to carry any hyperparameter optimization here, so, we will only need a train and a test set (i.e.  no need of a validation set for the example in this notebook)</p> In\u00a0[5]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating) <p>Let's see what we have to beat. What metrics would we obtain if we always predict the most common rating (3)?</p> In\u00a0[6]: Copied! <pre>most_common_pred = [train.rating.value_counts().index[0]] * len(test)\n\nmost_common_acc = accuracy_score(test.rating, most_common_pred)\nmost_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\")\n</pre> most_common_pred = [train.rating.value_counts().index[0]] * len(test)  most_common_acc = accuracy_score(test.rating, most_common_pred) most_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\") In\u00a0[7]: Copied! <pre>print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\")\n</pre> print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\") <pre>Accuracy: 0.553516143299425. F1 Score: 0.3944344218301668\n</pre> <p>ok, these are our \"baseline\" metrics.</p> <p>Let's start by using simply tf-idf + lightGBM</p> In\u00a0[8]: Copied! <pre># ?Tokenizer\n</pre> # ?Tokenizer In\u00a0[9]: Copied! <pre># this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.\n\n# When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm)\n# that can only be solved by using only one CPU\ntok = Tokenizer(n_cpus=1)\ntok_reviews_tr = tok.process_all(train.review_text.tolist())\ntok_reviews_te = tok.process_all(test.review_text.tolist())\n</pre> # this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.  # When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm) # that can only be solved by using only one CPU tok = Tokenizer(n_cpus=1) tok_reviews_tr = tok.process_all(train.review_text.tolist()) tok_reviews_te = tok.process_all(test.review_text.tolist()) In\u00a0[10]: Copied! <pre>vectorizer = TfidfVectorizer(\n    max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5\n)\n\nX_text_tr = vectorizer.fit_transform(tok_reviews_tr)\nX_text_te = vectorizer.transform(tok_reviews_te)\n</pre> vectorizer = TfidfVectorizer(     max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5 )  X_text_tr = vectorizer.fit_transform(tok_reviews_tr) X_text_te = vectorizer.transform(tok_reviews_te) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n  warnings.warn(\n</pre> In\u00a0[11]: Copied! <pre>X_text_tr\n</pre> X_text_tr Out[11]: <pre>&lt;18086x4566 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 884074 stored elements in Compressed Sparse Row format&gt;</pre> <p>We now move our matrices to lightGBM <code>Dataset</code> format</p> In\u00a0[12]: Copied! <pre>lgbtrain_text = lgbDataset(\n    X_text_tr,\n    train.rating.values,\n    free_raw_data=False,\n)\n\nlgbtest_text = lgbDataset(\n    X_text_te,\n    test.rating.values,\n    reference=lgbtrain_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_text = lgbDataset(     X_text_tr,     train.rating.values,     free_raw_data=False, )  lgbtest_text = lgbDataset(     X_text_te,     test.rating.values,     reference=lgbtrain_text,     free_raw_data=False, ) <p>and off we go. By the way, I think as we run the next cell, we should appreciate how fast lightGBM runs. Yes, the input is a sparse matrix, but still, trains on 18086x4566 in a matter of secs</p> In\u00a0[\u00a0]: Copied! <pre>lgb_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_text,\n    valid_sets=[lgbtest_text, lgbtrain_text],\n    valid_names=[\"test\", \"train\"],\n)\n</pre> lgb_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_text,     valid_sets=[lgbtest_text, lgbtrain_text],     valid_names=[\"test\", \"train\"], ) In\u00a0[14]: Copied! <pre>preds_text = lgb_text_model.predict(X_text_te)\npred_text_class = np.argmax(preds_text, 1)\n</pre> preds_text = lgb_text_model.predict(X_text_te) pred_text_class = np.argmax(preds_text, 1) In\u00a0[15]: Copied! <pre>acc_text = accuracy_score(lgbtest_text.label, pred_text_class)\nf1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\")\ncm_text = confusion_matrix(lgbtest_text.label, pred_text_class)\n</pre> acc_text = accuracy_score(lgbtest_text.label, pred_text_class) f1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\") cm_text = confusion_matrix(lgbtest_text.label, pred_text_class) In\u00a0[16]: Copied! <pre>print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\")\n</pre> print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\") <pre>LightGBM Accuracy: 0.6444051304732419. LightGBM F1 Score: 0.617154488246181\n</pre> In\u00a0[17]: Copied! <pre>print(f\"LightGBM Confusion Matrix: \\n {cm_text}\")\n</pre> print(f\"LightGBM Confusion Matrix: \\n {cm_text}\") <pre>LightGBM Confusion Matrix: \n [[ 199  135   61   79]\n [ 123  169  149  123]\n [  30   94  279  578]\n [  16   30  190 2267]]\n</pre> <p>Ok, so, with no hyperparameter optimization lightGBM gets an accuracy of 0.64 and a F1 score of 0.62. This is significantly better than predicting always the most popular.</p> <p>Let's see if in this implementation, some additional features, like <code>age</code> or <code>class_name</code> are of any help</p> In\u00a0[18]: Copied! <pre>tab_cols = [\n    \"age\",\n    \"division_name\",\n    \"department_name\",\n    \"class_name\",\n]\n\nfor tab_df in [train, test]:\n    for c in [\"division_name\", \"department_name\", \"class_name\"]:\n        tab_df[c] = tab_df[c].str.lower()\n        tab_df[c].fillna(\"missing\", inplace=True)\n</pre> tab_cols = [     \"age\",     \"division_name\",     \"department_name\",     \"class_name\", ]  for tab_df in [train, test]:     for c in [\"division_name\", \"department_name\", \"class_name\"]:         tab_df[c] = tab_df[c].str.lower()         tab_df[c].fillna(\"missing\", inplace=True) In\u00a0[19]: Copied! <pre># This is our LabelEncoder. A class that is designed to work with the models in this library but\n# can be used for general purposes\nle = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"])\ntrain_tab_le = le.fit_transform(train)\ntest_tab_le = le.transform(test)\n</pre> # This is our LabelEncoder. A class that is designed to work with the models in this library but # can be used for general purposes le = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"]) train_tab_le = le.fit_transform(train) test_tab_le = le.transform(test) In\u00a0[20]: Copied! <pre>train_tab_le.head()\n</pre> train_tab_le.head() Out[20]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 4541 836 35 None Bought this on sale in my reg size- 10. im 5'9... 2 1 2 1 1 1 18573 1022 25 Look like \"mom jeans\" Maybe i just have the wrong body type for thes... 1 0 0 2 2 2 1058 815 39 Ig brought me here Love the way this top layers under my jackets ... 2 1 0 1 1 1 12132 984 47 Runs small especially the arms I love this jacket. it's the prettiest and mos... 3 1 0 1 3 3 20756 1051 42 True red, true beauty. These pants are gorgeous--the fabric has a sat... 3 1 0 2 2 4 <p>let's for example have a look to the encodings for the categorical feature <code>class_name</code></p> In\u00a0[21]: Copied! <pre>le.encoding_dict[\"class_name\"]\n</pre> le.encoding_dict[\"class_name\"] Out[21]: <pre>{'blouses': 1,\n 'jeans': 2,\n 'jackets': 3,\n 'pants': 4,\n 'knits': 5,\n 'dresses': 6,\n 'skirts': 7,\n 'sweaters': 8,\n 'fine gauge': 9,\n 'legwear': 10,\n 'lounge': 11,\n 'shorts': 12,\n 'outerwear': 13,\n 'intimates': 14,\n 'swim': 15,\n 'trend': 16,\n 'sleep': 17,\n 'layering': 18,\n 'missing': 19,\n 'casual bottoms': 20,\n 'chemises': 21}</pre> In\u00a0[22]: Copied! <pre># tabular training and test sets\nX_tab_tr = csr_matrix(train_tab_le[tab_cols].values)\nX_tab_te = csr_matrix(test_tab_le[tab_cols].values)\n\n# text + tabular training and test sets\nX_tab_text_tr = hstack((X_tab_tr, X_text_tr))\nX_tab_text_te = hstack((X_tab_te, X_text_te))\n</pre> # tabular training and test sets X_tab_tr = csr_matrix(train_tab_le[tab_cols].values) X_tab_te = csr_matrix(test_tab_le[tab_cols].values)  # text + tabular training and test sets X_tab_text_tr = hstack((X_tab_tr, X_text_tr)) X_tab_text_te = hstack((X_tab_te, X_text_te)) In\u00a0[23]: Copied! <pre>X_tab_tr\n</pre> X_tab_tr Out[23]: <pre>&lt;18086x4 sparse matrix of type '&lt;class 'numpy.int64'&gt;'\n\twith 72344 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[24]: Copied! <pre>X_tab_text_tr\n</pre> X_tab_text_tr Out[24]: <pre>&lt;18086x4570 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 956418 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[25]: Copied! <pre>lgbtrain_tab_text = lgbDataset(\n    X_tab_text_tr,\n    train.rating.values,\n    categorical_feature=[0, 1, 2, 3],\n    free_raw_data=False,\n)\n\nlgbtest_tab_text = lgbDataset(\n    X_tab_text_te,\n    test.rating.values,\n    reference=lgbtrain_tab_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_tab_text = lgbDataset(     X_tab_text_tr,     train.rating.values,     categorical_feature=[0, 1, 2, 3],     free_raw_data=False, )  lgbtest_tab_text = lgbDataset(     X_tab_text_te,     test.rating.values,     reference=lgbtrain_tab_text,     free_raw_data=False, ) In\u00a0[26]: Copied! <pre>lgb_tab_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_tab_text,\n    valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],\n    valid_names=[\"test\", \"train\"],\n    verbose_eval=False,\n)\n</pre> lgb_tab_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_tab_text,     valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],     valid_names=[\"test\", \"train\"],     verbose_eval=False, ) <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2065: UserWarning: Using categorical_feature in Dataset.\n  _log_warning('Using categorical_feature in Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2068: UserWarning: categorical_feature in Dataset is overridden.\nNew categorical_feature is [0, 1, 2, 3]\n  _log_warning('categorical_feature in Dataset is overridden.\\n'\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/engine.py:239: UserWarning: 'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n  _log_warning(\"'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. \"\n</pre> <pre>[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.138280 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 143432\n[LightGBM] [Info] Number of data points in the train set: 18086, number of used features: 2289\n[LightGBM] [Info] Start training from score -2.255919\n[LightGBM] [Info] Start training from score -2.081545\n[LightGBM] [Info] Start training from score -1.528281\n[LightGBM] [Info] Start training from score -0.591354\n</pre> <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1780: UserWarning: Overriding the parameters from Reference Dataset.\n  _log_warning('Overriding the parameters from Reference Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1513: UserWarning: categorical_column in param dict is overridden.\n  _log_warning(f'{cat_alias} in param dict is overridden.')\n</pre> In\u00a0[27]: Copied! <pre>preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te)\npreds_tab_text_class = np.argmax(preds_tab_text, 1)\n\nacc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class)\nf1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\")\ncm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class)\n</pre> preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te) preds_tab_text_class = np.argmax(preds_tab_text, 1)  acc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class) f1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\") cm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class) In\u00a0[28]: Copied! <pre>print(\n    f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\"\n)\n</pre> print(     f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\" ) <pre>LightGBM text + tabular Accuracy: 0.6382131800088456. LightGBM text + tabular F1 Score: 0.6080251307242649\n</pre> In\u00a0[29]: Copied! <pre>print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\")\n</pre> print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\") <pre>LightGBM text + tabular Confusion Matrix:\n [[ 193  123   68   90]\n [ 123  146  157  138]\n [  37   90  272  582]\n [  16   37  175 2275]]\n</pre> <p>So, in this set up, the addition tabular columns do not help performance.</p> In\u00a0[30]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1\n)\n\nwd_X_text_tr = text_preprocessor.fit_transform(train)\nwd_X_text_te = text_preprocessor.transform(test)\n</pre> text_preprocessor = TextPreprocessor(     text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1 )  wd_X_text_tr = text_preprocessor.fit_transform(train) wd_X_text_te = text_preprocessor.transform(test) <pre>The vocabulary contains 4328 tokens\n</pre> In\u00a0[31]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n\n\nwd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4)\n</pre> basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], )   wd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4) In\u00a0[32]: Copied! <pre>wd_text_model\n</pre> wd_text_model Out[32]: <pre>WideDeep(\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[33]: Copied! <pre>text_trainer = Trainer(\n    wd_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> text_trainer = Trainer(     wd_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[34]: Copied! <pre>text_trainer.fit(\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> text_trainer.fit(     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.39it/s, loss=1.16, metrics={'acc': 0.5349, 'f1': 0.2011}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.35it/s, loss=0.964, metrics={'acc': 0.5827, 'f1': 0.3005}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.33it/s, loss=0.845, metrics={'acc': 0.6252, 'f1': 0.4133}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.99it/s, loss=0.765, metrics={'acc': 0.6575, 'f1': 0.4875}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.55it/s, loss=0.709, metrics={'acc': 0.6879, 'f1': 0.5423}]\n</pre> In\u00a0[35]: Copied! <pre>wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te)\nwd_pred_text_class = np.argmax(wd_pred_text, 1)\n</pre> wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te) wd_pred_text_class = np.argmax(wd_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 211.51it/s]\n</pre> In\u00a0[36]: Copied! <pre>wd_acc_text = accuracy_score(test.rating, wd_pred_text_class)\nwd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\")\nwd_cm_text = confusion_matrix(test.rating, wd_pred_text_class)\n</pre> wd_acc_text = accuracy_score(test.rating, wd_pred_text_class) wd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\") wd_cm_text = confusion_matrix(test.rating, wd_pred_text_class) In\u00a0[37]: Copied! <pre>print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\")\n</pre> print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\") <pre>Basic RNN Accuracy: 0.6076957098628926. Basic RNN F1 Score: 0.6017335854471788\n</pre> In\u00a0[38]: Copied! <pre>print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\")\n</pre> print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\") <pre>Basic RNN Confusion Matrix:\n [[ 327   76   62    9]\n [ 285  115  117   47]\n [ 131  122  315  413]\n [  42   69  401 1991]]\n</pre> <p>The performance is very similar to that of using simply tf-idf and lightgbm. Let see if adding tabular features helps when using <code>pytorch-widedeep</code></p> In\u00a0[39]: Copied! <pre># ?TabPreprocessor\n</pre> # ?TabPreprocessor In\u00a0[40]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)\n\nwd_X_tab_tr = tab_preprocessor.fit_transform(train)\nwd_X_tab_te = tab_preprocessor.transform(test)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)  wd_X_tab_tr = tab_preprocessor.fit_transform(train) wd_X_tab_te = tab_preprocessor.transform(test) In\u00a0[41]: Copied! <pre># ?TabMlp\n</pre> # ?TabMlp In\u00a0[42]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[43]: Copied! <pre>tab_model\n</pre> tab_model Out[43]: <pre>TabMlp(\n  (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_age): Embedding(78, 18, padding_idx=0)\n        (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n        (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n        (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.1, inplace=False)\n    )\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=35, out_features=100, bias=True)\n        (2): ReLU(inplace=True)\n      )\n      (dense_layer_1): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=100, out_features=50, bias=True)\n        (2): ReLU(inplace=True)\n      )\n    )\n  )\n)</pre> In\u00a0[44]: Copied! <pre>text_model = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n</pre> text_model = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], ) In\u00a0[45]: Copied! <pre>wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4)\n</pre> wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4) In\u00a0[46]: Copied! <pre>wd_tab_and_text_model\n</pre> wd_tab_and_text_model Out[46]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n        (cat_embed): DiffSizeCatEmbeddings(\n          (embed_layers): ModuleDict(\n            (emb_layer_age): Embedding(78, 18, padding_idx=0)\n            (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n            (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n            (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n          )\n          (embedding_dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=35, out_features=100, bias=True)\n            (2): ReLU(inplace=True)\n          )\n          (dense_layer_1): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=100, out_features=50, bias=True)\n            (2): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=50, out_features=4, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[47]: Copied! <pre>tab_and_text_trainer = Trainer(\n    wd_tab_and_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> tab_and_text_trainer = Trainer(     wd_tab_and_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[48]: Copied! <pre>tab_and_text_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> tab_and_text_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.04it/s, loss=1.13, metrics={'acc': 0.538, 'f1': 0.1911}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.28it/s, loss=0.936, metrics={'acc': 0.5887, 'f1': 0.3507}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.26it/s, loss=0.825, metrics={'acc': 0.6394, 'f1': 0.4545}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 51.33it/s, loss=0.757, metrics={'acc': 0.6696, 'f1': 0.5214}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 50.39it/s, loss=0.702, metrics={'acc': 0.6963, 'f1': 0.5654}]\n</pre> In\u00a0[49]: Copied! <pre>wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=wd_X_text_te\n)\nwd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1)\n</pre> wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=wd_X_text_te ) wd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 136.94it/s]\n</pre> In\u00a0[50]: Copied! <pre>wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class)\nwd_f1_tab_and_text = f1_score(\n    test.rating, wd_pred_tab_and_text_class, average=\"weighted\"\n)\nwd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class)\n</pre> wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class) wd_f1_tab_and_text = f1_score(     test.rating, wd_pred_tab_and_text_class, average=\"weighted\" ) wd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class) In\u00a0[51]: Copied! <pre>print(\n    f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\"\n)\nprint(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\")\n</pre> print(     f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\" ) print(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\") <pre>Basic RNN + Tabular  Accuracy: 0.6333480760725343. Basic RNN + TabularF1 Score: 0.6332310089593208\nBasic RNN + Tabular  Confusion Matrix:\n [[ 267  132   65   10]\n [ 198  168  159   39]\n [  57  113  410  401]\n [  12   58  414 2019]]\n</pre> <p>We are going to \"manually\" code the Tokenizer and the model and see how they can be used as part of the process along with the <code>pytorch-widedeep</code> library.</p> <p>Tokenizer:</p> In\u00a0[52]: Copied! <pre>class BertTokenizer(object):\n    def __init__(\n        self,\n        pretrained_tokenizer=\"distilbert-base-uncased\",\n        do_lower_case=True,\n        max_length=90,\n    ):\n        super(BertTokenizer, self).__init__()\n        self.pretrained_tokenizer = pretrained_tokenizer\n        self.do_lower_case = do_lower_case\n        self.max_length = max_length\n\n    def fit(self, texts):\n        self.tokenizer = DistilBertTokenizer.from_pretrained(\n            self.pretrained_tokenizer, do_lower_case=self.do_lower_case\n        )\n\n        return self\n\n    def transform(self, texts):\n        input_ids = []\n        for text in texts:\n            encoded_sent = self.tokenizer.encode_plus(\n                text=self._pre_rules(text),\n                add_special_tokens=True,\n                max_length=self.max_length,\n                padding=\"max_length\",\n                truncation=True,\n            )\n\n            input_ids.append(encoded_sent.get(\"input_ids\"))\n        return np.stack(input_ids)\n\n    def fit_transform(self, texts):\n        return self.fit(texts).transform(texts)\n\n    @staticmethod\n    def _pre_rules(text):\n        return fix_html(rm_useless_spaces(spec_add_spaces(text)))\n</pre> class BertTokenizer(object):     def __init__(         self,         pretrained_tokenizer=\"distilbert-base-uncased\",         do_lower_case=True,         max_length=90,     ):         super(BertTokenizer, self).__init__()         self.pretrained_tokenizer = pretrained_tokenizer         self.do_lower_case = do_lower_case         self.max_length = max_length      def fit(self, texts):         self.tokenizer = DistilBertTokenizer.from_pretrained(             self.pretrained_tokenizer, do_lower_case=self.do_lower_case         )          return self      def transform(self, texts):         input_ids = []         for text in texts:             encoded_sent = self.tokenizer.encode_plus(                 text=self._pre_rules(text),                 add_special_tokens=True,                 max_length=self.max_length,                 padding=\"max_length\",                 truncation=True,             )              input_ids.append(encoded_sent.get(\"input_ids\"))         return np.stack(input_ids)      def fit_transform(self, texts):         return self.fit(texts).transform(texts)      @staticmethod     def _pre_rules(text):         return fix_html(rm_useless_spaces(spec_add_spaces(text))) <p>Model:</p> In\u00a0[53]: Copied! <pre>class BertModel(nn.Module):\n    def __init__(\n        self,\n        model_name: str = \"distilbert-base-uncased\",\n        freeze_bert: bool = False,\n    ):\n        super(BertModel, self).__init__()\n\n        self.bert = DistilBertModel.from_pretrained(\n            model_name,\n        )\n\n        if freeze_bert:\n            for param in self.bert.parameters():\n                param.requires_grad = False\n\n    def forward(self, X_inp: Tensor) -&gt; Tensor:\n        attn_mask = (X_inp != 0).type(torch.int8)\n        outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)\n        return outputs[0][:, 0, :]\n\n    @property\n    def output_dim(self) -&gt; int:\n        # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must\n        # have a 'output_dim' property so the WideDeep class knows the incoming dims\n        # from the custom model. in this case, I hardcoded it\n        return 768\n</pre> class BertModel(nn.Module):     def __init__(         self,         model_name: str = \"distilbert-base-uncased\",         freeze_bert: bool = False,     ):         super(BertModel, self).__init__()          self.bert = DistilBertModel.from_pretrained(             model_name,         )          if freeze_bert:             for param in self.bert.parameters():                 param.requires_grad = False      def forward(self, X_inp: Tensor) -&gt; Tensor:         attn_mask = (X_inp != 0).type(torch.int8)         outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)         return outputs[0][:, 0, :]      @property     def output_dim(self) -&gt; int:         # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must         # have a 'output_dim' property so the WideDeep class knows the incoming dims         # from the custom model. in this case, I hardcoded it         return 768 In\u00a0[54]: Copied! <pre>bert_tokenizer = BertTokenizer()\nX_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist())\nX_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist())\n</pre> bert_tokenizer = BertTokenizer() X_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist()) X_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist()) <p>As I mentioned a number of times in the documentation and examples, <code>pytorch-widedeep</code> is designed for flexibility. For any of the data modes (tabular, text and images) there are available components/models in the library. However, the user can choose to use any model they want with the only requirement that such model must have a <code>output_dim</code> property.</p> <p>With that in mind, the <code>BertModel</code> class defined above can be used by <code>pytorch-widedeep</code> as any other of the internal components. In other words, simply...pass it to the <code>WideDeep</code> class. In this case we are going to add a FC-head as part of the classifier.</p> In\u00a0[55]: Copied! <pre>bert_model = BertModel(freeze_bert=True)\nwd_bert_model = WideDeep(\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> bert_model = BertModel(freeze_bert=True) wd_bert_model = WideDeep(     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) <pre>Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n</pre> In\u00a0[56]: Copied! <pre>wd_bert_model\n</pre> wd_bert_model Out[56]: <pre>WideDeep(\n  (deeptext): BertModel(\n    (bert): DistilBertModel(\n      (embeddings): Embeddings(\n        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n        (position_embeddings): Embedding(512, 768)\n        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (transformer): Transformer(\n        (layer): ModuleList(\n          (0): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (1): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (2): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (3): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (4): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (5): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n        )\n      )\n    )\n  )\n  (deephead): Sequential(\n    (0): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=768, out_features=256, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=256, out_features=128, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_2): Sequential(\n          (0): Linear(in_features=128, out_features=64, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n    (1): Linear(in_features=64, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[57]: Copied! <pre>wd_bert_trainer = Trainer(\n    wd_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n\nwd_bert_trainer.fit(\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_bert_trainer = Trainer(     wd_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work )  wd_bert_trainer.fit(     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.68it/s, loss=0.968, metrics={'acc': 0.5879, 'f1': 0.3591}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.63it/s, loss=0.884, metrics={'acc': 0.6178, 'f1': 0.4399}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.55it/s, loss=0.87, metrics={'acc': 0.6234, 'f1': 0.4527}]\n</pre> In\u00a0[58]: Copied! <pre>wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te)\nwd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1)\n</pre> wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te) wd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.97it/s]\n</pre> In\u00a0[59]: Copied! <pre>wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class)\nwd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\")\nwd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class)\n</pre> wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class) wd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\") wd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class) In\u00a0[60]: Copied! <pre>print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\")\nprint(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\")\n</pre> print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\") print(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\") <pre>Distilbert Accuracy: 0.6326846528084918. Distilbert F1 Score: 0.5796652991272998\nDistilbert Confusion Matrix:\n [[ 287   75   22   90]\n [ 197  136   62  169]\n [  68  119  123  671]\n [  40   64   84 2315]]\n</pre> <p>Now, adding a tabular model follows the exact same process as the one described in section 2.</p> In\u00a0[61]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[62]: Copied! <pre>wd_tab_bert_model = WideDeep(\n    deeptabular=tab_model,\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> wd_tab_bert_model = WideDeep(     deeptabular=tab_model,     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) In\u00a0[63]: Copied! <pre>wd_tab_bert_trainer = Trainer(\n    wd_tab_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> wd_tab_bert_trainer = Trainer(     wd_tab_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[64]: Copied! <pre>wd_tab_bert_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_tab_bert_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.15it/s, loss=0.974, metrics={'acc': 0.5838, 'f1': 0.3404}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.38it/s, loss=0.885, metrics={'acc': 0.618, 'f1': 0.4378}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.40it/s, loss=0.868, metrics={'acc': 0.6252, 'f1': 0.4575}]\n</pre> In\u00a0[65]: Copied! <pre>wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=X_bert_te\n)\nwd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1)\n</pre> wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=X_bert_te ) wd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.32it/s]\n</pre> In\u00a0[66]: Copied! <pre>wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class)\nwd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\")\nwd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class)\n</pre> wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class) wd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\") wd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class) In\u00a0[67]: Copied! <pre>print(\n    f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\"\n)\nprint(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\")\n</pre> print(     f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\" ) print(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\") <pre>Distilbert + Tabular Accuracy: 0.6242812914639541. Distilbert+ Tabular F1 Score: 0.5508351761564895\nDistilbert + Tabular Confusion Matrix:\n [[ 297   56   11  110]\n [ 229   91   38  206]\n [  86   90   71  734]\n [  49   48   42 2364]]\n</pre>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#using-a-hugginface-model","title":"Using a Hugginface model\u00b6","text":"<p>In this notebook we will show how to use an \"external\" Hugginface model along with any other model in the libray. In particular we will show how to combine it with a tabular DL model.</p> <p>Since we are here, we will also compare the performance of a few models on a text classification problem.</p> <p>The notebook will go as follows:</p> <ol> <li>Text classification using tf-idf + LightGBM</li> <li>Text classification using a basic RNN</li> <li>Text classification using Distilbert</li> </ol> <p>In all 3 cases we will add some tabular features to see if these help.</p> <p>In general, I would not pay much attention to the results since I have placed no effort in getting the best possible results (i.e. no hyperparameter optimization or trying different architectures, for example).</p> <p>Let's go</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#1-text-classification-using-tf-idf-lightgbm","title":"1. Text classification using tf-idf + LightGBM\u00b6","text":""},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#2-text-classification-using-pytorch-widedeeps-built-in-models-a-basic-rnn","title":"2. Text classification using pytorch-widedeep's built-in models (a basic RNN)\u00b6","text":"<p>Moving on now to fully using <code>pytorch-widedeep</code> in this dataset, let's have a look on how one could use a simple RNN to predict the ratings with the library.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#3-text-classification-using-a-hugginface-model-as-a-custom-model-in-pytorch-widedeeps","title":"3. Text classification using a Hugginface model as a custom model in pytorch-widedeep's\u00b6","text":""},{"location":"examples/17_feature_importance_via_attention_weights.html","title":"17 feature importance via attention weights","text":"In\u00a0[1]: Copied! <pre>import torch\n\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import torch  import numpy as np from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score   from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep from pytorch_widedeep.callbacks import EarlyStopping from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre># use_cuda = torch.cuda.is_available()\ndf = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\ntarget_colname = \"income_label\"\n</pre> # use_cuda = torch.cuda.is_available() df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True) target_colname = \"income_label\" In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_embed_cols = []\nfor col in df.columns:\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:\n        cat_embed_cols.append(col)\n</pre> cat_embed_cols = [] for col in df.columns:     if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:         cat_embed_cols.append(col) In\u00a0[5]: Copied! <pre># all cols will be categorical\nassert len(cat_embed_cols) == df.shape[1] - 1\n</pre> # all cols will be categorical assert len(cat_embed_cols) == df.shape[1] - 1 In\u00a0[6]: Copied! <pre>train, test = train_test_split(\n    df, test_size=0.1, random_state=1, stratify=df[[target_colname]]\n)\n</pre> train, test = train_test_split(     df, test_size=0.1, random_state=1, stratify=df[[target_colname]] ) In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True) In\u00a0[8]: Copied! <pre>X_tab_train = tab_preprocessor.fit_transform(train)\nX_tab_test = tab_preprocessor.transform(test)\ntarget = train[target_colname].values\n</pre> X_tab_train = tab_preprocessor.fit_transform(train) X_tab_test = tab_preprocessor.transform(test) target = train[target_colname].values In\u00a0[9]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    embed_continuous_method=\"standard\",\n    input_dim=8,\n    n_heads=2,\n    n_blocks=1,\n    attn_dropout=0.1,\n    transformer_activation=\"relu\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     embed_continuous_method=\"standard\",     input_dim=8,     n_heads=2,     n_blocks=1,     attn_dropout=0.1,     transformer_activation=\"relu\", ) In\u00a0[10]: Copied! <pre>model = WideDeep(deeptabular=tab_transformer)\n</pre> model = WideDeep(deeptabular=tab_transformer) In\u00a0[11]: Copied! <pre>optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[12]: Copied! <pre>lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[13]: Copied! <pre>early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[14]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=optimizer,\n    lr_schedulers=lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[early_stopping],\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=optimizer,     lr_schedulers=lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[early_stopping],     metrics=[Accuracy], ) <p>The feature importances will be computed after training, using a sample of the training dataset of size <code>feature_importance_sample_size</code></p> In\u00a0[15]: Copied! <pre>trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.80it/s, loss=0.334, metrics={'acc': 0.847}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.34it/s, loss=0.294, metrics={'acc': 0.8669}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.02it/s, loss=0.293, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.03it/s, loss=0.283, metrics={'acc': 0.8678}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 87.69it/s, loss=0.282, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.22it/s, loss=0.279, metrics={'acc': 0.8717}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.24it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.29it/s, loss=0.277, metrics={'acc': 0.8731}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.76it/s, loss=0.275, metrics={'acc': 0.8727}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.80it/s, loss=0.276, metrics={'acc': 0.8727}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.78it/s, loss=0.273, metrics={'acc': 0.873}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.43it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.271, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.14it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.29it/s, loss=0.271, metrics={'acc': 0.875}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.46it/s, loss=0.276, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.10it/s, loss=0.27, metrics={'acc': 0.8761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.49it/s, loss=0.275, metrics={'acc': 0.8728}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 70.40it/s, loss=0.269, metrics={'acc': 0.8747}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.47it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.83it/s, loss=0.268, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.03it/s, loss=0.274, metrics={'acc': 0.873}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.86it/s, loss=0.267, metrics={'acc': 0.8743}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.61it/s, loss=0.274, metrics={'acc': 0.8734}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.39it/s, loss=0.267, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.05it/s, loss=0.275, metrics={'acc': 0.8717}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.36it/s, loss=0.265, metrics={'acc': 0.8767}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 109.60it/s, loss=0.276, metrics={'acc': 0.8747}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.34it/s, loss=0.264, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.55it/s, loss=0.276, metrics={'acc': 0.8706}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.264, metrics={'acc': 0.8777}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.08it/s, loss=0.275, metrics={'acc': 0.8753}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.96it/s, loss=0.263, metrics={'acc': 0.877}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.83it/s, loss=0.277, metrics={'acc': 0.8739}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.38it/s, loss=0.263, metrics={'acc': 0.8779}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.18it/s, loss=0.278, metrics={'acc': 0.8714}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.61it/s, loss=0.261, metrics={'acc': 0.8784}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.19it/s, loss=0.278, metrics={'acc': 0.8712}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.43it/s, loss=0.261, metrics={'acc': 0.8791}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.87it/s, loss=0.28, metrics={'acc': 0.873}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.97it/s, loss=0.26, metrics={'acc': 0.8787}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 107.50it/s, loss=0.279, metrics={'acc': 0.8732}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.76it/s, loss=0.253, metrics={'acc': 0.8816}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.11it/s, loss=0.279, metrics={'acc': 0.8707}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.92it/s, loss=0.252, metrics={'acc': 0.8828}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.14it/s, loss=0.28, metrics={'acc': 0.8711}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.04it/s, loss=0.252, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 98.36it/s, loss=0.28, metrics={'acc': 0.8708}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.31it/s, loss=0.251, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.97it/s, loss=0.281, metrics={'acc': 0.8709}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.63it/s, loss=0.25, metrics={'acc': 0.8834}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.07it/s, loss=0.281, metrics={'acc': 0.8698}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.37it/s, loss=0.251, metrics={'acc': 0.884}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.75it/s, loss=0.281, metrics={'acc': 0.87}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.19it/s, loss=0.25, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.282, metrics={'acc': 0.8699}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.19it/s, loss=0.25, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.11it/s, loss=0.282, metrics={'acc': 0.8695}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.57it/s, loss=0.249, metrics={'acc': 0.8839}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.64it/s, loss=0.283, metrics={'acc': 0.8689}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.55it/s, loss=0.249, metrics={'acc': 0.8846}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.10it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.56it/s, loss=0.248, metrics={'acc': 0.8841}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.45it/s, loss=0.284, metrics={'acc': 0.8687}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.06it/s, loss=0.248, metrics={'acc': 0.8848}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 129.26it/s, loss=0.284, metrics={'acc': 0.8689}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.53it/s, loss=0.248, metrics={'acc': 0.8854}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.61it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.78it/s, loss=0.248, metrics={'acc': 0.8853}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.31it/s, loss=0.283, metrics={'acc': 0.8694}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.51it/s, loss=0.248, metrics={'acc': 0.8863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.94it/s, loss=0.283, metrics={'acc': 0.8693}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.35it/s, loss=0.247, metrics={'acc': 0.8844}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.77it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.62it/s, loss=0.248, metrics={'acc': 0.8837}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.62it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.82it/s, loss=0.248, metrics={'acc': 0.8842}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.64it/s, loss=0.283, metrics={'acc': 0.8695}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.86it/s, loss=0.247, metrics={'acc': 0.8855}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.34it/s, loss=0.283, metrics={'acc': 0.8692}]\n</pre> <pre>Best Epoch: 10. Best val_loss: 0.27451\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[16]: Copied! <pre>trainer.feature_importance\n</pre> trainer.feature_importance Out[16]: <pre>{'age': 0.09718182,\n 'workclass': 0.090637445,\n 'education': 0.08910798,\n 'marital_status': 0.08971319,\n 'occupation': 0.12546304,\n 'relationship': 0.086381145,\n 'race': 0.050686445,\n 'gender': 0.05116429,\n 'capital_gain': 0.08165918,\n 'capital_loss': 0.07702667,\n 'hours_per_week': 0.08205996,\n 'native_country': 0.07891885}</pre> In\u00a0[17]: Copied! <pre>preds = trainer.predict(X_tab=X_tab_test)\n</pre> preds = trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 199.63it/s]\n</pre> In\u00a0[18]: Copied! <pre>accuracy_score(preds, test.income_label)\n</pre> accuracy_score(preds, test.income_label) Out[18]: <pre>0.8685772773797339</pre> In\u00a0[19]: Copied! <pre>test.reset_index(drop=True, inplace=True)\n</pre> test.reset_index(drop=True, inplace=True) In\u00a0[20]: Copied! <pre>test[test.income_label == 0].head(1)\n</pre> test[test.income_label == 0].head(1) Out[20]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 26 Private Some-college Never-married Exec-managerial Not-in-family White Male 0 0 60 United-States 0 In\u00a0[21]: Copied! <pre>test[test.income_label == 1].head(1)\n</pre> test[test.income_label == 1].head(1) Out[21]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 3 36 Local-gov Doctorate Married-civ-spouse Prof-specialty Husband White Male 0 1887 50 United-States 1 <p>To get the feature importance of a test dataset, simply use the <code>explain</code> method</p> In\u00a0[22]: Copied! <pre>feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False)\n</pre> feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False) In\u00a0[23]: Copied! <pre>list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])])\n</pre> list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])]) Out[23]: <pre>['hours_per_week',\n 'education',\n 'relationship',\n 'occupation',\n 'workclass',\n 'capital_gain',\n 'native_country',\n 'marital_status',\n 'capital_loss',\n 'age',\n 'race',\n 'gender']</pre> In\u00a0[24]: Copied! <pre>list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])])\n</pre> list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])]) Out[24]: <pre>['age',\n 'capital_loss',\n 'hours_per_week',\n 'marital_status',\n 'native_country',\n 'relationship',\n 'race',\n 'education',\n 'occupation',\n 'capital_gain',\n 'gender',\n 'workclass']</pre> <p>We could do the same with the <code>ContextAttentionMLP</code></p> In\u00a0[25]: Copied! <pre>context_attn_mlp = ContextAttentionMLP(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.0,\n    input_dim=16,\n    attn_dropout=0.1,\n    attn_activation=\"relu\",\n)\n</pre> context_attn_mlp = ContextAttentionMLP(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.0,     input_dim=16,     attn_dropout=0.1,     attn_activation=\"relu\", ) In\u00a0[26]: Copied! <pre>mlp_model = WideDeep(deeptabular=context_attn_mlp)\n</pre> mlp_model = WideDeep(deeptabular=context_attn_mlp) In\u00a0[27]: Copied! <pre>mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[28]: Copied! <pre>mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    mlp_optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     mlp_optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[29]: Copied! <pre>mlp_early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> mlp_early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[30]: Copied! <pre>mlp_trainer = Trainer(\n    mlp_model,\n    objective=\"binary\",\n    optimizers=mlp_optimizer,\n    lr_schedulers=mlp_lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[mlp_early_stopping],\n    metrics=[Accuracy],\n)\n</pre> mlp_trainer = Trainer(     mlp_model,     objective=\"binary\",     optimizers=mlp_optimizer,     lr_schedulers=mlp_lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[mlp_early_stopping],     metrics=[Accuracy], ) In\u00a0[31]: Copied! <pre>mlp_trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> mlp_trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.405, metrics={'acc': 0.8094}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.26it/s, loss=0.309, metrics={'acc': 0.8583}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.70it/s, loss=0.332, metrics={'acc': 0.8447}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.36it/s, loss=0.293, metrics={'acc': 0.8646}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.42it/s, loss=0.319, metrics={'acc': 0.8505}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.05it/s, loss=0.293, metrics={'acc': 0.8654}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.00it/s, loss=0.312, metrics={'acc': 0.8554}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.49it/s, loss=0.291, metrics={'acc': 0.8661}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.50it/s, loss=0.308, metrics={'acc': 0.8583}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.48it/s, loss=0.287, metrics={'acc': 0.8669}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.84it/s, loss=0.303, metrics={'acc': 0.8605}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 128.70it/s, loss=0.288, metrics={'acc': 0.8673}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.93it/s, loss=0.301, metrics={'acc': 0.8597}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 121.47it/s, loss=0.298, metrics={'acc': 0.8628}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.56it/s, loss=0.3, metrics={'acc': 0.8592}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.84it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.17it/s, loss=0.298, metrics={'acc': 0.8619}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.32it/s, loss=0.28, metrics={'acc': 0.8716}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.13it/s, loss=0.297, metrics={'acc': 0.8615}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.50it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.54it/s, loss=0.293, metrics={'acc': 0.8641}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.57it/s, loss=0.284, metrics={'acc': 0.867}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.92it/s, loss=0.293, metrics={'acc': 0.863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.42it/s, loss=0.282, metrics={'acc': 0.8701}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.61it/s, loss=0.293, metrics={'acc': 0.8635}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.56it/s, loss=0.276, metrics={'acc': 0.8719}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.92it/s, loss=0.29, metrics={'acc': 0.8633}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.06it/s, loss=0.286, metrics={'acc': 0.8669}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.83it/s, loss=0.291, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.88it/s, loss=0.282, metrics={'acc': 0.8677}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.74it/s, loss=0.29, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.85it/s, loss=0.285, metrics={'acc': 0.8672}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.99it/s, loss=0.29, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.53it/s, loss=0.282, metrics={'acc': 0.8681}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.22it/s, loss=0.288, metrics={'acc': 0.8651}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.89it/s, loss=0.288, metrics={'acc': 0.8676}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.85it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.26it/s, loss=0.284, metrics={'acc': 0.8662}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.80it/s, loss=0.289, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.44it/s, loss=0.281, metrics={'acc': 0.8703}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.72it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.04it/s, loss=0.285, metrics={'acc': 0.8648}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.86it/s, loss=0.289, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.75it/s, loss=0.282, metrics={'acc': 0.8666}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.289, metrics={'acc': 0.8668}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.86it/s, loss=0.282, metrics={'acc': 0.8724}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.93it/s, loss=0.288, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.69it/s, loss=0.285, metrics={'acc': 0.8656}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.99it/s, loss=0.284, metrics={'acc': 0.8671}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.27it/s, loss=0.277, metrics={'acc': 0.8707}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.86it/s, loss=0.282, metrics={'acc': 0.8686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.94it/s, loss=0.276, metrics={'acc': 0.8712}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.86it/s, loss=0.283, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.11it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.43it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.64it/s, loss=0.277, metrics={'acc': 0.8712}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.39it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.83it/s, loss=0.277, metrics={'acc': 0.872}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.93it/s, loss=0.28, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.07it/s, loss=0.275, metrics={'acc': 0.8714}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.35it/s, loss=0.281, metrics={'acc': 0.8697}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.68it/s, loss=0.276, metrics={'acc': 0.872}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.72it/s, loss=0.28, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.50it/s, loss=0.276, metrics={'acc': 0.8709}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.28, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.07it/s, loss=0.277, metrics={'acc': 0.8709}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.58it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.07it/s, loss=0.274, metrics={'acc': 0.8719}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.03it/s, loss=0.28, metrics={'acc': 0.8687}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.15it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.279, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.10it/s, loss=0.278, metrics={'acc': 0.8705}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.30it/s, loss=0.279, metrics={'acc': 0.869}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.75it/s, loss=0.279, metrics={'acc': 0.8702}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.34it/s, loss=0.28, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.79it/s, loss=0.277, metrics={'acc': 0.8698}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.49it/s, loss=0.279, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.25it/s, loss=0.279, metrics={'acc': 0.87}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.51it/s, loss=0.28, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.90it/s, loss=0.277, metrics={'acc': 0.8694}]\nepoch 41: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.278, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.74it/s, loss=0.28, metrics={'acc': 0.8675}]\nepoch 42: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.279, metrics={'acc': 0.8695}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.76it/s, loss=0.277, metrics={'acc': 0.8699}]\nepoch 43: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:04&lt;00:00, 66.14it/s, loss=0.279, metrics={'acc': 0.8681}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.20it/s, loss=0.277, metrics={'acc': 0.8714}]\nepoch 44: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.37it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.18it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 45: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.23it/s, loss=0.278, metrics={'acc': 0.8702}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.83it/s, loss=0.278, metrics={'acc': 0.8707}]\nepoch 46: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.12it/s, loss=0.278, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.62it/s, loss=0.279, metrics={'acc': 0.8693}]\nepoch 47: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.276, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 48: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.25it/s, loss=0.278, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.52it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 49: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.35it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.82it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 50: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.15it/s, loss=0.277, metrics={'acc': 0.8717}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.62it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 51: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.277, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.63it/s, loss=0.278, metrics={'acc': 0.87}]\nepoch 52: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.04it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.39it/s, loss=0.278, metrics={'acc': 0.8697}]\nepoch 53: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.15it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.56it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 54: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.41it/s, loss=0.277, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.95it/s, loss=0.278, metrics={'acc': 0.8698}]\nepoch 55: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.90it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 56: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.83it/s, loss=0.277, metrics={'acc': 0.8707}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.13it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 57: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.20it/s, loss=0.277, metrics={'acc': 0.8722}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.16it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 58: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.33it/s, loss=0.276, metrics={'acc': 0.871}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.37it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 59: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.41it/s, loss=0.277, metrics={'acc': 0.8714}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.17it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 60: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.26it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.60it/s, loss=0.278, metrics={'acc': 0.869}]\nepoch 61: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.88it/s, loss=0.278, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.47it/s, loss=0.278, metrics={'acc': 0.8692}]\nepoch 62: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.13it/s, loss=0.276, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.80it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 63: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.20it/s, loss=0.277, metrics={'acc': 0.8715}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.50it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 64: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.11it/s, loss=0.276, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.52it/s, loss=0.278, metrics={'acc': 0.869}]\n</pre> <pre>Best Epoch: 34. Best val_loss: 0.27449\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[32]: Copied! <pre>mlp_trainer.feature_importance\n</pre> mlp_trainer.feature_importance Out[32]: <pre>{'age': 0.116632804,\n 'workclass': 0.050255153,\n 'education': 0.094621316,\n 'marital_status': 0.12328919,\n 'occupation': 0.107893184,\n 'relationship': 0.11747801,\n 'race': 0.054717205,\n 'gender': 0.07514235,\n 'capital_gain': 0.059732802,\n 'capital_loss': 0.06738944,\n 'hours_per_week': 0.0610674,\n 'native_country': 0.07178114}</pre> In\u00a0[33]: Copied! <pre>mlp_preds = mlp_trainer.predict(X_tab=X_tab_test)\n</pre> mlp_preds = mlp_trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 212.38it/s]\n</pre> In\u00a0[34]: Copied! <pre>accuracy_score(mlp_preds, test.income_label)\n</pre> accuracy_score(mlp_preds, test.income_label) Out[34]: <pre>0.8726714431934494</pre>"},{"location":"examples/17_feature_importance_via_attention_weights.html#feature-importance-via-the-attention-weights","title":"Feature Importance via the attention weights\u00b6","text":"<p>I will start by saying that I consider this feature of the library purely experimental. First of all I think there are multiple ways one could address finding the features importances for these models. However, and more importantly, one has to bear in mind that even tree-based algorithms on the same dataset produce different feature importances. This is more \"dramatic\" if one uses different techniques, such as shap or feature permutation (see for example this and references therein). All this to say that, sometimes, feature importance is just a measure contained within the experiment run, and for the model used.</p> <p>With that in mind, each instantiation of a deep tabular model, that has millions of trainable parameters, will potentially produce a different set of feature importances, even if the model has the same architecture. Moreover, this effect will become more apparent if the dataset is relatively easy and there are dependent/related columns so that one could get to the same success metric with different parameters.</p> <p>In summary, feature importances are implemented in this librray for all attention-based models for tabular data, with the exception of the <code>TabPerceiver</code>. However this functionality has to be used and interpreted with care and consider of value within the 'universe' (or context) of the model with which these features were produced.</p> <p>Nonetheless, let's have a look to how one would access to the feature importances when using this library.</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html","title":"18 wide and deep for recsys pt1","text":"<p>The goal of this, and the companion (part 2) notebooks is to illustrate how one could use this library in the context of recommendation systems. In particular, this notebook and the scripts at the <code>wide_deep_for_recsys</code> dir are a response to this issue. Therefore, we will use the Kaggle notebook referred in that issue here.</p> <p>In order to keep the length of the notebook tractable, we will split this exercise in 2. In this first notebook we will prepare the data in almost the exact same way as it is done in the Kaggle notebook and also show how one could use <code>pytorch-widedeep</code> to build a model almost identical to the one in that notebook.</p> <p>In a second notebook, we will show how one could use this library to implement other models, still following the same problem formulation.</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\nimport warnings\n\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep.datasets import load_movielens100k\n</pre> from pathlib import Path import warnings  import pandas as pd from sklearn.model_selection import train_test_split  from pytorch_widedeep.datasets import load_movielens100k In\u00a0[2]: Copied! <pre>warnings.filterwarnings(\"ignore\")\n</pre> warnings.filterwarnings(\"ignore\") In\u00a0[3]: Copied! <pre>save_path = Path(\"prepared_data\")\nif not save_path.exists():\n    save_path.mkdir(parents=True, exist_ok=True)\n</pre> save_path = Path(\"prepared_data\") if not save_path.exists():     save_path.mkdir(parents=True, exist_ok=True) In\u00a0[4]: Copied! <pre>data, users, items = load_movielens100k(as_frame=True)\n</pre> data, users, items = load_movielens100k(as_frame=True) In\u00a0[5]: Copied! <pre># Alternatively, as specified in the docs: 'The last 19 fields are the genres' so:\n# list_of_genres = items.columns.tolist()[-19:]\nlist_of_genres = [\n    \"unknown\",\n    \"Action\",\n    \"Adventure\",\n    \"Animation\",\n    \"Children's\",\n    \"Comedy\",\n    \"Crime\",\n    \"Documentary\",\n    \"Drama\",\n    \"Fantasy\",\n    \"Film-Noir\",\n    \"Horror\",\n    \"Musical\",\n    \"Mystery\",\n    \"Romance\",\n    \"Sci-Fi\",\n    \"Thriller\",\n    \"War\",\n    \"Western\",\n]\n</pre> # Alternatively, as specified in the docs: 'The last 19 fields are the genres' so: # list_of_genres = items.columns.tolist()[-19:] list_of_genres = [     \"unknown\",     \"Action\",     \"Adventure\",     \"Animation\",     \"Children's\",     \"Comedy\",     \"Crime\",     \"Documentary\",     \"Drama\",     \"Fantasy\",     \"Film-Noir\",     \"Horror\",     \"Musical\",     \"Mystery\",     \"Romance\",     \"Sci-Fi\",     \"Thriller\",     \"War\",     \"Western\", ] <p>Let's first start by loading the interactions, user and item data</p> In\u00a0[6]: Copied! <pre>data.head()\n</pre> data.head() Out[6]: user_id movie_id rating timestamp 0 196 242 3 881250949 1 186 302 3 891717742 2 22 377 1 878887116 3 244 51 2 880606923 4 166 346 1 886397596 In\u00a0[7]: Copied! <pre>users.head()\n</pre> users.head() Out[7]: user_id age gender occupation zip_code 0 1 24 M technician 85711 1 2 53 F other 94043 2 3 23 M writer 32067 3 4 24 M technician 43537 4 5 33 F other 15213 In\u00a0[8]: Copied! <pre>items.head()\n</pre> items.head() Out[8]: movie_id movie_title release_date video_release_date IMDb_URL unknown Action Adventure Animation Children's ... Fantasy Film-Noir Horror Musical Mystery Romance Sci-Fi Thriller War Western 0 1 Toy Story (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Toy%20Story%2... 0 0 0 1 1 ... 0 0 0 0 0 0 0 0 0 0 1 2 GoldenEye (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?GoldenEye%20(... 0 1 1 0 0 ... 0 0 0 0 0 0 0 1 0 0 2 3 Four Rooms (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Four%20Rooms%... 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 3 4 Get Shorty (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Get%20Shorty%... 0 1 0 0 0 ... 0 0 0 0 0 0 0 0 0 0 4 5 Copycat (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Copycat%20(1995) 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 <p>5 rows \u00d7 24 columns</p> In\u00a0[9]: Copied! <pre># adding a column with the number of movies watched per user\ndataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True)\ndataset[\"one\"] = 1\ndataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum()\ndataset.drop(\"one\", axis=1, inplace=True)\ndataset.head()\n</pre> # adding a column with the number of movies watched per user dataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True) dataset[\"one\"] = 1 dataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum() dataset.drop(\"one\", axis=1, inplace=True) dataset.head() Out[9]: user_id movie_id rating timestamp num_watched 0 1 168 5 874965478 1 1 1 172 5 874965478 2 2 1 165 5 874965518 3 3 1 156 4 874965556 4 4 1 196 5 874965677 5 In\u00a0[10]: Copied! <pre># adding a column with the mean rating at a point in time per user\ndataset[\"mean_rate\"] = (\n    dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> # adding a column with the mean rating at a point in time per user dataset[\"mean_rate\"] = (     dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"] ) dataset.head() Out[10]: user_id movie_id rating timestamp num_watched mean_rate 0 1 168 5 874965478 1 5.00 1 1 172 5 874965478 2 5.00 2 1 165 5 874965518 3 5.00 3 1 156 4 874965556 4 4.75 4 1 196 5 874965677 5 4.80 In\u00a0[11]: Copied! <pre>dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1)\n</pre> dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1) <p>Following the same processing used by the author in the before-mentioned Kaggle notebook, we build sequences of previous movies watched</p> In\u00a0[12]: Copied! <pre># Here the author builds the sequences\ndataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x))\ndataset[\"prev_movies\"] = (\n    dataset.groupby(\"user_id\")[\"prev_movies\"]\n    .apply(lambda x: (x + \" \").cumsum().str.strip())\n    .reset_index(drop=True)\n)\ndataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split())\ndataset.head()\n</pre> # Here the author builds the sequences dataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x)) dataset[\"prev_movies\"] = (     dataset.groupby(\"user_id\")[\"prev_movies\"]     .apply(lambda x: (x + \" \").cumsum().str.strip())     .reset_index(drop=True) ) dataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split()) dataset.head() Out[12]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies 0 1 168 5 874965478 1 5.00 172.0 [168] 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] <p>And now we add a <code>genre_rate</code> as the mean of all movies rated for a given genre per user</p> In\u00a0[13]: Copied! <pre>dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\")\nfor genre in list_of_genres:\n    dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]\n    dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()\n    dataset[f\"{genre}_rate\"] = (\n        dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]\n    )\ndataset[list_of_genres] = dataset[list_of_genres].apply(\n    lambda x: x / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\") for genre in list_of_genres:     dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]     dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()     dataset[f\"{genre}_rate\"] = (         dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]     ) dataset[list_of_genres] = dataset[list_of_genres].apply(     lambda x: x / dataset[\"num_watched\"] ) dataset.head() Out[13]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Fantasy_rate Film-Noir_rate Horror_rate Musical_rate Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN <p>5 rows \u00d7 46 columns</p> <p>Adding user features</p> In\u00a0[14]: Copied! <pre>dataset = dataset.merge(users, on=\"user_id\", how=\"left\")\ndataset.head()\n</pre> dataset = dataset.merge(users, on=\"user_id\", how=\"left\") dataset.head() Out[14]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN 24 M technician 85711 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 <p>5 rows \u00d7 50 columns</p> <p>Again, we use the same settings as those in the Kaggle notebook, but <code>COLD_START_TRESH</code> is pretty aggressive</p> In\u00a0[15]: Copied! <pre>COLD_START_TRESH = 5\n\nfiltred_data = dataset[\n    (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna())\n].sort_values(\"timestamp\")\ntrain_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False)\nvalid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False)\n</pre> COLD_START_TRESH = 5  filtred_data = dataset[     (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna()) ].sort_values(\"timestamp\") train_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False) valid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False) In\u00a0[16]: Copied! <pre>cols_to_drop = [\n    # \"rating\",\n    \"timestamp\",\n    \"num_watched\",\n]\n\ndf_train = train_data.drop(cols_to_drop, axis=1)\ndf_valid = valid_data.drop(cols_to_drop, axis=1)\ndf_test = test_data.drop(cols_to_drop, axis=1)\n\ndf_train.to_pickle(save_path / \"df_train.pkl\")\ndf_valid.to_pickle(save_path / \"df_valid.pkl\")\ndf_test.to_pickle(save_path / \"df_test.pkl\")\n</pre> cols_to_drop = [     # \"rating\",     \"timestamp\",     \"num_watched\", ]  df_train = train_data.drop(cols_to_drop, axis=1) df_valid = valid_data.drop(cols_to_drop, axis=1) df_test = test_data.drop(cols_to_drop, axis=1)  df_train.to_pickle(save_path / \"df_train.pkl\") df_valid.to_pickle(save_path / \"df_valid.pkl\") df_test.to_pickle(save_path / \"df_test.pkl\") <p>Let's now build a model that is nearly identical to the one use in the Kaggle notebook</p> In\u00a0[17]: Copied! <pre>import numpy as np\nimport torch\nfrom torch import nn\nfrom scipy.sparse import coo_matrix\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import numpy as np import torch from torch import nn from scipy.sparse import coo_matrix  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[18]: Copied! <pre>device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nsave_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n</pre> device = \"cuda\" if torch.cuda.is_available() else \"cpu\"  save_path = Path(\"prepared_data\")  PAD_IDX = 0 <p>Let's use some of the functions the author of the kaggle's notebook uses to prepare the data</p> In\u00a0[19]: Copied! <pre>def get_coo_indexes(lil):\n    rows = []\n    cols = []\n    for i, el in enumerate(lil):\n        if type(el) != list:\n            el = [el]\n        for j in el:\n            rows.append(i)\n            cols.append(j)\n    return rows, cols\n\n\ndef get_sparse_features(series, shape):\n    coo_indexes = get_coo_indexes(series.tolist())\n    sparse_df = coo_matrix(\n        (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape\n    )\n    return sparse_df\n\n\ndef sparse_to_idx(data, pad_idx=-1):\n    indexes = data.nonzero()\n    indexes_df = pd.DataFrame()\n    indexes_df[\"rows\"] = indexes[0]\n    indexes_df[\"cols\"] = indexes[1]\n    mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())\n    max_len = mdf.apply(lambda x: len(x)).max()\n    return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values\n</pre> def get_coo_indexes(lil):     rows = []     cols = []     for i, el in enumerate(lil):         if type(el) != list:             el = [el]         for j in el:             rows.append(i)             cols.append(j)     return rows, cols   def get_sparse_features(series, shape):     coo_indexes = get_coo_indexes(series.tolist())     sparse_df = coo_matrix(         (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape     )     return sparse_df   def sparse_to_idx(data, pad_idx=-1):     indexes = data.nonzero()     indexes_df = pd.DataFrame()     indexes_df[\"rows\"] = indexes[0]     indexes_df[\"cols\"] = indexes[1]     mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())     max_len = mdf.apply(lambda x: len(x)).max()     return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values <p>For the time being, we will not use a validation set for hyperparameter optimization, and we will simply concatenate the validation and the test set in one test set. I simply splitted the data into train/valid/test in case the reader wants to actually do hyperparameter optimization (and because I know in the future I will).</p> <p>There is also another caveat worth mentioning, related to the indexing of the movies. To build the matrices of movies watched, we use the entire dataset. A more realistic (and correct) approach would be to use ONLY the movies that appear in the training set and consider <code>unknown</code> or <code>unseen</code> those in the testing set that have not been seen during training. Nonetheless, this will not affect the purposes of this notebook, which is to illustrate how one could use <code>pytorch-widedeep</code> to build a recommendation algorithm. However, if one wanted to explore the performance of different algorithms in a \"proper\" way, these \"details\" need to be accounted for.</p> In\u00a0[20]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) In\u00a0[21]: Copied! <pre>id_cols = [\"user_id\", \"movie_id\"]\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> id_cols = [\"user_id\", \"movie_id\"] max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) In\u00a0[22]: Copied! <pre>X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_train = np.array(df_train.target.values, dtype=\"int64\")\ntrain_movies_watched = get_sparse_features(\n    df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1)\n)\n\nX_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_test = np.array(df_test.target.values, dtype=\"int64\")\ntest_movies_watched = get_sparse_features(\n    df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1)\n)\n</pre> X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_train = np.array(df_train.target.values, dtype=\"int64\") train_movies_watched = get_sparse_features(     df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1) )  X_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_test = np.array(df_test.target.values, dtype=\"int64\") test_movies_watched = get_sparse_features(     df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1) ) <p>let's have a look to the information in each dataset</p> In\u00a0[23]: Copied! <pre>X_train.head()\n</pre> X_train.head() Out[23]: mean_rate unknown Action Adventure Animation Children's Comedy Crime Documentary Drama ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 25423 4.000000 0.0 0.400000 0.200000 0.0 0.0 0.400000 0.0 0.0 0.200000 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25425 4.000000 0.0 0.285714 0.142857 0.0 0.0 0.428571 0.0 0.0 0.285714 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25424 4.000000 0.0 0.333333 0.166667 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25426 3.875000 0.0 0.250000 0.125000 0.0 0.0 0.375000 0.0 0.0 0.250000 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 25427 3.888889 0.0 0.222222 0.111111 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 <p>5 rows \u00d7 43 columns</p> In\u00a0[24]: Copied! <pre>y_train\n</pre> y_train Out[24]: <pre>array([772, 288, 108, ..., 183, 432, 509])</pre> In\u00a0[25]: Copied! <pre>train_movies_watched\n</pre> train_movies_watched Out[25]: <pre>&lt;76228x1683 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 7957390 stored elements in COOrdinate format&gt;</pre> In\u00a0[26]: Copied! <pre>sorted(df_train.prev_movies.tolist()[0])\n</pre> sorted(df_train.prev_movies.tolist()[0]) Out[26]: <pre>['173', '185', '255', '286', '298']</pre> In\u00a0[27]: Copied! <pre>np.where(train_movies_watched.todense()[0])\n</pre> np.where(train_movies_watched.todense()[0]) Out[27]: <pre>(array([0, 0, 0, 0, 0]), array([173, 185, 255, 286, 298]))</pre> <p>And from now on is when the specifics related to this library start to appear. The only component that is going to be a bit different is the so-called tabular component, referred as <code>continuous</code> in the notebook.</p> <p>In the case of <code>pytorch-widedeep</code> we have the <code>TabPreprocessor</code> that allows for a lot of flexibility as to how we would like to process the tabular component of this Wide and Deep model. In other words, here our tabular component is a bit more elaborated than that in the notebook, just a bit...</p> In\u00a0[28]: Copied! <pre>cat_cols = [\"gender\", \"occupation\", \"zip_code\"]\ncont_cols = [c for c in X_train if c not in cat_cols]\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_cols,\n    continuous_cols=cont_cols,\n)\n</pre> cat_cols = [\"gender\", \"occupation\", \"zip_code\"] cont_cols = [c for c in X_train if c not in cat_cols] tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_cols,     continuous_cols=cont_cols, ) In\u00a0[29]: Copied! <pre>X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0))\nX_test_tab = tab_preprocessor.transform(X_test.fillna(0))\n</pre> X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0)) X_test_tab = tab_preprocessor.transform(X_test.fillna(0)) <p>Now, in the notebook, the author moves the sparse matrices to sparse tensors and then turns them into dense tensors. In reality, this is not neccessary, one could feed sparse tensors to <code>nn.Linear</code> layers in pytorch. Nonetheless, this is not the most efficient implementation and is the reason why in our library the wide, linear component is implemented as an embedding layer.</p> <p>Nonetheless, to reproduce the notebook the best we can and because currently the <code>Wide</code> model in <code>pytorch-widedeep</code> is not designed to receive sparse tensors (we might consider implementing this functionality), we will turn the sparse COO matrices into dense arrays. We will then code a fairly simple, custom <code>Wide</code> component.</p> In\u00a0[30]: Copied! <pre>X_train_wide = np.array(train_movies_watched.todense())\nX_test_wide = np.array(test_movies_watched.todense())\n</pre> X_train_wide = np.array(train_movies_watched.todense()) X_test_wide = np.array(test_movies_watched.todense()) <p>Finally, the author of the notebook uses a simple <code>Embedding</code> layer to encode the sequences of movies watched, the <code>prev_movies</code> columns. In my opinion, there is an element of information redundancy here. This is because the wide and text components have implicitely the same information, but in different form. Moreover, both of the models used for these two components ignore the sequential element in the data. Nonetheless, we want to reproduce the Kaggle notebook as close as possible, AND as one can explore later (by simply performing simple ablation studies), the wide component seems to carry most of the predictive power.</p> In\u00a0[31]: Copied! <pre>X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX)\nX_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX)\n</pre> X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX) X_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX) <p>Let's now build the models</p> In\u00a0[32]: Copied! <pre>class Wide(nn.Module):\n    def __init__(self, input_dim: int, pred_dim: int):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # When I coded the library I never though that someone would want to code\n        # their own wide component. However, if you do, the wide component must have\n        # a 'wide_linear' attribute. In other words, the linear layer must be\n        # called 'wide_linear'\n        self.wide_linear = nn.Linear(input_dim, pred_dim)\n\n    def forward(self, X):\n        out = self.wide_linear(X.type(torch.float32))\n        return out\n\n\nwide = Wide(X_train_wide.shape[1], max_movie_index + 1)\n</pre> class Wide(nn.Module):     def __init__(self, input_dim: int, pred_dim: int):         super().__init__()          self.input_dim = input_dim         self.pred_dim = pred_dim          # When I coded the library I never though that someone would want to code         # their own wide component. However, if you do, the wide component must have         # a 'wide_linear' attribute. In other words, the linear layer must be         # called 'wide_linear'         self.wide_linear = nn.Linear(input_dim, pred_dim)      def forward(self, X):         out = self.wide_linear(X.type(torch.float32))         return out   wide = Wide(X_train_wide.shape[1], max_movie_index + 1) In\u00a0[33]: Copied! <pre>wide\n</pre> wide Out[33]: <pre>Wide(\n  (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n)</pre> In\u00a0[34]: Copied! <pre>class SimpleEmbed(nn.Module):\n    def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):\n        super().__init__()\n\n        self.vocab_size = vocab_size\n        self.embed_dim = embed_dim\n        self.pad_idx = pad_idx\n\n        # The sequences of movies watched are simply embedded in the Kaggle\n        # notebook. No RNN, Transformer or any model is used\n        self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)\n\n    def forward(self, X):\n        embed = self.embed(X)\n        embed_mean = torch.mean(embed, dim=1)\n        return embed_mean\n\n    @property\n    def output_dim(self) -&gt; int:\n        # All deep components in a custom 'pytorch-widedeep' model must have\n        # an output_dim property\n        return self.embed_dim\n\n\n#  In the notebook the author uses simply embeddings\nsimple_embed = SimpleEmbed(max_movie_index + 1, 16, 0)\n</pre> class SimpleEmbed(nn.Module):     def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):         super().__init__()          self.vocab_size = vocab_size         self.embed_dim = embed_dim         self.pad_idx = pad_idx          # The sequences of movies watched are simply embedded in the Kaggle         # notebook. No RNN, Transformer or any model is used         self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)      def forward(self, X):         embed = self.embed(X)         embed_mean = torch.mean(embed, dim=1)         return embed_mean      @property     def output_dim(self) -&gt; int:         # All deep components in a custom 'pytorch-widedeep' model must have         # an output_dim property         return self.embed_dim   #  In the notebook the author uses simply embeddings simple_embed = SimpleEmbed(max_movie_index + 1, 16, 0) In\u00a0[35]: Copied! <pre>simple_embed\n</pre> simple_embed Out[35]: <pre>SimpleEmbed(\n  (embed): Embedding(1683, 16, padding_idx=0)\n)</pre> <p>Maybe one would like to use an RNN to account for the sequence nature of the problem. If that was the case it would be as easy as:</p> In\u00a0[36]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=max_movie_index + 1,\n    embed_dim=16,\n    hidden_dim=32,\n    n_layers=2,\n    rnn_type=\"gru\",\n)\n</pre> basic_rnn = BasicRNN(     vocab_size=max_movie_index + 1,     embed_dim=16,     hidden_dim=32,     n_layers=2,     rnn_type=\"gru\", ) <p>And finally, the tabular component, which is the notebook is simply a stak of linear + Rely layers. In our case we have an embedding layer before the linear layers to encode categorial and numerical cols</p> In\u00a0[37]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     cont_norm_layer=None,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", ) In\u00a0[38]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[38]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n      (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=111, out_features=1024, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=1024, out_features=512, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_2): Sequential(\n        (0): Linear(in_features=512, out_features=256, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Finally, we simply wrap up all models with the <code>WideDeep</code> 'collector' class and we are ready to train.</p> In\u00a0[39]: Copied! <pre>wide_deep_model = WideDeep(\n    wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1\n)\n</pre> wide_deep_model = WideDeep(     wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1 ) In\u00a0[40]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[40]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n  )\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n          (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=111, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): SimpleEmbed(\n      (embed): Embedding(1683, 16, padding_idx=0)\n    )\n    (1): Linear(in_features=16, out_features=1683, bias=True)\n  )\n)</pre> <p>Note that the main difference between this wide and deep model and the Wide and Deep model in the Kaggle notebook is that in that notebook, the author concatenates the embedings and the tabular features, then passes this concatenation through a stack of linear + Relu layers with a final output dim of 256. Then concatenates this output with the binary features and connects this concatenation with the final linear layer (so the final weights are of dim (batch_size, 256 + 1683)). Our implementation follows the notation of the original paper and instead of concatenating the tabular, text and wide components and then connect them to the output neurons, we first compute their output, and then add it (see here: https://arxiv.org/pdf/1606.07792.pdf, their Eq 3). Note that this is effectively the same, with the caveat that while in one case one initialises a big weight matrix \"at once\", in our implementation we initialise different matrices for different components. Anyway, let's give it a go.</p> In\u00a0[41]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), ) In\u00a0[42]: Copied! <pre>trainer.fit(\n    X_train={\n        \"X_wide\": X_train_wide,\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_wide\": X_test_wide,\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=5,\n    batch_size=512,\n    shuffle=False,\n)\n</pre> trainer.fit(     X_train={         \"X_wide\": X_train_wide,         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_wide\": X_test_wide,         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=5,     batch_size=512,     shuffle=False, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.66it/s, loss=6.66]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.75it/s, loss=6.6]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:21&lt;00:00,  6.95it/s, loss=5.97]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 21.03it/s, loss=6.52]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.51it/s, loss=5.65]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 20.16it/s, loss=6.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:23&lt;00:00,  6.29it/s, loss=5.41]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 13.97it/s, loss=6.57]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.58it/s, loss=5.2]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.82it/s, loss=6.63]\n</pre> <p>Now one could continue to the 'compare' metrics section of the Kaggle notebook. However, for the purposes of illustrating how one could use <code>pytorch-widedeep</code> to build recommendation algorithms we consider this notebook completed and move onto part 2</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html#problem-formulation","title":"Problem formulation\u00b6","text":"<p>In this particular exercise the problem is formulated as predicting the next movie that will be watched (in consequence the last interactions will be discarded)</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt2.html","title":"18 wide and deep for recsys pt2","text":"<p>This is the second of the two notebooks where we aim to illustrate how one could use this library to build recommendation algorithms using the example in this Kaggle notebook as guidance. In the previous notebook we used <code>pytorch-widedeep</code> to build a model that replicated almost exactly that in the notebook. In this, shorter notebook we will show how one could use the library to explore other models, following the same problem formulation, this is: given a state of a user at a certain point in time having watched a series of movies, our goal is to predict which movie the user will watch next.</p> <p>Assuming that one has read (and run) the previous notebook, the required data will be stored in a local dir called <code>prepared_data</code>, so let's read it:</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\n\nimport numpy as np\nimport torch\nimport pandas as pd\nfrom torch import nn\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.utils import pad_sequences\nfrom pytorch_widedeep.models import TabMlp, WideDeep, Transformer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pathlib import Path  import numpy as np import torch import pandas as pd from torch import nn  from pytorch_widedeep import Trainer from pytorch_widedeep.utils import pad_sequences from pytorch_widedeep.models import TabMlp, WideDeep, Transformer from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre>save_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n\nid_cols = [\"user_id\", \"movie_id\"]\n\ndf_train = pd.read_pickle(save_path / \"df_train.pkl\")\ndf_valid = pd.read_pickle(save_path / \"df_valid.pkl\")\ndf_test = pd.read_pickle(save_path / \"df_test.pkl\")\n</pre> save_path = Path(\"prepared_data\")  PAD_IDX = 0  id_cols = [\"user_id\", \"movie_id\"]  df_train = pd.read_pickle(save_path / \"df_train.pkl\") df_valid = pd.read_pickle(save_path / \"df_valid.pkl\") df_test = pd.read_pickle(save_path / \"df_test.pkl\") <p>...remember that in the previous notebook we explained that we are not  going to use a validation set here (in a real-world example, or simply a more realistic example, one should always use it).</p> In\u00a0[3]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) <p>Also remember that, in the previous notebook we discussed that the <code>'maxlen'</code> and <code>'max_movie_index'</code> parameters should be computed using only the train set. In particular, to properly do the tokenization, one would have to use ONLY train tokens and add a token for new 'unknown'/'unseen' movies in the test set. This can also be done with this library or manually, so I will leave it to the reader to implement that tokenzation appraoch.</p> In\u00a0[4]: Copied! <pre>maxlen = max(\n    df_train.prev_movies.apply(lambda x: len(x)).max(),\n    df_test.prev_movies.apply(lambda x: len(x)).max(),\n)\n\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> maxlen = max(     df_train.prev_movies.apply(lambda x: len(x)).max(),     df_test.prev_movies.apply(lambda x: len(x)).max(), )  max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) <p>From now one things are pretty simple, moreover bearing in mind that in this example we are not going to use a wide component since, in pple, one would believe that the information in that component is also 'carried' by the movie sequences (However in the previous notebook, if one performs ablation studies, these suggest that most of the prediction power comes from the linear, wide model).</p> <p>In the example here we are going to explore one (of many) possibilities. We are simply going to encode the triplet <code>(user, item, rating)</code> and use it as a <code>deeptabular</code> component and the sequences of previously watched movies as the <code>deeptext</code> component. For the <code>deeptext</code> component we are going to use a basic encoder-only transformer model.</p> <p>Let's start with the tabular data preparation</p> In\u00a0[5]: Copied! <pre>df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntrain_movies_sequences = df_train.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_train = df_train.target.values.astype(int)\n\ndf_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntest_movies_sequences = df_test.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_test = df_test.target.values.astype(int)\n\ntab_preprocessor = tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"],\n)\nX_train_tab = tab_preprocessor.fit_transform(df_train_user_item)\nX_test_tab = tab_preprocessor.transform(df_test_user_item)\n</pre> df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] train_movies_sequences = df_train.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_train = df_train.target.values.astype(int)  df_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] test_movies_sequences = df_test.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_test = df_test.target.values.astype(int)  tab_preprocessor = tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"], ) X_train_tab = tab_preprocessor.fit_transform(df_train_user_item) X_test_tab = tab_preprocessor.transform(df_test_user_item) <p>And not the text component, simply padding the sequences:</p> In\u00a0[6]: Copied! <pre>X_train_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=PAD_IDX,\n        )\n        for s in train_movies_sequences\n    ]\n)\nX_test_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=0,\n        )\n        for s in test_movies_sequences\n    ]\n)\n</pre> X_train_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=PAD_IDX,         )         for s in train_movies_sequences     ] ) X_test_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=0,         )         for s in test_movies_sequences     ] ) <p>We now define the model components and the wide and deep model.</p> In\u00a0[7]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n\n# plenty of options here, see the docs\ntransformer = Transformer(\n    vocab_size=max_movie_index + 1,\n    embed_dim=32,\n    n_heads=2,\n    n_blocks=2,\n    seq_length=maxlen,\n)\n\nwide_deep_model = WideDeep(\n    deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", )  # plenty of options here, see the docs transformer = Transformer(     vocab_size=max_movie_index + 1,     embed_dim=32,     n_heads=2,     n_blocks=2,     seq_length=maxlen, )  wide_deep_model = WideDeep(     deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1 ) In\u00a0[8]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[8]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_user_id): Embedding(749, 65, padding_idx=0)\n          (emb_layer_movie_id): Embedding(1612, 100, padding_idx=0)\n          (emb_layer_rating): Embedding(6, 4, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=169, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): Transformer(\n      (embedding): Embedding(1683, 32, padding_idx=0)\n      (pos_encoder): PositionalEncoding(\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=23552, out_features=1683, bias=True)\n  )\n)</pre> <p>And as in the previous notebook, let's train (you will need a GPU for this)</p> In\u00a0[\u00a0]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n\ntrainer.fit(\n    X_train={\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=10,\n    batch_size=521,\n    shuffle=False,\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), )  trainer.fit(     X_train={         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=10,     batch_size=521,     shuffle=False, ) <pre>epoch 1:   0%|                                                                                                         | 0/147 [00:34&lt;?, ?it/s]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html","title":"19 load from folder functionality","text":"<p>In this notebook I want to illustrate how one can use our <code>[...]FromFolder</code> functionalities along with the <code>[...]ChunkPreProcessors</code> in those cases where the dataset is too bit to fit in memory.</p> <p>These functionalities in the library have been designed for the following scenarop</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\nfrom torch.utils.data import DataLoader\n\nfrom pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import TrainerFromFolder\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import (\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n    ChunkTabPreprocessor,\n    ChunkTextPreprocessor,\n)\nfrom pytorch_widedeep.load_from_folder import (\n    TabFromFolder,\n    TextFromFolder,\n    ImageFromFolder,\n    WideDeepDatasetFromFolder,\n)\n</pre> import numpy as np import torch import pandas as pd from torch.utils.data import DataLoader  from pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep from pytorch_widedeep.training import TrainerFromFolder from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import (     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor,     ChunkTabPreprocessor,     ChunkTextPreprocessor, ) from pytorch_widedeep.load_from_folder import (     TabFromFolder,     TextFromFolder,     ImageFromFolder,     WideDeepDatasetFromFolder, ) In\u00a0[2]: Copied! <pre># in my case, I place the data in a folder I call tmp_data, let's see how it looks\nairbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\nairbnb_data.sample(5)\n</pre> # in my case, I place the data in a folder I call tmp_data, let's see how it looks airbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") airbnb_data.sample(5) Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 39 53242.jpg 247650 A lovely big bright bedroom in a 2 bedroom fla... 2.0 t Lambeth 51.47075 -0.12913 t apartment ... 0 0 0 0 1 0 0 250.0 5.0 9.75 214 236716.jpg 1241070 We offer a warm welcome in our quiet double ro... 1.0 t Hackney 51.56593 -0.07482 t other ... 0 0 0 0 1 0 0 200.0 10.0 76.50 400 346523.jpg 1756532 Available for you to rent is a cozy studio in ... 2.0 t Kensington and Chelsea 51.48311 -0.18428 t other ... 0 0 0 0 1 0 0 0.0 50.0 180.90 512 389627.jpg 1949299 This gorgeous studio flat is situated in the v... 1.0 t Westminster 51.51838 -0.14238 f apartment ... 0 0 0 0 1 0 0 250.0 25.0 276.90 504 388767.jpg 1945165 If you want to experience London at it's best ... 2.0 f Camden 51.54293 -0.14073 t apartment ... 0 0 0 0 1 0 0 150.0 10.0 591.10 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># for example\nfrom IPython.display import Image\n\npil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\")\ndisplay(pil_img)\n</pre> # for example from IPython.display import Image  pil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\") display(pil_img) In\u00a0[4]: Copied! <pre># And the description for the property that that picture belongs to is:\nairbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist()\n</pre> # And the description for the property that that picture belongs to is: airbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist() Out[4]: <pre>[\"Bright, sunny beautiful room that will give you the perfect base to explore all of London. Come and explore one of London's best neighbourhoods - Herne Hill! As mentioned in (Website hidden by Airbnb)   (Website hidden by Airbnb)  WiFi availability with a fully stocked and clean uplifting home. Lovely sunny, airy and big double bedroom on a leafy south-London street.    Note: This room comes with a reserved Off-Street parking spot! The room is on the first floor and boasts an enormous Super King bed, gorgeous wooden floors, tall ceilings and large windows which let in the sunshine almost all day. (Yoga May or meditation cushion available on request) The flat is bright and airy and big! So lots of space for all.  Location wise you are only 10 minutes walk to either Herne Hill or West Dulwich stations, both of which will take you to Victoria and the city within minutes. You can also hop on a bus right outside the flat that will take you to Brixton tube station within 8 minutes where you \"]</pre> <p>Ok, so we have tabular data where one column is <code>description</code> and another <code>id</code>, points towards the images stored in disk. Now, remember the following, because this will appear a few times in the notebook: our \"reference dataset\" is the tabular data.</p> <p>Therefore, since I want to illustrate a \"semi-realistic\" case, if we need to split the data into training, validation and test datasets, these datasets needs to be separetely stored in disk. In my case I have done this and in the <code>tmp_data/airbnb</code> dir I have the following:</p> <pre>../tmp_data/airbnb\n\u251c\u2500\u2500 airbnb_sample.csv\n\u251c\u2500\u2500 airbnb_sample_eval.csv\n\u251c\u2500\u2500 airbnb_sample_test.csv\n\u251c\u2500\u2500 airbnb_sample_train.csv\n\u2514\u2500\u2500 property_picture\n</pre> <p>Where <code>airbnb_sample.csv</code> is the full sample (1001 rows) and the <code>train</code>, <code>eval</code> and <code>test</code> set is the corresponding splits. In a realistic example, the full sample would be the 'gigantic' dataset and the rest the corresponding splits. One has to do this 'offline', prior to start the coding.</p> <p>Also, one thing that one needs to know is the number of total observations/rows, as well as the splits. In our case the train size is 800, and the eval and test sizes are 100 and 101 respectively.</p> <p>With all that info, let's star</p> In\u00a0[5]: Copied! <pre># path to the tabular data and the splits\ndata_path = \"../tmp_data/airbnb/\"\ntrain_fname = \"airbnb_sample_train.csv\"\neval_fname = \"airbnb_sample_eval.csv\"\ntest_fname = \"airbnb_sample_test.csv\"\n\n# split sizes\ntrain_size = 800\neval_size = 100\ntest_size = 101\n\n# number of chunks for the Chunk Preprocessors\nchunksize = 100\nn_chunks = int(np.ceil(train_size / chunksize))\n\n# path to the image dataset and name of the image col\nimg_path = \"../tmp_data/airbnb/property_picture/\"\nimg_col = \"id\"\n\n# name of the text col\ntext_col = \"description\"\n\n# mane of the target\ntarget_col = \"yield\"\n\n# definition of the categorical and continuous cols for the TabPreprocessor\ncat_embed_cols = [\n    \"host_listings_count\",\n    \"neighbourhood_cleansed\",\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"accommodates\",\n    \"bathrooms\",\n    \"bedrooms\",\n    \"beds\",\n    \"guests_included\",\n    \"minimum_nights\",\n    \"instant_bookable\",\n    \"cancellation_policy\",\n    \"has_house_rules\",\n    \"host_gender\",\n    \"accommodates_catg\",\n    \"guests_included_catg\",\n    \"minimum_nights_catg\",\n    \"host_listings_count_catg\",\n    \"bathrooms_catg\",\n    \"bedrooms_catg\",\n    \"beds_catg\",\n    \"security_deposit\",\n    \"extra_people\",\n]\ncont_cols = [\"latitude\", \"longitude\"]\n</pre> # path to the tabular data and the splits data_path = \"../tmp_data/airbnb/\" train_fname = \"airbnb_sample_train.csv\" eval_fname = \"airbnb_sample_eval.csv\" test_fname = \"airbnb_sample_test.csv\"  # split sizes train_size = 800 eval_size = 100 test_size = 101  # number of chunks for the Chunk Preprocessors chunksize = 100 n_chunks = int(np.ceil(train_size / chunksize))  # path to the image dataset and name of the image col img_path = \"../tmp_data/airbnb/property_picture/\" img_col = \"id\"  # name of the text col text_col = \"description\"  # mane of the target target_col = \"yield\"  # definition of the categorical and continuous cols for the TabPreprocessor cat_embed_cols = [     \"host_listings_count\",     \"neighbourhood_cleansed\",     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"accommodates\",     \"bathrooms\",     \"bedrooms\",     \"beds\",     \"guests_included\",     \"minimum_nights\",     \"instant_bookable\",     \"cancellation_policy\",     \"has_house_rules\",     \"host_gender\",     \"accommodates_catg\",     \"guests_included_catg\",     \"minimum_nights_catg\",     \"host_listings_count_catg\",     \"bathrooms_catg\",     \"bedrooms_catg\",     \"beds_catg\",     \"security_deposit\",     \"extra_people\", ] cont_cols = [\"latitude\", \"longitude\"] In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    default_embed_dim=8,\n    verbose=0,\n)\n\ntext_preprocessor = TextPreprocessor(\n    text_col=text_col,\n    n_cpus=1,\n)\n\nimg_preprocessor = ImagePreprocessor(\n    img_col=img_col,\n    img_path=img_path,\n)\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     default_embed_dim=8,     verbose=0, )  text_preprocessor = TextPreprocessor(     text_col=text_col,     n_cpus=1, )  img_preprocessor = ImagePreprocessor(     img_col=img_col,     img_path=img_path, ) In\u00a0[7]: Copied! <pre>tab_preprocessor.fit(airbnb_data)\ntext_preprocessor.fit(airbnb_data)\nimg_preprocessor.fit(airbnb_data)\n</pre> tab_preprocessor.fit(airbnb_data) text_preprocessor.fit(airbnb_data) img_preprocessor.fit(airbnb_data) <pre>The vocabulary contains 2192 tokens\n</pre> Out[7]: <pre>ImagePreprocessor(img_col=id, img_path=../tmp_data/airbnb/property_picture/, width=224, height=224, verbose=1)</pre> In\u00a0[8]: Copied! <pre>chunk_tab_preprocessor = ChunkTabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    n_chunks=n_chunks,\n    default_embed_dim=8,\n    verbose=0,\n)\n\nchunk_text_preprocessor = ChunkTextPreprocessor(\n    n_chunks=n_chunks,\n    text_col=text_col,\n    n_cpus=1,\n    verbose=0,\n)\n\nfor i, chunk in enumerate(\n    pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize)\n):\n    print(f\"chunk in loop: {i + 1}\")\n    chunk_tab_preprocessor.fit(chunk)\n    chunk_text_preprocessor.fit(chunk)\n</pre> chunk_tab_preprocessor = ChunkTabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     n_chunks=n_chunks,     default_embed_dim=8,     verbose=0, )  chunk_text_preprocessor = ChunkTextPreprocessor(     n_chunks=n_chunks,     text_col=text_col,     n_cpus=1,     verbose=0, )  for i, chunk in enumerate(     pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize) ):     print(f\"chunk in loop: {i + 1}\")     chunk_tab_preprocessor.fit(chunk)     chunk_text_preprocessor.fit(chunk) <pre>chunk in loop: 1\nchunk in loop: 2\nchunk in loop: 3\nchunk in loop: 4\nchunk in loop: 5\nchunk in loop: 6\nchunk in loop: 7\nchunk in loop: 8\n</pre> In\u00a0[9]: Copied! <pre>train_tab_folder = TabFromFolder(\n    fname=train_fname,\n    directory=data_path,\n    target_col=target_col,\n    preprocessor=tab_preprocessor,\n    text_col=text_col,\n    img_col=img_col,\n)\n\n# Note how we can use the `train_tab_folder` as reference so we don't have to\n# define all parameters again\neval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)\n\n# Note that for the test set we can ignore the target as no metric will be\n# computed by the `predict` method\ntest_tab_folder = TabFromFolder(\n    fname=test_fname, reference=train_tab_folder, ignore_target=True\n)\n</pre> train_tab_folder = TabFromFolder(     fname=train_fname,     directory=data_path,     target_col=target_col,     preprocessor=tab_preprocessor,     text_col=text_col,     img_col=img_col, )  # Note how we can use the `train_tab_folder` as reference so we don't have to # define all parameters again eval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)  # Note that for the test set we can ignore the target as no metric will be # computed by the `predict` method test_tab_folder = TabFromFolder(     fname=test_fname, reference=train_tab_folder, ignore_target=True ) In\u00a0[10]: Copied! <pre># for the text and image datasets we do not need to specify eval or test loaders\ntext_folder = TextFromFolder(preprocessor=text_preprocessor)\nimg_folder = ImageFromFolder(preprocessor=img_preprocessor)\n</pre> # for the text and image datasets we do not need to specify eval or test loaders text_folder = TextFromFolder(preprocessor=text_preprocessor) img_folder = ImageFromFolder(preprocessor=img_preprocessor) In\u00a0[11]: Copied! <pre>train_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=train_size,\n    tab_from_folder=train_tab_folder,\n    text_from_folder=text_folder,\n    img_from_folder=img_folder,\n)\n\n# Note that the eval and test loaders only need their corresponding\n# `TabFromFolder` classes. The rest of the parameters can be defined\n# via a `reference` `TabFromFolder` class\neval_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=eval_size,\n    tab_from_folder=eval_tab_folder,\n    reference=train_dataset_folder,\n)\n\ntest_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=test_size,\n    tab_from_folder=test_tab_folder,\n    reference=train_dataset_folder,\n)\n</pre> train_dataset_folder = WideDeepDatasetFromFolder(     n_samples=train_size,     tab_from_folder=train_tab_folder,     text_from_folder=text_folder,     img_from_folder=img_folder, )  # Note that the eval and test loaders only need their corresponding # `TabFromFolder` classes. The rest of the parameters can be defined # via a `reference` `TabFromFolder` class eval_dataset_folder = WideDeepDatasetFromFolder(     n_samples=eval_size,     tab_from_folder=eval_tab_folder,     reference=train_dataset_folder, )  test_dataset_folder = WideDeepDatasetFromFolder(     n_samples=test_size,     tab_from_folder=test_tab_folder,     reference=train_dataset_folder, ) In\u00a0[12]: Copied! <pre>train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1)\neval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1)\ntest_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1)\n</pre> train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1) eval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1) test_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1) <p>And from here on is business as usual:</p> In\u00a0[13]: Copied! <pre># for example\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=32,\n    hidden_dim=64,\n    n_layers=2,\n)\n\ndeepimage = Vision()\n\ndeepdense = TabMlp(\n    mlp_hidden_dims=[32, 16],\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=cont_cols,\n)\n\nmodel = WideDeep(\n    deeptabular=deepdense,\n    deeptext=basic_rnn,\n    deepimage=deepimage,\n)\n\nmodel\n</pre> # for example basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=32,     hidden_dim=64,     n_layers=2, )  deepimage = Vision()  deepdense = TabMlp(     mlp_hidden_dims=[32, 16],     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=cont_cols, )  model = WideDeep(     deeptabular=deepdense,     deeptext=basic_rnn,     deepimage=deepimage, )  model Out[13]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_host_listings_count): Embedding(28, 10, padding_idx=0)\n          (emb_layer_neighbourhood_cleansed): Embedding(33, 11, padding_idx=0)\n          (emb_layer_is_location_exact): Embedding(3, 2, padding_idx=0)\n          (emb_layer_property_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_room_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates): Embedding(14, 7, padding_idx=0)\n          (emb_layer_bathrooms): Embedding(11, 6, padding_idx=0)\n          (emb_layer_bedrooms): Embedding(7, 4, padding_idx=0)\n          (emb_layer_beds): Embedding(11, 6, padding_idx=0)\n          (emb_layer_guests_included): Embedding(11, 6, padding_idx=0)\n          (emb_layer_minimum_nights): Embedding(25, 9, padding_idx=0)\n          (emb_layer_instant_bookable): Embedding(3, 2, padding_idx=0)\n          (emb_layer_cancellation_policy): Embedding(6, 4, padding_idx=0)\n          (emb_layer_has_house_rules): Embedding(3, 2, padding_idx=0)\n          (emb_layer_host_gender): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_guests_included_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_minimum_nights_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_host_listings_count_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_bathrooms_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_bedrooms_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_beds_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_security_deposit): Embedding(53, 15, padding_idx=0)\n          (emb_layer_extra_people): Embedding(39, 12, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=128, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=32, out_features=16, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=16, out_features=1, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(2192, 32, padding_idx=1)\n      (rnn): LSTM(32, 64, num_layers=2, batch_first=True, dropout=0.1)\n      (rnn_mlp): Identity()\n    )\n    (1): Linear(in_features=64, out_features=1, bias=True)\n  )\n  (deepimage): Sequential(\n    (0): Vision(\n      (features): Sequential(\n        (conv_layer_0): Sequential(\n          (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n          (1): BatchNorm2d(64, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (maxpool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n        )\n        (conv_layer_1): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_2): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_3): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (adaptiveavgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n        )\n      )\n    )\n    (1): Linear(in_features=512, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[14]: Copied! <pre>trainer = TrainerFromFolder(\n    model,\n    objective=\"regression\",\n)\n\ntrainer.fit(\n    train_loader=train_loader,\n    eval_loader=eval_loader,\n)\n</pre> trainer = TrainerFromFolder(     model,     objective=\"regression\", )  trainer.fit(     train_loader=train_loader,     eval_loader=eval_loader, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 50/50 [03:41&lt;00:00,  4.42s/it, loss=1.64e+4]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:23&lt;00:00,  3.30s/it, loss=6.27e+3]\n</pre> In\u00a0[15]: Copied! <pre>preds = trainer.predict(test_loader=test_loader)\n</pre> preds = trainer.predict(test_loader=test_loader) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:22&lt;00:00,  3.26s/it]\n</pre> <p>Note that in the case of predict you could also choose to do this</p> In\u00a0[16]: Copied! <pre>df_test = pd.read_csv(\"/\".join([data_path, test_fname]))\n</pre> df_test = pd.read_csv(\"/\".join([data_path, test_fname])) In\u00a0[17]: Copied! <pre># if the images for the test set fit in memory\nX_tab_test = chunk_tab_preprocessor.transform(df_test)\nX_text_test = chunk_text_preprocessor.transform(df_test)\nX_img_test = img_preprocessor.transform(df_test)\n</pre> # if the images for the test set fit in memory X_tab_test = chunk_tab_preprocessor.transform(df_test) X_text_test = chunk_text_preprocessor.transform(df_test) X_img_test = img_preprocessor.transform(df_test) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 101/101 [00:00&lt;00:00, 708.23it/s]</pre> <pre>Computing normalisation metrics\n</pre> <pre>\n</pre> In\u00a0[18]: Copied! <pre>preds = trainer.predict(\n    X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32\n)\n</pre> preds = trainer.predict(     X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32 ) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:03&lt;00:00,  1.14it/s]\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html#scenario","title":"Scenario\u00b6","text":"<p>We have a tabular dataset combined with images and text and either some, or all these datasets do not fit in memory. Note that the tabular dataset MUST ALWAYS be present as it is considered the rerefence. This is, if we have an image dataset, the tabular dataset must contain a column that points to the image file names as stored in disk. Similarly, if we have a text dataset, then the tabular dataset must contain a column with the texts themselves or a the file names of the text files as stored in disk.</p> <p>If you only have text and/or images and not a tabular component, I would suggest using other libraries (such as hugginface probably).</p> <p>Within this scenario, they are two possible scenarios that we will cover here:</p> <ol> <li><p>The tabular data itsel fits in memory and is only the images that do not: in this case you could use the 'standard' <code>Preprocessors</code> in the library and off you go, move directly to the <code>[...]FromFolder</code> functionalities</p> </li> <li><p>The tabular data is also very large and does not fit in memory, so we have to process it in chuncks. For this second case I have created the so called <code>Chunk[...]Preprocessor</code> (Wide, Tab and Text).</p> </li> </ol> <p>Note that at the moment ONLY csv format is allowed for the tabular file. More formats will be supported in the future.</p> <p>Let's see a complete example to illustrate how each of these cases would be addressed with the new functionalities in the library. For this example we will use a sample of the airbnb dataset</p> <p>The airbnb dataset, which you could get from here, is too big to be included in our datasets module (when including images). Therefore, what I did was, go there, download it, and use the download_images.py script to get the images and the airbnb_data_processing.py to process the data. I did this ages ago and I believe the format of the dataset might be different now. Nonetheless, I will show samples of the dataset as we go through so you can extrapolate the content of this notebook to your particular problem.</p> <p>In the future we will find better datasets\ud83d\ude42. Finally, note that here we are only using a small sample to illustrate the use, so PLEASE ignore the results, just focus on usage.</p>"},{"location":"examples/19_load_from_folder_functionality.html#setting-variables-and-constants","title":"Setting variables and constants\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-1-the-preprocessors","title":"Step 1: the preprocessors\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#scenario-1-only-the-images-do-not-fit-in-disk","title":"Scenario 1: only the images do not fit in disk\u00b6","text":"<p>In this case we can prepare the data in the 'standard' way</p>"},{"location":"examples/19_load_from_folder_functionality.html#scenario-2-the-tabular-data-is-also-huge","title":"Scenario 2: the tabular data is also huge\u00b6","text":"<p>Then we need to prepare it in chunks. Note that, unfortunately, the tabular and text preprocessors need to see the whole dataset once. This is because to process tabular or text data we need to encode values. For those encodings to be consistent they need to have seen the whole dataset. Alternatively, one could code a solution with some streaming encoder for both datasets. However, such implementation is not trivial for this library (and in general). I also don't think that having to see the whole data once is such a big limitation. Let's see how is done.</p> <p>Note that I have not mentioned the image dataset. This is because the processing of the image dataset does not require any form of encoding and in consequence can be done 'on the fly'. Therefore, no <code>ChunkImgPreprocessor</code> processor is needed.</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-2-the-fromfolder-classes","title":"Step 2: the <code>[...]FromFolder</code> classes\u00b6","text":"<p>Once we have the preprocessors, we need to instantiate the classes that will enable us to load the data from their respective folders. From now on I am going to proceed with the <code>chunk_tab_preprocessor</code>, <code>chunk_text_preprocessor</code> and <code>img_preprocessor</code>, but the code would be identical if instead of the first two preprocessors we decided to use the <code>tab_preprocessor</code> and <code>text_preprocessor</code>.</p> <p>Once more, our reference datasets are the tabular datasets, which we have splitted in train, eval and test prior to start the coding. Therefore, we will eventually need a loader for each split</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-3-pytorch-datasets-and-dataloaders","title":"Step 3: pytorch datasets and dataloaders\u00b6","text":"<p>From here in advance, is all very 'standard' if you are familiar with pytorch. One needs to define a class that inherits from the <code>Dataset</code> class in pytorch. Then this will be passed to a <code>DataLoader</code> class and we are ready to train. Our <code>Dataset</code> child class is <code>WideDeepDatasetFromFolder</code>. This class will use the tabular dataset and the corresponding text and image columns to load the adequate data in the batches</p> <p>Let's do it</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-4-define-the-model","title":"Step 4: define the model\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-5-fit-and-predict","title":"Step 5: fit and predict\u00b6","text":""},{"location":"examples/20_Using_huggingface_within_widedeep.html","title":"20 Using huggingface within widedeep","text":"<p>In this notebook we will show how to use Hugginface's tokenizers and models as they are integrated within the library. In notebook number 17 you can find examples on how to code your own, custom, Hugginface (hereafter HF) model and use it in combination of any other model in the library</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import HFModel, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.preprocessing import HFPreprocessor\n</pre> import numpy as np import pandas as pd from sklearn.metrics import f1_score, accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import HFModel, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.preprocessing import HFPreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore\n</pre> df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore In\u00a0[3]: Copied! <pre>df.shape\n</pre> df.shape Out[3]: <pre>(23486, 10)</pre> In\u00a0[4]: Copied! <pre>df.sample(3)\n</pre> df.sample(3) Out[4]: Clothing ID Age Title Review Text Rating Recommended IND Positive Feedback Count Division Name Department Name Class Name 7004 862 43 Cute and feminine Loved this sweater wrap and bought it in both ... 5 1 2 General Tops Knits 12508 975 66 Love it The linen fabric is elegantly thin feels and l... 5 1 3 General Jackets Jackets 10288 950 41 Perfect for fall This sweater is just as pictured. the fit is t... 5 1 0 General Tops Sweaters In\u00a0[5]: Copied! <pre># Let's do some mild preprocessing\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n</pre> # Let's do some mild preprocessing df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\") In\u00a0[6]: Copied! <pre># drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[7]: Copied! <pre>df.shape\n</pre> df.shape Out[7]: <pre>(22608, 10)</pre> In\u00a0[8]: Copied! <pre># if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random\n# If you run this on a GPU you can comment out the following two cells\ndf.rating.value_counts()\n</pre> # if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random # If you run this on a GPU you can comment out the following two cells df.rating.value_counts() Out[8]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> In\u00a0[9]: Copied! <pre>df = (\n    df.groupby(\"rating\", group_keys=False)\n    .apply(lambda x: x.sample(min(len(x), 2369)))\n    .sample(1000)\n)\n</pre> df = (     df.groupby(\"rating\", group_keys=False)     .apply(lambda x: x.sample(min(len(x), 2369)))     .sample(1000) ) <pre>/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/ipykernel_5886/895673206.py:3: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n  .apply(lambda x: x.sample(min(len(x), 2369)))\n</pre> In\u00a0[10]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n\n# possible model names currently supported in the library\nmodel_names = [\n    \"distilbert-base-uncased\",\n    \"bert-base-uncased\",\n    \"FacebookAI/roberta-base\",\n    \"albert-base-v2\",\n    \"google/electra-base-discriminator\",\n]\n\n# Let's choose one. The syntax is the same for all the models\nmodel_name = \"distilbert-base-uncased\"\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)  # possible model names currently supported in the library model_names = [     \"distilbert-base-uncased\",     \"bert-base-uncased\",     \"FacebookAI/roberta-base\",     \"albert-base-v2\",     \"google/electra-base-discriminator\", ]  # Let's choose one. The syntax is the same for all the models model_name = \"distilbert-base-uncased\" <p>Now we can use the <code>HFPreprocessor</code> class. As most things in this library, the integration with HF has been coded aiming for a flexible use. With this in mind, there are two ways one can use a <code>HFPreprocessor</code> class.</p> <ol> <li>Passing a <code>text_col</code> and <code>encode_params</code> as the class is instantiated and then using the <code>fit</code> and <code>transform</code> as with any other preprocessor in the library</li> <li>Without passing  <code>text_col</code> and <code>encode_params</code> as the class is instantiated and using the <code>encode</code> method of the <code>HFPreprocessor</code> which is simply a wrapper around the encode method of HF's tokenizers</li> </ol> <p>Let's have a look</p> In\u00a0[11]: Copied! <pre>tokenizer1 = HFPreprocessor(\n    model_name=model_name,\n    text_col=\"review_text\",\n    num_workers=1,\n    encode_params={\n        \"max_length\": 90,\n        \"padding\": \"max_length\",\n        \"truncation\": True,\n        \"add_special_tokens\": True,\n    },\n)\n\nX_text_tr1 = tokenizer1.fit_transform(train)\nX_text_te1 = tokenizer1.transform(test)\n</pre> tokenizer1 = HFPreprocessor(     model_name=model_name,     text_col=\"review_text\",     num_workers=1,     encode_params={         \"max_length\": 90,         \"padding\": \"max_length\",         \"truncation\": True,         \"add_special_tokens\": True,     }, )  X_text_tr1 = tokenizer1.fit_transform(train) X_text_te1 = tokenizer1.transform(test) In\u00a0[12]: Copied! <pre>tokenizer2 = HFPreprocessor(\n    model_name=model_name,\n    num_workers=1,\n)\n\nX_text_tr2 = tokenizer2.encode(\n    train.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\nX_text_te2 = tokenizer2.encode(\n    test.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\n</pre> tokenizer2 = HFPreprocessor(     model_name=model_name,     num_workers=1, )  X_text_tr2 = tokenizer2.encode(     train.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) X_text_te2 = tokenizer2.encode(     test.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) In\u00a0[13]: Copied! <pre>all(X_text_tr1[0] == X_text_tr2[0])\n</pre> all(X_text_tr1[0] == X_text_tr2[0]) Out[13]: <pre>True</pre> In\u00a0[14]: Copied! <pre># Now we define a model which is as easy as:\n# Note that this will instantiation will lead to NO parameter trainable in the HF model.\n# If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument.\n# Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details)\nhf_model = HFModel(model_name=model_name)\n</pre> # Now we define a model which is as easy as: # Note that this will instantiation will lead to NO parameter trainable in the HF model. # If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument. # Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details) hf_model = HFModel(model_name=model_name) In\u00a0[15]: Copied! <pre># And from here on is the same as any other WideDeep model\nmodel = WideDeep(\n    deeptext=hf_model,\n    pred_dim=4,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"multiclass\",\n    metrics=[Accuracy(), F1Score(average=True)],\n)\n\ntrainer.fit(\n    X_text=X_text_tr2,\n    target=train.rating.values,\n    n_epochs=1,\n    batch_size=64,\n)\n# If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo\n</pre> # And from here on is the same as any other WideDeep model model = WideDeep(     deeptext=hf_model,     pred_dim=4, )  trainer = Trainer(     model,     objective=\"multiclass\",     metrics=[Accuracy(), F1Score(average=True)], )  trainer.fit(     X_text=X_text_tr2,     target=train.rating.values,     n_epochs=1,     batch_size=64, ) # If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 13/13 [02:06&lt;00:00,  9.75s/it, loss=3.2, metrics={'acc': 0.235, 'f1': 0.2336}]  \n</pre> In\u00a0[17]: Copied! <pre>preds_text = trainer.predict_proba(X_text=X_text_te2)\npred_text_class = np.argmax(preds_text, 1)\n\nacc_text = accuracy_score(test.rating, pred_text_class)\nf1_text = f1_score(test.rating, pred_text_class, average=\"weighted\")\nprint(f\"Accuracy: {acc_text:.4f}\")\nprint(f\"F1: {f1_text:.4f}\")\n</pre> preds_text = trainer.predict_proba(X_text=X_text_te2) pred_text_class = np.argmax(preds_text, 1)  acc_text = accuracy_score(test.rating, pred_text_class) f1_text = f1_score(test.rating, pred_text_class, average=\"weighted\") print(f\"Accuracy: {acc_text:.4f}\") print(f\"F1: {f1_text:.4f}\") <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:05&lt;00:00,  1.43s/it]</pre> <pre>Accuracy: 0.2500\nF1: 0.1000\n</pre> <pre>\n</pre>"},{"location":"pytorch-widedeep/bayesian_models.html","title":"The <code>bayesian models</code> module","text":"<p>This module contains the two Bayesian Models available in this library, namely the bayesian version of the <code>Wide</code> and <code>TabMlp</code> models, referred as <code>BayesianWide</code> and <code>BayesianTabMlp</code>. These models are very useful in scenarios where getting a measure of uncertainty is important.</p> <p>The models in this module are based on the publication: Weight Uncertainty in Neural Networks.</p>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide","title":"BayesianWide","text":"<pre><code>BayesianWide(\n    input_dim,\n    pred_dim=1,\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0.0,\n    posterior_rho_init=-7.0,\n)\n</code></pre> <p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>Wide</code> model. This is a linear model where the non-linearlities are captured via crossed-columns</p> <p>Parameters:</p> <ul> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide component. For example, if the wide component receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>size of the ouput tensor containing the predictions</p> </li> <li> <code>prior_sigma_1</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> </li> <li> <code>prior_sigma_2</code>               (<code>float</code>, default:                   <code>0.002</code> )           \u2013            <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution</p> </li> <li> <code>prior_pi</code>               (<code>float</code>, default:                   <code>0.8</code> )           \u2013            <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution</p> </li> <li> <code>posterior_mu_init</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>The posterior sample of the weights is defined as:</p> \\[    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} \\] <p>where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> </li> <li> <code>posterior_rho_init</code>               (<code>float</code>, default:                   <code>-7.0</code> )           \u2013            <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>bayesian_wide_linear</code>               (<code>Module</code>)           \u2013            <p>the linear layer that comprises the wide branch of the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n&gt;&gt;&gt; wide = BayesianWide(input_dim=X.unique().size(0), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code> <pre><code>def __init__(\n    self,\n    input_dim: int,\n    pred_dim: int = 1,\n    prior_sigma_1: float = 1.0,\n    prior_sigma_2: float = 0.002,\n    prior_pi: float = 0.8,\n    posterior_mu_init: float = 0.0,\n    posterior_rho_init: float = -7.0,\n):\n    super(BayesianWide, self).__init__()\n    #  Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n    self.bayesian_wide_linear = bnn.BayesianEmbedding(\n        n_embed=input_dim + 1,\n        embed_dim=pred_dim,\n        padding_idx=0,\n        prior_sigma_1=prior_sigma_1,\n        prior_sigma_2=prior_sigma_2,\n        prior_pi=prior_pi,\n        posterior_mu_init=posterior_mu_init,\n        posterior_rho_init=posterior_rho_init,\n    )\n    self.bias = nn.Parameter(torch.zeros(pred_dim))\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp","title":"BayesianTabMlp","text":"<pre><code>BayesianTabMlp(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    embed_continuous=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    use_cont_bias=None,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[200, 100],\n    mlp_activation=\"leaky_relu\",\n    prior_sigma_1=1,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0.0,\n    posterior_rho_init=-7.0,\n    pred_dim=1\n)\n</code></pre> <p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>BayesianTabMlp</code> model.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of probabilistic dense layers (i.e. a MLP).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm', 'batchnorm' or None.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded (i.e. passed each through a linear layer with or without activation)</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings</p> </li> <li> <code>use_cont_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the continuous embeddings</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>prior_sigma_1</code>               (<code>float</code>, default:                   <code>1</code> )           \u2013            <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> </li> <li> <code>prior_sigma_2</code>               (<code>float</code>, default:                   <code>0.002</code> )           \u2013            <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution for each Bayesian linear and embedding layer</p> </li> <li> <code>prior_pi</code>               (<code>float</code>, default:                   <code>0.8</code> )           \u2013            <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution ffor each Bayesian linear and embedding layer</p> </li> <li> <code>posterior_mu_init</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>The posterior sample of the weights is defined as:</p> <p>$$    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} $$ where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> </li> <li> <code>posterior_rho_init</code>               (<code>float</code>, default:                   <code>-7.0</code> )           \u2013            <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>bayesian_cat_and_cont_embed</code>               (<code>Module</code>)           \u2013            <p>This is the module that processes the categorical and continuous columns</p> </li> <li> <code>bayesian_tab_mlp</code>               (<code>Sequential</code>)           \u2013            <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    embed_continuous: Optional[bool] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    use_cont_bias: Optional[bool] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    mlp_hidden_dims: List[int] = [200, 100],\n    mlp_activation: str = \"leaky_relu\",\n    prior_sigma_1: float = 1,\n    prior_sigma_2: float = 0.002,\n    prior_pi: float = 0.8,\n    posterior_mu_init: float = 0.0,\n    posterior_rho_init: float = -7.0,\n    pred_dim=1,  # Bayesian models will require their own trainer and need the output layer\n):\n    super(BayesianTabMlp, self).__init__()\n\n    self.column_idx = column_idx\n    self.cat_embed_input = cat_embed_input\n    self.cat_embed_activation = cat_embed_activation\n\n    self.continuous_cols = continuous_cols\n    self.cont_norm_layer = cont_norm_layer\n    self.embed_continuous = embed_continuous\n    self.cont_embed_dim = cont_embed_dim\n    self.cont_embed_dropout = cont_embed_dropout\n    self.use_cont_bias = use_cont_bias\n    self.cont_embed_activation = cont_embed_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n\n    self.prior_sigma_1 = prior_sigma_1\n    self.prior_sigma_2 = prior_sigma_2\n    self.prior_pi = prior_pi\n    self.posterior_mu_init = posterior_mu_init\n    self.posterior_rho_init = posterior_rho_init\n\n    self.pred_dim = pred_dim\n\n    allowed_activations = [\"relu\", \"leaky_relu\", \"tanh\", \"gelu\"]\n    if self.mlp_activation not in allowed_activations:\n        raise ValueError(\n            \"Currently, only the following activation functions are supported \"\n            \"for the Bayesian MLP's dense layers: {}. Got '{}' instead\".format(\n                \", \".join(allowed_activations),\n                self.mlp_activation,\n            )\n        )\n\n    # Categorical\n    if self.cat_embed_input is not None:\n        self.cat_embed = BayesianDiffSizeCatEmbeddings(\n            column_idx=self.column_idx,\n            embed_input=self.cat_embed_input,\n            prior_sigma_1=self.prior_sigma_1,\n            prior_sigma_2=self.prior_sigma_2,\n            prior_pi=self.prior_pi,\n            posterior_mu_init=self.posterior_mu_init,\n            posterior_rho_init=self.posterior_rho_init,\n            activation_fn=self.cat_embed_activation,\n        )\n        self.cat_out_dim = int(np.sum([embed[2] for embed in self.cat_embed_input]))\n    else:\n        self.cat_out_dim = 0\n\n    # Continuous\n    if self.continuous_cols is not None:\n        self.cont_idx = [self.column_idx[col] for col in self.continuous_cols]\n        if cont_norm_layer == \"layernorm\":\n            self.cont_norm: NormLayers = nn.LayerNorm(len(self.continuous_cols))\n        elif cont_norm_layer == \"batchnorm\":\n            self.cont_norm = nn.BatchNorm1d(len(self.continuous_cols))\n        else:\n            self.cont_norm = nn.Identity()\n        if self.embed_continuous:\n            assert self.cont_embed_dim is not None, (\n                \"If 'embed_continuous' is True, 'cont_embed_dim' must be \"\n                \"provided\"\n            )\n            self.cont_embed = BayesianContEmbeddings(\n                n_cont_cols=len(self.continuous_cols),\n                embed_dim=self.cont_embed_dim,\n                prior_sigma_1=self.prior_sigma_1,\n                prior_sigma_2=self.prior_sigma_2,\n                prior_pi=self.prior_pi,\n                posterior_mu_init=self.posterior_mu_init,\n                posterior_rho_init=self.posterior_rho_init,\n                use_bias=(\n                    False if self.use_cont_bias is None else self.use_cont_bias\n                ),\n                activation_fn=self.cont_embed_activation,\n            )\n            self.cont_out_dim = len(self.continuous_cols) * self.cont_embed_dim\n        else:\n            self.cont_out_dim = len(self.continuous_cols)\n    else:\n        self.cont_out_dim = 0\n\n    self.output_dim = self.cat_out_dim + self.cont_out_dim\n\n    mlp_hidden_dims = [self.output_dim] + mlp_hidden_dims + [pred_dim]\n    self.bayesian_tab_mlp = BayesianMLP(\n        mlp_hidden_dims,\n        mlp_activation,\n        True,  # use_bias\n        prior_sigma_1,\n        prior_sigma_2,\n        prior_pi,\n        posterior_mu_init,\n        posterior_rho_init,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html","title":"Training Deep Learning Probabilistic Models","text":""},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer","title":"BayesianTrainer","text":"<pre><code>BayesianTrainer(\n    model,\n    objective,\n    custom_loss_function=None,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    metrics=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseBayesianTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Both the Bayesian models and the Trainer in this repo are based on the paper: Weight Uncertainty in Neural Networks.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>BaseBayesianModel</code>)           \u2013            <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code> section here in the docs.</p> </li> <li> <code>objective</code>               (<code>str</code>)           \u2013            <p>Defines the objective, loss or cost function. Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code> Possible values are: 'binary', 'multiclass', 'regression'</p> </li> <li> <code>custom_loss_function</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>If none of the loss functions available suits the user, it is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples folder in the repo.</p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>metrics</code>               (<code>Optional[Union[List[Metric], List[Metric]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as   long as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the Examples folder in the repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library Examples   classification-metrics&gt;<code>_. It can also be a torchmetric custom metric as   long as it is an object of type</code>Metric<code>.   See</code>the instructions</li> </ul> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cyclic_lr</code>               (<code>bool</code>)           \u2013            <p>Attribute that indicates if  the lr_scheduler is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See <code>Pytorch schedulers &lt;https://pytorch.org/docs/stable/optim.html&gt;</code>_.</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>@alias(  # noqa: C901\n    \"objective\",\n    [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n)\ndef __init__(\n    self,\n    model: BaseBayesianModel,\n    objective: str,\n    custom_loss_function: Optional[Module] = None,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        objective=objective,\n        custom_loss_function=custom_loss_function,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        metrics=metrics,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.fit","title":"fit","text":"<pre><code>fit(\n    X_tab,\n    target,\n    X_tab_val=None,\n    target_val=None,\n    val_split=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    n_train_samples=2,\n    n_val_samples=2,\n)\n</code></pre> <p>Fit method.</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>target</code>               (<code>ndarray</code>)           \u2013            <p>target values</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data</p> </li> <li> <code>target_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation target values</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> <li> <code>n_train_samples</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of samples to average over during the training process. See Weight Uncertainty in Neural Networks for details.</p> </li> <li> <code>n_val_samples</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of samples to average over during the validation process. See Weight Uncertainty in Neural Networks for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def fit(  # noqa: C901\n    self,\n    X_tab: np.ndarray,\n    target: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    target_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    n_train_samples: int = 2,\n    n_val_samples: int = 2,\n):\n    r\"\"\"Fit method.\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    target: np.ndarray\n        target values\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    target_val: np.ndarray, Optional, default = None\n        validation target values\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    n_train_samples: int, default=2\n        number of samples to average over during the training process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    n_val_samples: int, default=2\n        number of samples to average over during the validation process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = tabular_train_val_split(\n        self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n    )\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (X, y) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(\n                    X, y, n_train_samples, train_steps, batch_idx\n                )\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (X, y) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(\n                        X, y, n_val_samples, train_steps, i\n                    )\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict","title":"predict","text":"<pre><code>predict(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predictions</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>n_samples</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>number of samples that will be either returned or averaged to produce an overal prediction</p> </li> <li> <code>return_samples</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the n samples will be averaged or directly returned</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>256</code> )           \u2013            <p>batch size</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <p>array with the predictions</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n    axis = 2 if return_samples else 1\n\n    if self.objective == \"regression\":\n        return preds.squeeze(axis)\n    if self.objective == \"binary\":\n        return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n    if self.objective == \"multiclass\":\n        return np.argmax(preds, axis)\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predicted probabilities</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>n_samples</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>number of samples that will be either returned or averaged to produce an overal prediction</p> </li> <li> <code>return_samples</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the n samples will be averaged or directly returned</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>256</code> )           \u2013            <p>batch size</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>array with the probabilities per class</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict_proba(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n    if self.objective == \"binary\":\n        if return_samples:\n            preds = preds.squeeze(2)\n            probs = np.zeros([n_samples, preds.shape[1], 2])\n            for i in range(n_samples):\n                probs[i, :, 0] = 1 - preds[i]\n                probs[i, :, 1] = preds[i]\n        else:\n            preds = preds.squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n        return probs\n    if self.objective == \"multiclass\":\n        return preds\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"bayesian_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers or learning rate schedulers) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>Parameters:</p> <ul> <li> <code>path</code>               (<code>str</code>)           \u2013            <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> </li> <li> <code>model_filename</code>               (<code>str</code>, default:                   <code>'bayesian_model.pt'</code> )           \u2013            <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"bayesian_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers or\n    learning rate schedulers) have to be defined externally and then\n    passed to the `Trainer`. As a result, the `Trainer` does not\n    generate any attribute or additional data products that need to be\n    saved other than the `model` object itself, which can be saved as\n    any other torch model (e.g. `torch.save(model, path)`).\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.model.state_dict(), model_path)\n    else:\n        torch.save(self.model, model_path)\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html","title":"Callbacks","text":"<p>Here are the 4 callbacks available to the user in <code>pytorch-widedepp</code>: <code>LRHistory</code>, <code>ModelCheckpoint</code>, <code>EarlyStopping</code> and <code>RayTuneReporter</code>.</p> <p> NOTE: other callbacks , like <code>History</code>, run always  by default. In particular, the <code>History</code> callback saves the metrics in the  <code>history</code> attribute of the <code>Trainer</code>.</p>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.LRHistory","title":"LRHistory","text":"<pre><code>LRHistory(n_epochs)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Saves the learning rates during training in the <code>lr_history</code> attribute of the <code>Trainer</code>.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>n_epochs</code>               (<code>int</code>)           \u2013            <p>number of training epochs</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(self, n_epochs: int):\n    super(LRHistory, self).__init__()\n    self.n_epochs = n_epochs\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.ModelCheckpoint","title":"ModelCheckpoint","text":"<pre><code>ModelCheckpoint(\n    filepath=None,\n    monitor=\"val_loss\",\n    min_delta=0.0,\n    verbose=0,\n    save_best_only=False,\n    mode=\"auto\",\n    period=1,\n    max_save=-1,\n)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Saves the model after every epoch.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>filepath</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Full path to save the output weights. It must contain only the root of the filenames. Epoch number and <code>.pt</code> extension (for pytorch) will be added. e.g. <code>filepath=\"path/to/output_weights/weights_out\"</code> And the saved files in that directory will be named: 'weights_out_1.pt', 'weights_out_2.pt', .... If set to <code>None</code> the class just report best metric and best_epoch.</p> </li> <li> <code>monitor</code>               (<code>str</code>, default:                   <code>'val_loss'</code> )           \u2013            <p>quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> </li> <li> <code>min_delta</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>verbosity mode</p> </li> <li> <code>save_best_only</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>the latest best model according to the quantity monitored will not be overwritten.</p> </li> <li> <code>mode</code>               (<code>str</code>, default:                   <code>'auto'</code> )           \u2013            <p>If <code>save_best_only=True</code>, the decision to overwrite the current save file is made based on either the maximization or the minimization of the monitored quantity. For 'acc', this should be 'max', for 'loss' this should be 'min', etc. In 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> </li> <li> <code>period</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Interval (number of epochs) between checkpoints.</p> </li> <li> <code>max_save</code>               (<code>int</code>, default:                   <code>-1</code> )           \u2013            <p>Maximum number of outputs to save. If -1 will save all outputs</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>best</code>               (<code>float</code>)           \u2013            <p>best metric</p> </li> <li> <code>best_epoch</code>               (<code>int</code>)           \u2013            <p>best epoch</p> </li> <li> <code>best_state_dict</code>               (<code>dict</code>)           \u2013            <p>best model state dictionary. To restore model to its best state use <code>Trainer.model.load_state_dict (model_checkpoint.best_state_dict)</code> where <code>model_checkpoint</code> is an instance of the class <code>ModelCheckpoint</code>. See the Examples folder in the repo or the Examples section in this documentation for details</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(\n    self,\n    filepath: Optional[str] = None,\n    monitor: str = \"val_loss\",\n    min_delta: float = 0.0,\n    verbose: int = 0,\n    save_best_only: bool = False,\n    mode: str = \"auto\",\n    period: int = 1,\n    max_save: int = -1,\n):\n    super(ModelCheckpoint, self).__init__()\n\n    self.filepath = filepath\n    self.monitor = monitor\n    self.min_delta = min_delta\n    self.verbose = verbose\n    self.save_best_only = save_best_only\n    self.mode = mode\n    self.period = period\n    self.max_save = max_save\n\n    self.epochs_since_last_save = 0\n\n    if self.filepath:\n        if len(self.filepath.split(\"/\")[:-1]) == 0:\n            raise ValueError(\n                \"'filepath' must be the full path to save the output weights,\"\n                \" including the root of the filenames. e.g. 'checkpoints/weights_out'\"\n            )\n\n        root_dir = (\"/\").join(self.filepath.split(\"/\")[:-1])\n        if not os.path.exists(root_dir):\n            os.makedirs(root_dir)\n\n    if self.max_save &gt; 0:\n        self.old_files: List[str] = []\n\n    if self.mode not in [\"auto\", \"min\", \"max\"]:\n        warnings.warn(\n            \"ModelCheckpoint mode %s is unknown, \"\n            \"fallback to auto mode.\" % (self.mode),\n            RuntimeWarning,\n        )\n        self.mode = \"auto\"\n    if self.mode == \"min\":\n        self.monitor_op = np.less\n        self.best = np.Inf\n    elif self.mode == \"max\":\n        self.monitor_op = np.greater  # type: ignore[assignment]\n        self.best = -np.Inf\n    else:\n        if _is_metric(self.monitor):\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            self.monitor_op = np.less\n            self.best = np.Inf\n\n    if self.monitor_op == np.greater:\n        self.min_delta *= 1\n    else:\n        self.min_delta *= -1\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.EarlyStopping","title":"EarlyStopping","text":"<pre><code>EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=0.0,\n    patience=10,\n    verbose=0,\n    mode=\"auto\",\n    baseline=None,\n    restore_best_weights=False,\n)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Stop training when a monitored quantity has stopped improving.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>monitor</code>               (<code>str</code>, default:                   <code>'val_loss'</code> )           \u2013            <p>Quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> </li> <li> <code>min_delta</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> </li> <li> <code>patience</code>               (<code>int</code>, default:                   <code>10</code> )           \u2013            <p>Number of epochs that produced the monitored quantity with no improvement after which training will be stopped.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>verbosity mode.</p> </li> <li> <code>mode</code>               (<code>str</code>, default:                   <code>'auto'</code> )           \u2013            <p>one of {'auto', 'min', 'max'}. In 'min' mode, training will stop when the quantity monitored has stopped decreasing; in 'max' mode it will stop when the quantity monitored has stopped increasing; in 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> </li> <li> <code>baseline</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Baseline value for the monitored quantity to reach. Training will stop if the model does not show improvement over the baseline.</p> </li> <li> <code>restore_best_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Whether to restore model weights from the epoch with the best value of the monitored quantity. If <code>False</code>, the model weights obtained at the last step of training are used.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>best</code>               (<code>float</code>)           \u2013            <p>best metric</p> </li> <li> <code>stopped_epoch</code>               (<code>int</code>)           \u2013            <p>epoch when the training stopped</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(\n    self,\n    monitor: str = \"val_loss\",\n    min_delta: float = 0.0,\n    patience: int = 10,\n    verbose: int = 0,\n    mode: str = \"auto\",\n    baseline: Optional[float] = None,\n    restore_best_weights: bool = False,\n):\n    super(EarlyStopping, self).__init__()\n\n    self.monitor = monitor\n    self.min_delta = min_delta\n    self.patience = patience\n    self.verbose = verbose\n    self.mode = mode\n    self.baseline = baseline\n    self.restore_best_weights = restore_best_weights\n\n    self.wait = 0\n    self.stopped_epoch = 0\n    self.state_dict = None\n\n    if self.mode not in [\"auto\", \"min\", \"max\"]:\n        warnings.warn(\n            \"EarlyStopping mode %s is unknown, \"\n            \"fallback to auto mode.\" % self.mode,\n            RuntimeWarning,\n        )\n        self.mode = \"auto\"\n\n    if self.mode == \"min\":\n        self.monitor_op = np.less\n    elif self.mode == \"max\":\n        self.monitor_op = np.greater  # type: ignore[assignment]\n    else:\n        if _is_metric(self.monitor):\n            self.monitor_op = np.greater  # type: ignore[assignment]\n        else:\n            self.monitor_op = np.less\n\n    if self.monitor_op == np.greater:\n        self.min_delta *= 1\n    else:\n        self.min_delta *= -1\n</code></pre>"},{"location":"pytorch-widedeep/dataloaders.html","title":"Dataloaders","text":"<p> NOTE: This module should contain custom dataloaders  that the user might want to implement. At the moment <code>pytorch-widedeep</code>  offers one custom dataloader, <code>DataLoaderImbalanced</code>.</p>"},{"location":"pytorch-widedeep/dataloaders.html#pytorch_widedeep.dataloaders.DataLoaderImbalanced","title":"DataLoaderImbalanced","text":"<pre><code>DataLoaderImbalanced(\n    dataset, batch_size, num_workers, **kwargs\n)\n</code></pre> <p>               Bases: <code>DataLoader</code></p> <p>Class to load and shuffle batches with adjusted weights for imbalanced datasets. If the classes do not begin from 0 remapping is necessary. See here.</p> <p>Parameters:</p> <ul> <li> <code>dataset</code>               (<code>WideDeepDataset</code>)           \u2013            <p>see <code>pytorch_widedeep.training._wd_dataset</code></p> </li> <li> <code>batch_size</code>               (<code>int</code>)           \u2013            <p>size of batch</p> </li> <li> <code>num_workers</code>               (<code>int</code>)           \u2013            <p>number of workers</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>This can include any parameter that can be passed to the 'standard' pytorch DataLoader and that is not already explicitely passed to the class. In addition, the dictionary can also include the extra parameter <code>oversample_mul</code> which will multiply the number of samples of the minority class to be sampled by the <code>WeightedRandomSampler</code>.</p> <p>In other words, the <code>num_samples</code> param in <code>WeightedRandomSampler</code> will be defined as:</p> \\[ minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul \\] </li> </ul> Source code in <code>pytorch_widedeep/dataloaders.py</code> <pre><code>def __init__(\n    self, dataset: WideDeepDataset, batch_size: int, num_workers: int, **kwargs\n):\n    assert dataset.Y is not None, (\n        \"The 'dataset' instance of WideDeepDataset must contain a \"\n        \"target array 'Y'\"\n    )\n\n    if \"oversample_mul\" in kwargs:\n        oversample_mul = kwargs[\"oversample_mul\"]\n        del kwargs[\"oversample_mul\"]\n    else:\n        oversample_mul = 1\n    weights, minor_cls_cnt, num_clss = get_class_weights(dataset)\n    num_samples = int(minor_cls_cnt * num_clss * oversample_mul)\n    samples_weight = list(np.array([weights[i] for i in dataset.Y]))\n    sampler = WeightedRandomSampler(samples_weight, num_samples, replacement=True)\n    super().__init__(\n        dataset, batch_size, num_workers=num_workers, sampler=sampler, **kwargs\n    )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html","title":"The <code>load_from_folder</code> module","text":"<p>The <code>load_from_folder</code> module contains the classes that are necessary to load data from disk and these are inspired by the <code>ImageFolder</code> class in the <code>torchvision</code> library. This module is designed with one specific case in mind. Such case is the following: given a multi-modal dataset with tabular data, images and text, the images do not fit in memory, and therefore, they have to be loaded from disk. However, as any other functionality in this library, there is some flexibility and some additional cases can also be addressed using this module.</p> <p>For this module to be used, the datasets must be prepared in a certain way:</p> <ol> <li> <p>the tabular data must contain a column with the images names as stored in disk, including the extension (<code>.jpg</code>, <code>.png</code>, etc...).</p> </li> <li> <p>Regarding to the text dataset, the tabular data can contain a column with the texts themselves or the names of the files containing the texts as stored in disk.</p> </li> </ol> <p>The tabular data might or might not fit in disk itself. If it does not, please see the <code>ChunkPreprocessor</code> utilities at the[<code>preprocessing</code>] (preprocessing.md) module and the examples folder in the repo, which illustrate such case. Finally note that only <code>csv</code> format is currently supported in that case(more formats coming soon).</p>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder","title":"TabFromFolder","text":"<pre><code>TabFromFolder(\n    fname,\n    directory=None,\n    target_col=None,\n    preprocessor=None,\n    text_col=None,\n    img_col=None,\n    ignore_target=False,\n    reference=None,\n    verbose=1,\n)\n</code></pre> <p>This class is used to load tabular data from disk. The current constrains are:</p> <ol> <li>The only file format supported right now is csv</li> <li>The csv file must contain headers</li> </ol> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>fname</code>               (<code>str</code>)           \u2013            <p>the name of the csv file</p> </li> <li> <code>directory</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the csv file is located. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the target column. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>preprocessor</code>               (<code>Optional[TabularPreprocessor]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>text_col</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>img_col</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>ignore_target</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>whether to ignore the target column. This is normally set to True when this class is used for a test dataset.</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a reference <code>TabFromFolder</code> object. If provided, the <code>TabFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>TabFromFolder</code> object for evaluation or test purposes</p> </li> <li> <code>verbose</code>               (<code>Optional[int]</code>, default:                   <code>1</code> )           \u2013            <p>verbosity. If 0, no output will be printed during the process.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def __init__(\n    self,\n    fname: str,\n    directory: Optional[str] = None,\n    target_col: Optional[str] = None,\n    preprocessor: Optional[TabularPreprocessor] = None,\n    text_col: Optional[Union[str, List[str]]] = None,\n    img_col: Optional[Union[str, List[str]]] = None,\n    ignore_target: bool = False,\n    reference: Optional[Any] = None,  # is Type[\"TabFromFolder\"],\n    verbose: Optional[int] = 1,\n):\n    self.fname = fname\n    self.ignore_target = ignore_target\n    self.verbose = verbose\n\n    if reference is not None:\n        (\n            self.directory,\n            self.target_col,\n            self.preprocessor,\n            self.text_col,\n            self.img_col,\n        ) = self._set_from_reference(reference, preprocessor)\n    else:\n        assert (\n            directory is not None\n            and (target_col is not None and not ignore_target)\n            and preprocessor is not None\n        ), (\n            \"if no reference is provided, 'directory', 'target_col' and 'preprocessor' \"\n            \"must be provided\"\n        )\n\n        self.directory = directory\n        self.target_col = target_col\n        self.preprocessor = preprocessor\n        self.text_col = text_col\n        self.img_col = img_col\n\n    assert (\n        self.preprocessor.is_fitted\n    ), \"The preprocessor must be fitted before passing it to this class\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.WideFromFolder","title":"WideFromFolder","text":"<pre><code>WideFromFolder(\n    fname,\n    directory=None,\n    target_col=None,\n    preprocessor=None,\n    text_col=None,\n    img_col=None,\n    ignore_target=False,\n    reference=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>TabFromFolder</code></p> <p>This class is mostly identical to <code>TabFromFolder</code> but exists because we want to separate the treatment of the wide and the deep tabular components</p> <p>Parameters:</p> <ul> <li> <code>fname</code>               (<code>str</code>)           \u2013            <p>the name of the csv file</p> </li> <li> <code>directory</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the csv file is located. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the target column. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>preprocessor</code>               (<code>Optional[TabularPreprocessor]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>text_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>WideFromFolder</code> reference object must be provided=</p> </li> <li> <code>img_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>ignore_target</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>whether to ignore the target column. This is normally used when this class is used for a test dataset.</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a reference <code>WideFromFolder</code> object. If provided, the <code>WideFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>WideFromFolder</code> object for evaluation or test purposes</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>verbosity. If 0, no output will be printed during the process.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def __init__(\n    self,\n    fname: str,\n    directory: Optional[str] = None,\n    target_col: Optional[str] = None,\n    preprocessor: Optional[TabularPreprocessor] = None,\n    text_col: Optional[str] = None,\n    img_col: Optional[str] = None,\n    ignore_target: bool = False,\n    reference: Optional[Any] = None,  # is Type[\"WideFromFolder\"],\n    verbose: int = 1,\n):\n    super(WideFromFolder, self).__init__(\n        fname=fname,\n        directory=directory,\n        target_col=target_col,\n        preprocessor=preprocessor,\n        text_col=text_col,\n        img_col=img_col,\n        reference=reference,\n        ignore_target=ignore_target,\n        verbose=verbose,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder","title":"TextFromFolder","text":"<pre><code>TextFromFolder(preprocessor)\n</code></pre> <p>This class is used to load the text dataset (i.e. the text files) from a folder, or to retrieve the text given a texts column specified within the preprocessor object.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>preprocessor</code>               (<code>Union[TextPreprocessor, ChunkTextPreprocessor, HFPreprocessor, ChunkHFPreprocessor, List[TextPreprocessor], List[ChunkTextPreprocessor], List[HFPreprocessor], List[ChunkHFPreprocessor]]</code>)           \u2013            <p>The preprocessor used to process the text. It must be fitted before using this class</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>def __init__(\n    self,\n    preprocessor: Union[\n        TextPreprocessor,\n        ChunkTextPreprocessor,\n        HFPreprocessor,\n        ChunkHFPreprocessor,\n        List[TextPreprocessor],\n        List[ChunkTextPreprocessor],\n        List[HFPreprocessor],\n        List[ChunkHFPreprocessor],\n    ],\n):\n    if isinstance(preprocessor, list):\n        for p in preprocessor:\n            assert (\n                p.is_fitted\n            ), \"All preprocessors must be fitted before using this class\"\n    else:\n        assert (\n            preprocessor.is_fitted\n        ), \"The preprocessor must be fitted before using this class\"\n\n    self.preprocessor = preprocessor\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder","title":"ImageFromFolder","text":"<pre><code>ImageFromFolder(\n    directory=None,\n    preprocessor=None,\n    loader=default_loader,\n    extensions=None,\n    transforms=None,\n)\n</code></pre> <p>This class is used to load the image dataset from disk. It is inspired by the <code>ImageFolder</code> class at the <code>torchvision</code> library. Here, we have simply adapted to work within the context of a Wide and Deep multi-modal model.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>directory</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the images are located. If None, a preprocessor must be provided.</p> </li> <li> <code>preprocessor</code>               (<code>Optional[Union[ImagePreprocessor, List[ImagePreprocessor]]]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>ImagePreprocessor</code> object.</p> </li> <li> <code>loader</code>               (<code>Callable[[str], Any]</code>, default:                   <code>default_loader</code> )           \u2013            <p>a function to load a sample given its path.</p> </li> <li> <code>extensions</code>               (<code>Optional[Tuple[str, ...]]</code>, default:                   <code>None</code> )           \u2013            <p>a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be used where IMG_EXTENSIONS =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"</p> </li> <li> <code>transforms</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a <code>torchvision.transforms</code> object. If None, this class will simply return an array representation of the PIL Image</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>def __init__(\n    self,\n    directory: Optional[Union[str, List[str]]] = None,\n    preprocessor: Optional[\n        Union[ImagePreprocessor, List[ImagePreprocessor]]\n    ] = None,\n    loader: Callable[[str], Any] = default_loader,\n    extensions: Optional[Tuple[str, ...]] = None,\n    transforms: Optional[Any] = None,\n) -&gt; None:\n    assert (\n        directory is not None or preprocessor is not None\n    ), \"Either a directory or an instance of ImagePreprocessor(s) must be provided\"\n\n    if directory is not None and preprocessor is not None:  # pragma: no cover\n        error_msg = (\n            \"If both 'directory' and 'preprocessor' are provided, the 'img_path' \"\n            \"attribute of the 'preprocessor' must be the same as the 'directory'\"\n        )\n        if isinstance(directory, list):\n            assert isinstance(preprocessor, list)\n            assert len(directory) == len(preprocessor)\n            for d, p in zip(directory, preprocessor):\n                assert d == p.img_path, error_msg\n        else:\n            assert isinstance(preprocessor, ImagePreprocessor)\n            assert directory == preprocessor.img_path, error_msg\n\n    if directory is not None:\n        self.directory = directory\n    else:\n        assert (\n            preprocessor is not None\n        ), \"Either a directory or an instance of ImagePreprocessor must be provided\"\n        if isinstance(preprocessor, list):\n            self.directory = [p.img_path for p in preprocessor]\n        else:\n            self.directory = preprocessor.img_path\n\n    self.preprocessor = preprocessor\n    self.loader = loader\n    self.extensions = extensions if extensions is not None else IMG_EXTENSIONS\n    self.transforms = transforms\n    if self.transforms:\n        self.transforms_names = [\n            tr.__class__.__name__ for tr in self.transforms.transforms\n        ]\n    else:\n        self.transforms_names = []\n\n        self.transpose = True\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.wd_dataset_from_folder.WideDeepDatasetFromFolder","title":"WideDeepDatasetFromFolder","text":"<pre><code>WideDeepDatasetFromFolder(\n    n_samples,\n    tab_from_folder=None,\n    wide_from_folder=None,\n    text_from_folder=None,\n    img_from_folder=None,\n    reference=None,\n)\n</code></pre> <p>               Bases: <code>Dataset</code></p> <p>This class is the Dataset counterpart of the <code>WideDeepDataset</code> class.</p> <p>Given a reference tabular dataset, with columns that indicate the path to the images and to the text files or the texts themselves, it will use the <code>[...]FromFolder</code> classes to load the data consistently from disk per batch.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>n_samples</code>               (<code>int</code>)           \u2013            <p>Number of samples in the dataset</p> </li> <li> <code>tab_from_folder</code>               (<code>Optional[TabFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>TabFromFolder</code> class</p> </li> <li> <code>wide_from_folder</code>               (<code>Optional[WideFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>WideFromFolder</code> class</p> </li> <li> <code>text_from_folder</code>               (<code>Optional[TextFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>TextFromFolder</code> class</p> </li> <li> <code>img_from_folder</code>               (<code>Optional[ImageFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>ImageFromFolder</code> class</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>If not None, the 'text_from_folder' and 'img_from_folder' objects will be retrieved from the reference class. This is useful when we want to use a <code>WideDeepDatasetFromFolder</code> class used for a train dataset as a reference for the validation and test datasets. In this case, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be the same for all three datasets, so there is no need to create a new instance for each dataset.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py</code> <pre><code>def __init__(\n    self,\n    n_samples: int,\n    tab_from_folder: Optional[TabFromFolder] = None,\n    wide_from_folder: Optional[WideFromFolder] = None,\n    text_from_folder: Optional[TextFromFolder] = None,\n    img_from_folder: Optional[ImageFromFolder] = None,\n    reference: Optional[Any] = None,  # is Type[\"WideDeepDatasetFromFolder\"],\n):\n    super(WideDeepDatasetFromFolder, self).__init__()\n\n    if tab_from_folder is None and wide_from_folder is None:\n        raise ValueError(\n            \"Either 'tab_from_folder' or 'wide_from_folder' must be not None\"\n        )\n\n    if reference is not None:\n        assert (\n            img_from_folder is None and text_from_folder is None\n        ), \"If reference is not None, 'img_from_folder' and 'text_from_folder' left as None\"\n        self.text_from_folder, self.img_from_folder = self._get_from_reference(\n            reference\n        )\n    else:\n        assert (\n            text_from_folder is not None and img_from_folder is not None\n        ), \"If reference is None, 'img_from_folder' and 'text_from_folder' must be not None\"\n        self.text_from_folder = text_from_folder\n        self.img_from_folder = img_from_folder\n\n    self.n_samples = n_samples\n    self.tab_from_folder = tab_from_folder\n    self.wide_from_folder = wide_from_folder\n</code></pre>"},{"location":"pytorch-widedeep/losses.html","title":"Losses","text":"<p><code>pytorch-widedeep</code> accepts a number of losses and objectives that can be passed to the <code>Trainer</code> class via the parameter <code>objective</code> (see <code>pytorch-widedeep.training.Trainer</code>). For most cases the loss function that <code>pytorch-widedeep</code> will use internally is already implemented in Pytorch.</p> <p>In addition, <code>pytorch-widedeep</code> implements a series of  \"custom\" loss functions. These are described below for completion since, as mentioned before, they are used internally by the <code>Trainer</code>. Of course, onen could always use them on their own and can be imported as:</p> <p><code>from pytorch_widedeep.losses import FocalLoss</code></p> <p> NOTE:  Losses in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss","title":"MSELoss","text":"<pre><code>MSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss","title":"MSLELoss","text":"<pre><code>MSLELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss","title":"RMSELoss","text":"<pre><code>RMSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Root mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss","title":"RMSLELoss","text":"<pre><code>RMSLELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Root mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss","title":"QuantileLoss","text":"<pre><code>QuantileLoss(\n    quantiles=[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Quantile loss defined as:</p> \\[ Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y)) \\] <p>All credits go to the implementation at pytorch-forecasting.</p> <p>Parameters:</p> <ul> <li> <code>quantiles</code>               (<code>List[float]</code>, default:                   <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code> )           \u2013            <p>List of quantiles</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    quantiles: List[float] = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98],\n):\n    super().__init__()\n    self.quantiles = quantiles\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # REGRESSION\n&gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n&gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n&gt;&gt;&gt; loss = qloss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # REGRESSION\n    &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n    &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n    &gt;&gt;&gt; loss = qloss(input, target)\n    \"\"\"\n\n    assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n        \"The input and target have inconsistent shape. The dimension of the prediction \"\n        \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n        f\"i.e. {len(self.quantiles)}.\"\n    )\n    target = target.view(-1, 1).float()\n    losses = []\n    for i, q in enumerate(self.quantiles):\n        errors = target - input[..., i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n    loss = torch.cat(losses, dim=2)\n\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss","title":"FocalLoss","text":"<pre><code>FocalLoss(alpha=0.25, gamma=1.0)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Implementation of the Focal loss for both binary and multiclass classification:</p> \\[ FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t) \\] <p>where, for a case of a binary classification problem</p> \\[ \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation} \\] <p>Parameters:</p> <ul> <li> <code>alpha</code>               (<code>float</code>, default:                   <code>0.25</code> )           \u2013            <p>Focal Loss <code>alpha</code> parameter</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, alpha: float = 0.25, gamma: float = 1.0):\n    super().__init__()\n    self.alpha = alpha\n    self.gamma = gamma\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # BINARY\n&gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # MULTICLASS\n&gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # BINARY\n    &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # MULTICLASS\n    &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    \"\"\"\n    input_prob = torch.sigmoid(input)\n    if input.size(1) == 1:\n        input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n        num_class = 2\n    else:\n        num_class = input_prob.size(1)\n    binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n    if use_cuda:\n        binary_target = binary_target.cuda()\n    binary_target = binary_target.contiguous()\n    weight = self._get_weight(input_prob, binary_target)\n\n    return F.binary_cross_entropy(\n        input_prob, binary_target, weight, reduction=\"mean\"\n    )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss","title":"BayesianSELoss","text":"<pre><code>BayesianSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Squared Loss (log Gaussian) for the case of a regression as specified in the original publication Weight Uncertainty in Neural Networks.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n    \"\"\"\n    return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss","title":"TweedieLoss","text":"<pre><code>TweedieLoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Tweedie loss for extremely unbalanced zero-inflated data</p> <p>All credits go to Wenbo Shi. See this post and the original publication for details.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss.forward","title":"forward","text":"<pre><code>forward(input, target, p=1.5)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> <li> <code>p</code>               (<code>float</code>, default:                   <code>1.5</code> )           \u2013            <p>the power to be used to compute the loss. See the original publication for details</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = TweedieLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    p: float = 1.5,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n    p: float, default = 1.5\n        the power to be used to compute the loss. See the original\n        publication for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = TweedieLoss()(input, target)\n    \"\"\"\n\n    assert (\n        input.min() &gt; 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n    loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n        2 - p\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss","title":"ZILNLoss","text":"<pre><code>ZILNLoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Adjusted implementation of the Zero Inflated LogNormal Loss</p> <p>See A Deep Probabilistic Model for Customer Lifetime Value Prediction and the corresponding code.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions with spape (N,3), where N is the batch size</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual target values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n&gt;&gt;&gt; loss = ZILNLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions with spape (N,3), where N is the batch size\n    target: Tensor\n        Target tensor with the actual target values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n    &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n    \"\"\"\n    positive = target &gt; 0\n    positive = positive.float()\n\n    assert input.shape == torch.Size([target.shape[0], 3]), (\n        \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n        \"model that is using ZILNLoss must be equal to 3.\"\n    )\n\n    positive_input = input[..., :1]\n\n    classification_loss = F.binary_cross_entropy_with_logits(\n        positive_input, positive, reduction=\"none\"\n    ).flatten()\n\n    loc = input[..., 1:2]\n\n    # when using max the two input tensors (input and other) have to be of\n    # the same type\n    max_input = F.softplus(input[..., 2:])\n    max_other = torch.sqrt(torch.Tensor([torch.finfo(torch.double).eps])).type(\n        max_input.type()\n    )\n    scale = torch.max(max_input, max_other)\n    safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n    regression_loss = -torch.mean(\n        positive\n        * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n            safe_labels\n        ),\n        dim=-1,\n    )\n\n    return torch.mean(classification_loss + regression_loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss","title":"L1Loss","text":"<pre><code>L1Loss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>L1 loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss","title":"FocalR_L1Loss","text":"<pre><code>FocalR_L1Loss(beta=0.2, gamma=1.0, activation_fn='sigmoid')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R L1 loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss","title":"FocalR_MSELoss","text":"<pre><code>FocalR_MSELoss(\n    beta=0.2, gamma=1.0, activation_fn=\"sigmoid\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R MSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss","title":"FocalR_RMSELoss","text":"<pre><code>FocalR_RMSELoss(\n    beta=0.2, gamma=1.0, activation_fn=\"sigmoid\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R RMSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss","title":"HuberLoss","text":"<pre><code>HuberLoss(beta=0.2)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Hubbler Loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, beta: float = 0.2):\n    super().__init__()\n    self.beta = beta\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = HuberLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = HuberLoss()(input, target)\n    \"\"\"\n    l1_loss = torch.abs(input - target)\n    cond = l1_loss &lt; self.beta\n    loss = torch.where(\n        cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss","title":"InfoNCELoss","text":"<pre><code>InfoNCELoss(temperature=0.1, reduction='mean')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>InfoNCE Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Partially inspired by the code in this repo</p> <p>Parameters:</p> <ul> <li> <code>temperature</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>The logits are divided by the temperature before computing the loss value</p> </li> <li> <code>reduction</code>               (<code>str</code>, default:                   <code>'mean'</code> )           \u2013            <p>Loss reduction method</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, temperature: float = 0.1, reduction: str = \"mean\"):\n    super(InfoNCELoss, self).__init__()\n\n    self.temperature = temperature\n    self.reduction = reduction\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss.forward","title":"forward","text":"<pre><code>forward(g_projs)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>g_projs</code>               (<code>Tuple[Tensor, Tensor]</code>)           \u2013            <p>Tuple with the two tensors corresponding to the output of the two projection heads, as described 'SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training'.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n&gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n&gt;&gt;&gt; loss = InfoNCELoss()\n&gt;&gt;&gt; res = loss(g_projs)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    g_projs: Tuple\n        Tuple with the two tensors corresponding to the output of the two\n        projection heads, as described 'SAINT: Improved Neural Networks\n        for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n    &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n    &gt;&gt;&gt; loss = InfoNCELoss()\n    &gt;&gt;&gt; res = loss(g_projs)\n    \"\"\"\n    z, z_ = g_projs[0], g_projs[1]\n\n    norm_z = F.normalize(z, dim=-1).flatten(1)\n    norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n    logits = (norm_z @ norm_z_.t()) / self.temperature\n    logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n    # the target/labels are the entries on the diagonal\n    target = torch.arange(len(norm_z), device=norm_z.device)\n\n    loss = F.cross_entropy(logits, target, reduction=self.reduction)\n    loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n    return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss","title":"DenoisingLoss","text":"<pre><code>DenoisingLoss(\n    lambda_cat=1.0, lambda_cont=1.0, reduction=\"mean\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Denoising Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Parameters:</p> <ul> <li> <code>lambda_cat</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Multiplicative factor that will be applied to loss associated to the categorical features</p> </li> <li> <code>lambda_cont</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Multiplicative factor that will be applied to loss associated to the continuous features</p> </li> <li> <code>reduction</code>               (<code>str</code>, default:                   <code>'mean'</code> )           \u2013            <p>Loss reduction method</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self, lambda_cat: float = 1.0, lambda_cont: float = 1.0, reduction: str = \"mean\"\n):\n    super(DenoisingLoss, self).__init__()\n\n    self.lambda_cat = lambda_cat\n    self.lambda_cont = lambda_cont\n    self.reduction = reduction\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss.forward","title":"forward","text":"<pre><code>forward(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>x_cat_and_cat_</code>               (<code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code>)           \u2013            <p>Tuple of tensors containing the raw input features and their encodings, referred in the SAINT paper as \\(x\\) and \\(x''\\) respectively. If one denoising MLP is used per categorical feature <code>x_cat_and_cat_</code> will be a list of tuples, one per categorical feature</p> </li> <li> <code>x_cont_and_cont_</code>               (<code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code>)           \u2013            <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n&gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n&gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n&gt;&gt;&gt; loss = DenoisingLoss()\n&gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    x_cat_and_cat_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n    x_cont_and_cont_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_cat_and_cat_: tuple of Tensors or lists of tuples\n        Tuple of tensors containing the raw input features and their\n        encodings, referred in the SAINT paper as $x$ and $x''$\n        respectively. If one denoising MLP is used per categorical\n        feature `x_cat_and_cat_` will be a list of tuples, one per\n        categorical feature\n    x_cont_and_cont_: tuple of Tensors or lists of tuples\n        same as `x_cat_and_cat_` but for continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n    &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n    &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n    &gt;&gt;&gt; loss = DenoisingLoss()\n    &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n    \"\"\"\n\n    loss_cat = (\n        self._compute_cat_loss(x_cat_and_cat_)\n        if x_cat_and_cat_ is not None\n        else torch.tensor(0.0)\n    )\n    loss_cont = (\n        self._compute_cont_loss(x_cont_and_cont_)\n        if x_cont_and_cont_ is not None\n        else torch.tensor(0.0)\n    )\n\n    return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss","title":"EncoderDecoderLoss","text":"<pre><code>EncoderDecoderLoss(eps=1e-09)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>'Standard' Encoder Decoder Loss. Loss applied during the Endoder-Decoder  Self-Supervised Pre-Training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>The implementation of this lost is based on that at the tabnet repo, which is in itself an adaptation of that in the original paper TabNet: Attentive Interpretable Tabular Learning.</p> <p>Parameters:</p> <ul> <li> <code>eps</code>               (<code>float</code>, default:                   <code>1e-09</code> )           \u2013            <p>Simply a small number to avoid dividing by zero</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, eps: float = 1e-9):\n    super(EncoderDecoderLoss, self).__init__()\n    self.eps = eps\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss.forward","title":"forward","text":"<pre><code>forward(x_true, x_pred, mask)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>x_true</code>               (<code>Tensor</code>)           \u2013            <p>Embeddings of the input data</p> </li> <li> <code>x_pred</code>               (<code>Tensor</code>)           \u2013            <p>Reconstructed embeddings</p> </li> <li> <code>mask</code>               (<code>Tensor</code>)           \u2013            <p>Mask with 1s indicated that the reconstruction, and therefore the loss, is based on those features.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n&gt;&gt;&gt; x_true = torch.rand(3, 3)\n&gt;&gt;&gt; x_pred = torch.rand(3, 3)\n&gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n&gt;&gt;&gt; loss = EncoderDecoderLoss()\n&gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_true: Tensor\n        Embeddings of the input data\n    x_pred: Tensor\n        Reconstructed embeddings\n    mask: Tensor\n        Mask with 1s indicated that the reconstruction, and therefore the\n        loss, is based on those features.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n    &gt;&gt;&gt; x_true = torch.rand(3, 3)\n    &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n    &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n    &gt;&gt;&gt; loss = EncoderDecoderLoss()\n    &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n    \"\"\"\n\n    errors = x_pred - x_true\n\n    reconstruction_errors = torch.mul(errors, mask) ** 2\n\n    x_true_means = torch.mean(x_true, dim=0)\n    x_true_means[x_true_means == 0] = 1\n\n    x_true_stds = torch.std(x_true, dim=0) ** 2\n    x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n    features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n    nb_reconstructed_variables = torch.sum(mask, dim=1)\n    features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n    loss = torch.mean(features_loss_norm)\n\n    return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss","title":"MultiTargetRegressionLoss","text":"<pre><code>MultiTargetRegressionLoss(weights=None, reduction='mean')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch MSELoss. It allows for multi-target regression problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the MSELoss. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n&gt;&gt;&gt; input = torch.randn(3, 2)\n&gt;&gt;&gt; target = torch.randn(3, 2)\n&gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(\n    self,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n):\n    super(MultiTargetRegressionLoss, self).__init__()\n\n    self.weights = weights\n    self.reduction = reduction\n\n    if self.reduction not in [\"mean\", \"sum\"]:\n        raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss","title":"MultiTargetClassificationLoss","text":"<pre><code>MultiTargetClassificationLoss(\n    binary_config=None,\n    multiclass_config=None,\n    weights=None,\n    reduction=\"mean\",\n    binary_trick=False,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and cross_entropy losses. It allows for multi-target classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>binary_config</code>               (<code>Optional[List[Union[int, Tuple[int, float]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> </li> <li> <code>multiclass_config</code>               (<code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> </li> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss or the binary_cross_entropy_with_logits. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> </li> <li> <code>binary_trick</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 4)\n&gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output_1 = loss_1(input, target)\n&gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n... reduction=\"sum\", weights=[0.5, 0.5])\n&gt;&gt;&gt; output_2 = loss_2(input, target)\n&gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n&gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(  # noqa: C901\n    self,\n    binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n    multiclass_config: Optional[\n        List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n    ] = None,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    binary_trick: bool = False,\n):\n    super(MultiTargetClassificationLoss, self).__init__()\n\n    if reduction not in [\"mean\", \"sum\"]:\n        raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n    self.binary_config = binary_config\n    self.multiclass_config = multiclass_config\n    self.weights = weights\n    self.reduction = reduction\n    self.binary_trick = binary_trick\n\n    if self.weights is not None:\n        if len(self.weights) != (\n            len(self.binary_config) if self.binary_config is not None else 0\n        ) + (\n            len(self.multiclass_config) if self.multiclass_config is not None else 0\n        ):\n            raise ValueError(\n                \"The number of weights must match the number of binary and multiclass targets\"\n            )\n\n    if self.binary_trick:\n        self._check_inputs_with_binary_trick()\n        self._binary_config: List[int] = binary_config  # type: ignore[assignment]\n        self._multiclass_config: List[Tuple[int, int]] = self.multiclass_config  # type: ignore[assignment]\n    else:\n        self.binary_config_with_pos_weights = (\n            (self._set_binary_config_without_binary_trick())\n            if self.binary_config is not None\n            else None\n        )\n        self.multiclass_config_with_weights = (\n            (self._set_multiclass_config_without_binary_trick())\n            if self.multiclass_config is not None\n            else None\n        )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss","title":"MutilTargetRegressionAndClassificationLoss","text":"<pre><code>MutilTargetRegressionAndClassificationLoss(\n    regression_config=[],\n    binary_config=None,\n    multiclass_config=None,\n    weights=None,\n    reduction=\"mean\",\n    binary_trick=False,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the MultiTargetRegressionLoss and the MultiTargetClassificationLoss. It allows for multi-target regression and classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>regression_config</code>               (<code>List[int]</code>, default:                   <code>[]</code> )           \u2013            <p>List of integers with the indices of the regression targets</p> </li> <li> <code>binary_config</code>               (<code>Optional[List[Union[int, Tuple[int, float]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> </li> <li> <code>multiclass_config</code>               (<code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> </li> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss, the binary_cross_entropy_with_logits or the MSELoss. This reduction is applied after each target has been computed. Alias: 'target_reduction'</p> </li> <li> <code>binary_trick</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n... multiclass_config=[(2, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(  # noqa: C901\n    self,\n    regression_config: List[int] = [],\n    binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n    multiclass_config: Optional[\n        List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n    ] = None,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    binary_trick: bool = False,\n):\n\n    super(MutilTargetRegressionAndClassificationLoss, self).__init__()\n\n    self.regression_config = regression_config\n\n    assert binary_config is not None or multiclass_config is not None, (\n        \"Either binary_config or multiclass_config must be provided. \"\n        \"Otherwise, use the MultiTargetRegressionLoss\"\n    )\n\n    if binary_trick:\n        self._check_inputs_with_binary_trick(\n            regression_config, binary_config, multiclass_config\n        )\n\n    if weights is not None:\n        if len(weights) != (\n            len(regression_config)\n            + (len(binary_config) if binary_config is not None else 0)\n            + (len(multiclass_config) if multiclass_config is not None else 0)\n        ):\n            raise ValueError(\n                \"The number of weights must match the number of regression, binary and multiclass targets\"\n            )\n\n        self.weights_regression = self._prepare_weights_for_regression_targets(\n            weights, regression_config\n        )\n        self.weights_binary = self._prepare_weights_per_binary_targets(\n            weights, binary_config\n        )\n        self.weights_multiclass = self._prepare_weights_per_multiclass_targets(\n            weights, multiclass_config\n        )\n        self.weights = weights\n    else:\n        self.weights_regression = None\n        self.weights_binary = None\n        self.weights_multiclass = None\n\n    self.multi_target_regression_loss = MultiTargetRegressionLoss(\n        weights=self.weights_regression, reduction=reduction\n    )\n\n    self.multi_target_classification_loss = MultiTargetClassificationLoss(\n        binary_config=binary_config,\n        multiclass_config=multiclass_config,\n        weights=(\n            self.weights_binary + self.weights_multiclass\n            if self.weights_binary is not None\n            and self.weights_multiclass is not None\n            else (\n                self.weights_binary\n                if self.weights_binary is not None\n                else self.weights_multiclass\n            )\n        ),\n        reduction=reduction,\n        binary_trick=binary_trick,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html","title":"Metrics","text":"<p> NOTE: metrics in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems: \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p> <p>We have added the possibility of using the metrics available at the torchmetrics library. Note that this library is still in its early versions and therefore this option should be used with caution. To use <code>torchmetrics</code> simply import them and use them as any of the <code>pytorch-widedeep</code> metrics described below.</p> <pre><code>from torchmetrics import Accuracy, Precision\n\naccuracy = Accuracy(average=None, num_classes=2)\nprecision = Precision(average='micro', num_classes=2)\n\ntrainer = Trainer(model, objective=\"binary\", metrics=[accuracy, precision])\n</code></pre> <p>A functioning example for <code>pytorch-widedeep</code> using <code>torchmetrics</code> can be found in the Examples folder</p> <p> NOTE: the forward method for all metrics in this  module takes two tensors, <code>y_pred</code> and <code>y_true</code> (in that order). Therefore,  we do not include the method in the documentation.</p>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy","title":"Accuracy","text":"<pre><code>Accuracy(top_k=1)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the accuracy for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>top_k</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Accuracy will be computed using the top k most likely classes in multiclass problems</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy(top_k=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.66666667)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, top_k: int = 1):\n    super(Accuracy, self).__init__()\n\n    self.top_k = top_k\n    self.correct_count = 0\n    self.total_count = 0\n    self._name = \"acc\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.correct_count = 0\n    self.total_count = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision","title":"Precision","text":"<pre><code>Precision(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the precision for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate precision for each label, and finds their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(Precision, self).__init__()\n\n    self.average = average\n    self.true_positives = 0\n    self.all_positives = 0\n    self.eps = 1e-20\n    self._name = \"prec\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.all_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall","title":"Recall","text":"<pre><code>Recall(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the recall for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate recall for each label, and finds their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(Recall, self).__init__()\n\n    self.average = average\n    self.true_positives = 0\n    self.actual_positives = 0\n    self.eps = 1e-20\n    self._name = \"rec\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.actual_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore","title":"FBetaScore","text":"<pre><code>FBetaScore(beta, average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the fbeta score for both binary and categorical problems</p> \\[ F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)} \\] <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>int</code>)           \u2013            <p>Coefficient to control the balance between precision and recall</p> </li> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta for each label, and find their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, beta: int, average: bool = True):\n    super(FBetaScore, self).__init__()\n\n    self.beta = beta\n    self.average = average\n    self.precision = Precision(average=False)\n    self.recall = Recall(average=False)\n    self.eps = 1e-20\n    self._name = \"\".join([\"f\", str(self.beta)])\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets precision and recall</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets precision and recall\n    \"\"\"\n    self.precision.reset()\n    self.recall.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score","title":"F1Score","text":"<pre><code>F1Score(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the f1 score for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for each label, and find their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(F1Score, self).__init__()\n\n    self.average = average\n    self.f1 = FBetaScore(beta=1, average=self.average)\n    self._name = self.f1._name\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.f1.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score","title":"R2Score","text":"<pre><code>R2Score()\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Calculates R-Squared, the coefficient of determination:</p> \\[ R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2} \\] <p>where \\(\\hat{y_j}\\) is the ground truth, \\(y_j\\) is the predicted value and \\(\\bar{y}\\) is the mean of the ground truth.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; r2 = R2Score()\n&gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n&gt;&gt;&gt; r2(y_pred, y_true)\narray(0.94860814)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self):\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n\n    self._name = \"r2\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html","title":"The <code>models</code> module","text":"<p>This module contains the models that can be used as the four main components that will comprise a Wide and Deep model (<code>wide</code>, <code>deeptabular</code>, <code>deeptext</code>, <code>deepimage</code>), as well as the <code>WideDeep</code> \"constructor\" class. Note that each of the four components can be used independently. It also contains all the documentation for the models that can be used for self-supervised pre-training with tabular data.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide","title":"Wide","text":"<pre><code>Wide(input_dim, pred_dim=1)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Defines a <code>Wide</code> (linear) model where the non-linearities are captured via the so-called crossed-columns. This can be used as the <code>wide</code> component of a Wide &amp; Deep model.</p> <p>Parameters:</p> <ul> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>size of the Linear layer (implemented via an Embedding layer). <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide model. For example, if the wide model receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>size of the ouput tensor containing the predictions. Note that unlike all the other models, the wide model is connected directly to the output neuron(s) when used to build a Wide and Deep model. Therefore, it requires the <code>pred_dim</code> parameter.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_linear</code>               (<code>Module</code>)           \u2013            <p>the linear layer that comprises the wide branch of the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Wide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(4)\n&gt;&gt;&gt; wide = Wide(input_dim=X.unique().size(0), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>@alias(\"pred_dim\", [\"pred_size\", \"num_class\"])\ndef __init__(self, input_dim: int, pred_dim: int = 1):\n    super(Wide, self).__init__()\n\n    self.input_dim = input_dim\n    self.pred_dim = pred_dim\n\n    # Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n    self.wide_linear = nn.Embedding(input_dim + 1, pred_dim, padding_idx=0)\n    # (Sum(Embedding) + bias) is equivalent to (OneHotVector + Linear)\n    self.bias = nn.Parameter(torch.zeros(pred_dim))\n    self._reset_parameters()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide.forward","title":"forward","text":"<pre><code>forward(X)\n</code></pre> <p>Forward pass. Simply connecting the Embedding layer with the ouput neuron(s)</p> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>def forward(self, X: Tensor) -&gt; Tensor:\n    r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n    neuron(s)\"\"\"\n    out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n    return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp","title":"TabMlp","text":"<pre><code>TabMlp(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=[200, 100],\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabMlp</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of dense layers (i.e. a MLP).</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: List[int] = [200, 100],\n    mlp_activation: str = \"relu\",\n    mlp_dropout: Union[float, List[float]] = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabMlp, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n    # Mlp\n    mlp_input_dim = self.cat_out_dim + self.cont_out_dim\n    mlp_hidden_dims = [mlp_input_dim] + mlp_hidden_dims\n    self.encoder = MLP(\n        mlp_hidden_dims,\n        mlp_activation,\n        mlp_dropout,\n        mlp_batchnorm,\n        mlp_batchnorm_last,\n        mlp_linear_first,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder","title":"TabMlpDecoder","text":"<pre><code>TabMlpDecoder(\n    embed_dim,\n    mlp_hidden_dims=[100, 200],\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabMlp</code> model (which can be considered an encoder itself).</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). The <code>TabMlpDecoder</code> will receive the output from the MLP and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor that needs to be reconstructed.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[100, 200]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>mlp model that will receive the output of the encoder</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    mlp_hidden_dims: List[int] = [100, 200],\n    mlp_activation: str = \"relu\",\n    mlp_dropout: Union[float, List[float]] = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabMlpDecoder, self).__init__()\n\n    self.embed_dim = embed_dim\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.decoder = MLP(\n        mlp_hidden_dims + [self.embed_dim],\n        mlp_activation,\n        mlp_dropout,\n        mlp_batchnorm,\n        mlp_batchnorm_last,\n        mlp_linear_first,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet","title":"TabResnet","text":"<pre><code>TabResnet(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    blocks_dims=[200, 100, 100],\n    blocks_dropout=0.1,\n    simplify_blocks=False,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabResnet</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of Resnet blocks. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>blocks_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100, 100]</code> )           \u2013            <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> </li> <li> <code>blocks_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Block's internal dropout.</p> </li> <li> <code>simplify_blocks</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>deep dense Resnet model that will receive the concatenation of the embeddings and the continuous columns</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp model that will receive the results of the concatenation of the embeddings and the continuous columns -- if present --.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n&gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_deep)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    blocks_dims: List[int] = [200, 100, 100],\n    blocks_dropout: float = 0.1,\n    simplify_blocks: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabResnet, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    if len(blocks_dims) &lt; 2:\n        raise ValueError(\n            \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n        )\n\n    self.blocks_dims = blocks_dims\n    self.blocks_dropout = blocks_dropout\n    self.simplify_blocks = simplify_blocks\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n\n    # Resnet\n    dense_resnet_input_dim = self.cat_out_dim + self.cont_out_dim\n    self.encoder = DenseResnet(\n        dense_resnet_input_dim, blocks_dims, blocks_dropout, self.simplify_blocks\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.blocks_dims[-1]] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                True if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder","title":"TabResnetDecoder","text":"<pre><code>TabResnetDecoder(\n    embed_dim,\n    blocks_dims=[100, 100, 200],\n    blocks_dropout=0.1,\n    simplify_blocks=False,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabResnet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the ResNet blocks or the MLP(if present) and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor to be reconstructed.</p> </li> <li> <code>blocks_dims</code>               (<code>List[int]</code>, default:                   <code>[100, 100, 200]</code> )           \u2013            <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> </li> <li> <code>blocks_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Block's internal dropout.</p> </li> <li> <code>simplify_blocks</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>deep dense Resnet model that will receive the output of the encoder IF <code>mlp_hidden_dims</code> is None</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist in an MLP that will receive the output of the encoder followed by the deep dense Resnet.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    blocks_dims: List[int] = [100, 100, 200],\n    blocks_dropout: float = 0.1,\n    simplify_blocks: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabResnetDecoder, self).__init__()\n\n    if len(blocks_dims) &lt; 2:\n        raise ValueError(\n            \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n        )\n\n    self.embed_dim = embed_dim\n\n    self.blocks_dims = blocks_dims\n    self.blocks_dropout = blocks_dropout\n    self.simplify_blocks = simplify_blocks\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                True if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n        self.decoder = DenseResnet(\n            self.mlp_hidden_dims[-1],\n            blocks_dims,\n            blocks_dropout,\n            self.simplify_blocks,\n        )\n    else:\n        self.mlp = None\n        self.decoder = DenseResnet(\n            blocks_dims[0], blocks_dims, blocks_dropout, self.simplify_blocks\n        )\n\n    self.reconstruction_layer = nn.Linear(blocks_dims[-1], embed_dim, bias=False)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet","title":"TabNet","text":"<pre><code>TabNet(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    n_steps=3,\n    step_dim=8,\n    attn_dim=8,\n    dropout=0.0,\n    n_glu_step_dependent=2,\n    n_glu_shared=2,\n    ghost_bn=True,\n    virtual_batch_size=128,\n    momentum=0.02,\n    gamma=1.3,\n    epsilon=1e-15,\n    mask_type=\"sparsemax\"\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a TabNet model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>The implementation in this library is fully based on that here by the dreamquark-ai team, simply adapted so that it can work within the <code>WideDeep</code> frame. Therefore, ALL CREDIT TO THE DREAMQUARK-AI TEAM.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>n_steps</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> </li> <li> <code>step_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> </li> <li> <code>attn_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Attention dimension</p> </li> <li> <code>dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>GLU block's internal dropout</p> </li> <li> <code>n_glu_step_dependent</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> </li> <li> <code>n_glu_shared</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> </li> <li> <code>ghost_bn</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if Ghost Batch Normalization will be used.</p> </li> <li> <code>virtual_batch_size</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Batch size when using Ghost Batch Normalization</p> </li> <li> <code>momentum</code>               (<code>float</code>, default:                   <code>0.02</code> )           \u2013            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.3</code> )           \u2013            <p>Relaxation parameter in the paper. When gamma = 1, a feature is enforced to be used only at one decision step. As gamma increases, more flexibility is provided to use a feature at multiple decision steps</p> </li> <li> <code>epsilon</code>               (<code>float</code>, default:                   <code>1e-15</code> )           \u2013            <p>Float to avoid log(0). Always keep low</p> </li> <li> <code>mask_type</code>               (<code>str</code>, default:                   <code>'sparsemax'</code> )           \u2013            <p>Mask function to use. Either 'sparsemax' or 'entmax'</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>the TabNet encoder. For details see the original publication.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    n_steps: int = 3,\n    step_dim: int = 8,\n    attn_dim: int = 8,\n    dropout: float = 0.0,\n    n_glu_step_dependent: int = 2,\n    n_glu_shared: int = 2,\n    ghost_bn: bool = True,\n    virtual_batch_size: int = 128,\n    momentum: float = 0.02,\n    gamma: float = 1.3,\n    epsilon: float = 1e-15,\n    mask_type: str = \"sparsemax\",\n):\n    super(TabNet, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_steps = n_steps\n    self.step_dim = step_dim\n    self.attn_dim = attn_dim\n    self.dropout = dropout\n    self.n_glu_step_dependent = n_glu_step_dependent\n    self.n_glu_shared = n_glu_shared\n    self.ghost_bn = ghost_bn\n    self.virtual_batch_size = virtual_batch_size\n    self.momentum = momentum\n    self.gamma = gamma\n    self.epsilon = epsilon\n    self.mask_type = mask_type\n\n    # Embeddings are instantiated at the base model\n    self.embed_out_dim = self.cat_out_dim + self.cont_out_dim\n\n    # TabNet\n    self.encoder = TabNetEncoder(\n        self.embed_out_dim,\n        n_steps,\n        step_dim,\n        attn_dim,\n        dropout,\n        n_glu_step_dependent,\n        n_glu_shared,\n        ghost_bn,\n        virtual_batch_size,\n        momentum,\n        gamma,\n        epsilon,\n        mask_type,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder","title":"TabNetDecoder","text":"<pre><code>TabNetDecoder(\n    embed_dim,\n    n_steps=3,\n    step_dim=8,\n    dropout=0.0,\n    n_glu_step_dependent=2,\n    n_glu_shared=2,\n    ghost_bn=True,\n    virtual_batch_size=128,\n    momentum=0.02,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabNet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the <code>TabNet</code> encoder (i.e. the output from the so called 'steps') and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor to be reconstructed.</p> </li> <li> <code>n_steps</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> </li> <li> <code>step_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> </li> <li> <code>dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>GLU block's internal dropout</p> </li> <li> <code>n_glu_step_dependent</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> </li> <li> <code>n_glu_shared</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> </li> <li> <code>ghost_bn</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if Ghost Batch Normalization will be used.</p> </li> <li> <code>virtual_batch_size</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Batch size when using Ghost Batch Normalization</p> </li> <li> <code>momentum</code>               (<code>float</code>, default:                   <code>0.02</code> )           \u2013            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>decoder that will receive the output from the encoder's steps and will reconstruct the embeddings</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n&gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n&gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    n_steps: int = 3,\n    step_dim: int = 8,\n    dropout: float = 0.0,\n    n_glu_step_dependent: int = 2,\n    n_glu_shared: int = 2,\n    ghost_bn: bool = True,\n    virtual_batch_size: int = 128,\n    momentum: float = 0.02,\n):\n    super(TabNetDecoder, self).__init__()\n\n    self.n_steps = n_steps\n    self.step_dim = step_dim\n    self.dropout = dropout\n    self.n_glu_step_dependent = n_glu_step_dependent\n    self.n_glu_shared = n_glu_shared\n    self.ghost_bn = ghost_bn\n    self.virtual_batch_size = virtual_batch_size\n    self.momentum = momentum\n\n    shared_layers = nn.ModuleList()\n    for i in range(n_glu_shared):\n        if i == 0:\n            shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n        else:\n            shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n\n    self.decoder = nn.ModuleList()\n    for step in range(n_steps):\n        transformer = FeatTransformer(\n            step_dim,\n            step_dim,\n            dropout,\n            shared_layers,\n            n_glu_step_dependent,\n            ghost_bn,\n            virtual_batch_size,\n            momentum=momentum,\n        )\n        self.decoder.append(transformer)\n\n    self.reconstruction_layer = nn.Linear(step_dim, embed_dim, bias=False)\n    initialize_non_glu(self.reconstruction_layer, step_dim, embed_dim)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP","title":"ContextAttentionMLP","text":"<pre><code>ContextAttentionMLP(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    attn_dropout=0.2,\n    with_addnorm=False,\n    attn_activation=\"leaky_relu\",\n    n_blocks=3\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>ContextAttentionMLP</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by a <code>ContextAttentionEncoder</code>. Such encoder is in part inspired by the attention mechanism described in Hierarchical Attention Networks for Document Classification. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout for each attention block</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if residual connections will be used in the attention blocks</p> </li> <li> <code>attn_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of attention encoders.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    attn_dropout: float = 0.2,\n    with_addnorm: bool = False,\n    attn_activation: str = \"leaky_relu\",\n    n_blocks: int = 3,\n):\n    super(ContextAttentionMLP, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.attn_dropout = attn_dropout\n    self.with_addnorm = with_addnorm\n    self.attn_activation = attn_activation\n    self.n_blocks = n_blocks\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    # Embeddings are instantiated at the base model\n    # Attention Blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"attention_block\" + str(i),\n            ContextAttentionEncoder(\n                input_dim,\n                attn_dropout,\n                with_addnorm,\n                attn_activation,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, F)\\), where \\(N\\) is the batch size and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP","title":"SelfAttentionMLP","text":"<pre><code>SelfAttentionMLP(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    attn_dropout=0.2,\n    n_heads=8,\n    use_bias=False,\n    with_addnorm=False,\n    attn_activation=\"leaky_relu\",\n    n_blocks=3\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>SelfAttentionMLP</code> model that can be used as the deeptabular component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by what we would refer as a simplified <code>SelfAttentionEncoder</code>. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details. The reason to use a simplified version of self attention is because we observed that the 'standard' attention mechanism used in the TabTransformer has a notable tendency to overfit.</p> <p>In more detail, this model only uses Q and K (and not V). If we think about it as in terms of text (and intuitively), the Softmax(QK^T) is the attention mechanism that tells us how much, at each position in the input sentence, each word is represented or 'expressed'. We refer to that as 'attention weights'. These attention weighst are normally multiplied by a Value matrix to further strength the focus on the words that each word should be attending to (again, intuitively).</p> <p>In this implementation we skip this last multiplication and instead we multiply the attention weights directly by the input tensor. This is a simplification that we expect is beneficial in terms of avoiding overfitting for tabular data.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout for each attention block</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per attention block.</p> </li> <li> <code>use_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K projection layers.</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if residual connections will be used in the attention blocks</p> </li> <li> <code>attn_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cat_and_cont_embed</code>               (<code>Module</code>)           \u2013            <p>This is the module that processes the categorical and continuous columns</p> </li> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of attention encoders.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    attn_dropout: float = 0.2,\n    n_heads: int = 8,\n    use_bias: bool = False,\n    with_addnorm: bool = False,\n    attn_activation: str = \"leaky_relu\",\n    n_blocks: int = 3,\n):\n    super(SelfAttentionMLP, self).__init__(\n        column_idx=column_idx,\n        input_dim=input_dim,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.attn_dropout = attn_dropout\n    self.n_heads = n_heads\n    self.use_bias = use_bias\n    self.with_addnorm = with_addnorm\n    self.attn_activation = attn_activation\n    self.n_blocks = n_blocks\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    # Embeddings are instantiated at the base model\n    # Attention Blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"attention_block\" + str(i),\n            SelfAttentionEncoder(\n                input_dim,\n                attn_dropout,\n                use_bias,\n                n_heads,\n                with_addnorm,\n                attn_activation,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the WideDeep class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer","title":"TabTransformer","text":"<pre><code>TabTransformer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_heads=8,\n    use_qkv_bias=False,\n    n_blocks=4,\n    attn_dropout=0.2,\n    ff_dropout=0.1,\n    ff_factor=4,\n    transformer_activation=\"gelu\",\n    use_linear_attention=False,\n    use_flash_attention=False,\n    mlp_hidden_dims=None,\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines our adptation of the TabTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an enhanced adaptation of the model described in the paper. It can be considered as the flagship of our transformer family of models for tabular data and offers mutiple, additional features relative to the original publication(and some other models in the library)</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of Transformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>use_linear_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if Linear Attention (from Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention) will be used. The inclusing of this mode of attention is inspired by this post, where the Uber team finds that this attention mechanism leads to the best results for their tabular data.</p> </li> <li> <code>use_flash_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if Flash Attention will be used. </p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of Transformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_heads: int = 8,\n    use_qkv_bias: bool = False,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.2,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    transformer_activation: str = \"gelu\",\n    use_linear_attention: bool = False,\n    use_flash_attention: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: str = \"relu\",\n    mlp_dropout: float = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabTransformer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        input_dim=input_dim,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_heads = n_heads\n    self.use_qkv_bias = use_qkv_bias\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.transformer_activation = transformer_activation\n    self.use_linear_attention = use_linear_attention\n    self.use_flash_attention = use_flash_attention\n    self.ff_factor = ff_factor\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    if self.n_cont and not self.n_cat and not self.embed_continuous:\n        raise ValueError(\n            \"If only continuous features are used 'embed_continuous' must be set to 'True'\"\n        )\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"transformer_block\" + str(i),\n            TransformerEncoder(\n                input_dim,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                transformer_activation,\n                use_linear_attention,\n                use_flash_attention,\n            ),\n        )\n\n    self.mlp_first_hidden_dim = self._mlp_first_hidden_dim()\n\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p> <p> NOTE: if flash attention or linear attention are used, no attention weights are saved during the training process and calling this property will throw a ValueError</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT","title":"SAINT","text":"<pre><code>SAINT(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    use_qkv_bias=False,\n    n_heads=8,\n    n_blocks=2,\n    attn_dropout=0.1,\n    ff_dropout=0.2,\n    ff_factor=4,\n    transformer_activation=\"gelu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a SAINT model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an slightly modified and enhanced  version of the model described in the paper,</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>Number of SAINT-Transformer blocks.</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention column and row layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of SAINT-Transformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    use_qkv_bias: bool = False,\n    n_heads: int = 8,\n    n_blocks: int = 2,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.2,\n    ff_factor: int = 4,\n    transformer_activation: str = \"gelu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(SAINT, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.use_qkv_bias = use_qkv_bias\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"saint_block\" + str(i),\n            SaintEncoder(\n                input_dim,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                transformer_activation,\n                self.n_feats,\n            ),\n        )\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and the second elements are the column and row attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>column attention: \\((N, H, F, F)\\)</p> </li> <li> <p>row attention: \\((1, H, N, N)\\)</p> </li> </ul> <p>where \\(N\\) is the batch size, \\(H\\) is the number of heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer","title":"FTTransformer","text":"<pre><code>FTTransformer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=64,\n    kv_compression_factor=0.5,\n    kv_sharing=False,\n    use_qkv_bias=False,\n    n_heads=8,\n    n_blocks=4,\n    attn_dropout=0.2,\n    ff_dropout=0.1,\n    ff_factor=1.33,\n    transformer_activation=\"reglu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a FTTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> </li> <li> <code>kv_compression_factor</code>               (<code>float</code>, default:                   <code>0.5</code> )           \u2013            <p>By default, the FTTransformer uses Linear Attention (See Linformer: Self-Attention with Linear Complexity ). The compression factor that will be used to reduce the input sequence length. If we denote the resulting sequence length as \\(k = int(kv_{compression \\space factor} \\times s)\\) where \\(s\\) is the input sequence length.</p> </li> <li> <code>kv_sharing</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the \\(E\\) and \\(F\\) projection matrices will share weights.  See Linformer: Self-Attention with Linear Complexity for details</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per FTTransformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of FTTransformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the Linear-Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>float</code>, default:                   <code>1.33</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4, but they use 4/3 in the paper.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'reglu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final FTTransformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of FTTransformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 64,\n    kv_compression_factor: float = 0.5,\n    kv_sharing: bool = False,\n    use_qkv_bias: bool = False,\n    n_heads: int = 8,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.2,\n    ff_dropout: float = 0.1,\n    ff_factor: float = 1.33,\n    transformer_activation: str = \"reglu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(FTTransformer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.kv_compression_factor = kv_compression_factor\n    self.kv_sharing = kv_sharing\n    self.use_qkv_bias = use_qkv_bias\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    is_first = True\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"fttransformer_block\" + str(i),\n            FTTransformerEncoder(\n                input_dim,\n                self.n_feats,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                kv_compression_factor,\n                kv_sharing,\n                transformer_activation,\n                is_first,\n            ),\n        )\n        is_first = False\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is: \\((N, H, F, k)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads, \\(F\\) is the number of features/columns and \\(k\\) is the reduced sequence length or dimension, i.e. \\(k = int(kv_{compression \\space factor} \\times s)\\)</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver","title":"TabPerceiver","text":"<pre><code>TabPerceiver(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_cross_attns=1,\n    n_cross_attn_heads=4,\n    n_latents=16,\n    latent_dim=128,\n    n_latent_heads=4,\n    n_latent_blocks=4,\n    n_perceiver_blocks=4,\n    share_weights=False,\n    attn_dropout=0.1,\n    ff_dropout=0.1,\n    ff_factor=4,\n    transformer_activation=\"geglu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a Perceiver  that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model  or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> </li> <li> <code>n_cross_attns</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Number of times each perceiver block will cross attend to the input data (i.e. number of cross attention components per perceiver block). This should normally be 1. However, in the paper they describe some architectures (normally computer vision-related problems) where the Perceiver attends multiple times to the input array. Therefore, maybe multiple cross attention to the input array is also useful in some cases for tabular data  .</p> </li> <li> <code>n_cross_attn_heads</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of attention heads for the cross attention component</p> </li> <li> <code>n_latents</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Number of latents. This is the \\(N\\) parameter in the paper. As indicated in the paper, this number should be significantly lower than \\(M\\) (the number of columns in the dataset). Setting \\(N\\) closer to \\(M\\) defies the main purpose of the Perceiver, which is to overcome the transformer quadratic bottleneck</p> </li> <li> <code>latent_dim</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Latent dimension.</p> </li> <li> <code>n_latent_heads</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of attention heads per Latent Transformer</p> </li> <li> <code>n_latent_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of transformer encoder blocks (normalised MHA + normalised FF) per Latent Transformer</p> </li> <li> <code>n_perceiver_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of Perceiver blocks defined as [Cross Attention + Latent Transformer]</p> </li> <li> <code>share_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the weights will be shared between Perceiver blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'geglu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>ModuleDict</code>)           \u2013            <p>ModuleDict with the Perceiver blocks</p> </li> <li> <code>latents</code>               (<code>Parameter</code>)           \u2013            <p>Latents that will be used for prediction</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n... n_perceiver_blocks=2)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_cross_attns: int = 1,\n    n_cross_attn_heads: int = 4,\n    n_latents: int = 16,\n    latent_dim: int = 128,\n    n_latent_heads: int = 4,\n    n_latent_blocks: int = 4,\n    n_perceiver_blocks: int = 4,\n    share_weights: bool = False,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    transformer_activation: str = \"geglu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabPerceiver, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_cross_attns = n_cross_attns\n    self.n_cross_attn_heads = n_cross_attn_heads\n    self.n_latents = n_latents\n    self.latent_dim = latent_dim\n    self.n_latent_heads = n_latent_heads\n    self.n_latent_blocks = n_latent_blocks\n    self.n_perceiver_blocks = n_perceiver_blocks\n    self.share_weights = share_weights\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.latents = nn.init.trunc_normal_(\n        nn.Parameter(torch.empty(n_latents, latent_dim))\n    )\n\n    self.encoder = nn.ModuleDict()\n    first_perceiver_block = self._build_perceiver_block()\n    self.encoder[\"perceiver_block0\"] = first_perceiver_block\n\n    if share_weights:\n        for n in range(1, n_perceiver_blocks):\n            self.encoder[\"perceiver_block\" + str(n)] = first_perceiver_block\n    else:\n        for n in range(1, n_perceiver_blocks):\n            self.encoder[\"perceiver_block\" + str(n)] = self._build_perceiver_block()\n\n    self.mlp_first_hidden_dim = self.latent_dim\n\n    # Mlp\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. If the weights are not shared between perceiver blocks each element of the list will be a list itself containing the Cross Attention and Latent Transformer attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>Cross Attention: \\((N, C, L, F)\\)</p> </li> <li> <p>Latent Attention: \\((N, T, L, L)\\)</p> </li> </ul> <p>WHere \\(N\\) is the batch size, \\(C\\) is the number of Cross Attention heads, \\(L\\) is the number of Latents, \\(F\\) is the number of features/columns in the dataset and \\(T\\) is the number of Latent Attention heads</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer","title":"TabFastFormer","text":"<pre><code>TabFastFormer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_heads=8,\n    use_bias=False,\n    n_blocks=4,\n    attn_dropout=0.1,\n    ff_dropout=0.2,\n    ff_factor=4,\n    share_qv_weights=False,\n    share_weights=False,\n    transformer_activation=\"relu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a FastFormer that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per FastFormer block</p> </li> <li> <code>use_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of FastFormer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Additive Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>share_qv_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Following the paper, this is a boolean indicating if the Value (\\(V\\)) and the Query (\\(Q\\)) transformation parameters will be shared.</p> </li> <li> <code>share_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>In addition to sharing the \\(V\\) and \\(Q\\) transformation parameters, the parameters across different Fastformer layers can also be shared. Please, see <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> for details</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>MLP hidden dimensions. If not provided no MLP on top of the final FTTransformer block will be used</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of FasFormer blocks.</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_heads: int = 8,\n    use_bias: bool = False,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.2,\n    ff_factor: int = 4,\n    share_qv_weights: bool = False,\n    share_weights: bool = False,\n    transformer_activation: str = \"relu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabFastFormer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=None,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_heads = n_heads\n    self.use_bias = use_bias\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.share_qv_weights = share_qv_weights\n    self.share_weights = share_weights\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    first_fastformer_block = FastFormerEncoder(\n        input_dim,\n        n_heads,\n        use_bias,\n        attn_dropout,\n        ff_dropout,\n        ff_factor,\n        share_qv_weights,\n        transformer_activation,\n    )\n    self.encoder.add_module(\"fastformer_block0\", first_fastformer_block)\n    for i in range(1, n_blocks):\n        if share_weights:\n            self.encoder.add_module(\n                \"fastformer_block\" + str(i), first_fastformer_block\n            )\n        else:\n            self.encoder.add_module(\n                \"fastformer_block\" + str(i),\n                FastFormerEncoder(\n                    input_dim,\n                    n_heads,\n                    use_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    share_qv_weights,\n                    transformer_activation,\n                ),\n            )\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and second elements are the \\(\\alpha\\) and \\(\\beta\\) attention weights in the paper.</p> <p>The shape of the attention weights is \\((N, H, F)\\) where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN","title":"BasicRNN","text":"<pre><code>BasicRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.0,\n    bidirectional=False,\n    use_hidden_state=True,\n    padding_idx=1,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Standard text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) that can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the stack of RNNs</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>n_layers</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of recurrent layers</p> </li> <li> <code>rnn_dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>Dropout for each RNN layer except the last layer</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>use_hidden_state</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    n_layers: int = 3,\n    rnn_dropout: float = 0.0,\n    bidirectional: bool = False,\n    use_hidden_state: bool = True,\n    padding_idx: int = 1,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(BasicRNN, self).__init__()\n\n    if embed_dim is None and embed_matrix is None:\n        raise ValueError(\n            \"If no 'embed_matrix' is passed, the embedding dimension must\"\n            \"be specified with 'embed_dim'\"\n        )\n\n    if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n        )\n\n    if (\n        embed_dim is not None\n        and embed_matrix is not None\n        and not embed_dim == embed_matrix.shape[1]\n    ):\n        warnings.warn(\n            \"the input embedding dimension {} and the dimension of the \"\n            \"pretrained embeddings {} do not match. The pretrained embeddings \"\n            \"dimension ({}) will be used\".format(\n                embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n            ),\n            UserWarning,\n        )\n\n    self.vocab_size = vocab_size\n    self.embed_trainable = embed_trainable\n    self.embed_dim = embed_dim\n\n    self.rnn_type = rnn_type\n    self.hidden_dim = hidden_dim\n    self.n_layers = n_layers\n    self.rnn_dropout = rnn_dropout\n    self.bidirectional = bidirectional\n    self.use_hidden_state = use_hidden_state\n    self.padding_idx = padding_idx\n\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    # Embeddings\n    if embed_matrix is not None:\n        self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n    else:\n        assert self.embed_dim is not None\n        self.word_embed = nn.Embedding(\n            self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n        )\n\n    # RNN\n    rnn_params = {\n        \"input_size\": self.embed_dim,\n        \"hidden_size\": hidden_dim,\n        \"num_layers\": n_layers,\n        \"bidirectional\": bidirectional,\n        \"dropout\": rnn_dropout,\n        \"batch_first\": True,\n    }\n    if self.rnn_type.lower() == \"lstm\":\n        self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n    elif self.rnn_type.lower() == \"gru\":\n        self.rnn = nn.GRU(**rnn_params)\n    else:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n        )\n\n    self.rnn_output_dim = hidden_dim * 2 if bidirectional else hidden_dim\n\n    # FC-Head (Mlp)\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    else:\n        # simple hack to add readability in the forward pass\n        self.rnn_mlp = nn.Identity()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN","title":"AttentiveRNN","text":"<pre><code>AttentiveRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.1,\n    bidirectional=False,\n    use_hidden_state=True,\n    padding_idx=1,\n    attn_concatenate=True,\n    attn_dropout=0.1,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BasicRNN</code></p> <p>Text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) plus an attention layer. This model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of attention layer</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>n_layers</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of recurrent layers</p> </li> <li> <code>rnn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout for each RNN layer except the last layer</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>use_hidden_state</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>attn_concatenate</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state.</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Internal dropout for the attention mechanism</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    n_layers: int = 3,\n    rnn_dropout: float = 0.1,\n    bidirectional: bool = False,\n    use_hidden_state: bool = True,\n    padding_idx: int = 1,\n    attn_concatenate: bool = True,\n    attn_dropout: float = 0.1,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(AttentiveRNN, self).__init__(\n        vocab_size=vocab_size,\n        embed_dim=embed_dim,\n        embed_matrix=embed_matrix,\n        embed_trainable=embed_trainable,\n        rnn_type=rnn_type,\n        hidden_dim=hidden_dim,\n        n_layers=n_layers,\n        rnn_dropout=rnn_dropout,\n        bidirectional=bidirectional,\n        use_hidden_state=use_hidden_state,\n        padding_idx=padding_idx,\n        head_hidden_dims=head_hidden_dims,\n        head_activation=head_activation,\n        head_dropout=head_dropout,\n        head_batchnorm=head_batchnorm,\n        head_batchnorm_last=head_batchnorm_last,\n        head_linear_first=head_linear_first,\n    )\n\n    # Embeddings and RNN defined in the BasicRNN inherited class\n\n    # Attention\n    self.attn_concatenate = attn_concatenate\n    self.attn_dropout = attn_dropout\n\n    if bidirectional and attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 4\n    elif bidirectional or attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 2\n    else:\n        self.rnn_output_dim = hidden_dim\n    self.attn = ContextAttention(\n        self.rnn_output_dim, attn_dropout, sum_along_seq=True\n    )\n\n    # FC-Head (Mlp)\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights</p> <p>The shape of the attention weights is \\((N, S)\\), where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN","title":"StackedAttentiveRNN","text":"<pre><code>StackedAttentiveRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    bidirectional=False,\n    padding_idx=1,\n    n_blocks=3,\n    attn_concatenate=False,\n    attn_dropout=0.1,\n    with_addnorm=False,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Text classifier/regressor comprised by a stack of blocks: <code>[RNN + Attention]</code>. This can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the attentiob blocks</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks. Each block is comprised by an RNN and a Context Attention Encoder</p> </li> <li> <code>attn_concatenate</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state or simply</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Internal dropout for the attention mechanism</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the output of each block will be added to the input and normalised</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    bidirectional: bool = False,\n    padding_idx: int = 1,\n    n_blocks: int = 3,\n    attn_concatenate: bool = False,\n    attn_dropout: float = 0.1,\n    with_addnorm: bool = False,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(StackedAttentiveRNN, self).__init__()\n\n    if (\n        embed_dim is not None\n        and embed_matrix is not None\n        and not embed_dim == embed_matrix.shape[1]\n    ):\n        warnings.warn(\n            \"the input embedding dimension {} and the dimension of the \"\n            \"pretrained embeddings {} do not match. The pretrained embeddings \"\n            \"dimension ({}) will be used\".format(\n                embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n            ),\n            UserWarning,\n        )\n\n    if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n        )\n\n    self.vocab_size = vocab_size\n    self.embed_trainable = embed_trainable\n    self.embed_dim = embed_dim\n\n    self.rnn_type = rnn_type\n    self.hidden_dim = hidden_dim\n    self.bidirectional = bidirectional\n    self.padding_idx = padding_idx\n\n    self.n_blocks = n_blocks\n    self.attn_concatenate = attn_concatenate\n    self.attn_dropout = attn_dropout\n    self.with_addnorm = with_addnorm\n\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    # Embeddings\n    self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n\n    # Linear Projection: if embed_dim is different that the input of the\n    # attention blocks we add a linear projection\n    if bidirectional and attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 4\n    elif bidirectional or attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 2\n    else:\n        self.rnn_output_dim = hidden_dim\n\n    if self.rnn_output_dim != self.embed_dim:\n        self.embed_proj: Union[nn.Linear, nn.Identity] = nn.Linear(\n            self.embed_dim, self.rnn_output_dim\n        )\n    else:\n        self.embed_proj = nn.Identity()\n\n    # RNN\n    rnn_params = {\n        \"input_size\": self.rnn_output_dim,\n        \"hidden_size\": hidden_dim,\n        \"bidirectional\": bidirectional,\n        \"batch_first\": True,\n    }\n    if self.rnn_type.lower() == \"lstm\":\n        self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n    elif self.rnn_type.lower() == \"gru\":\n        self.rnn = nn.GRU(**rnn_params)\n\n    # FC-Head (Mlp)\n    self.attention_blks = nn.ModuleList()\n    for i in range(n_blocks):\n        self.attention_blks.append(\n            ContextAttentionEncoder(\n                self.rnn,\n                self.rnn_output_dim,\n                attn_dropout,\n                attn_concatenate,\n                with_addnorm=with_addnorm if i != n_blocks - 1 else False,\n                sum_along_seq=i == n_blocks - 1,\n            )\n        )\n\n    # Mlp\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    else:\n        # simple hack to add readability in the forward pass\n        self.rnn_mlp = nn.Identity()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, S)\\) Where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.miscellaneous.basic_transformer.Transformer","title":"Transformer","text":"<pre><code>Transformer(\n    vocab_size,\n    seq_length,\n    input_dim,\n    n_heads,\n    n_blocks,\n    attn_dropout=0.1,\n    ff_dropout=0.1,\n    ff_factor=4,\n    activation=\"gelu\",\n    use_linear_attention=False,\n    use_flash_attention=False,\n    padding_idx=0,\n    with_cls_token=False,\n    *,\n    with_pos_encoding=True,\n    pos_encoding_dropout=0.1,\n    pos_encoder=None\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Basic Encoder-Only Transformer Model for text classification/regression. As all other models in the library this model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: This model is introduced in the context of recommendation systems and thought for sequences of any nature (e.g. items). It can, of course, still be used for text. However, at this stage, we have decided to not include the possibility of loading pretrained word vectors since we aim to integrate the library wit Huggingface in the (hopefully) near future</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the token embeddings</p> <p>Param aliases: <code>embed_dim</code>, <code>d_model</code>. </p> </li> <li> <code>seq_length</code>               (<code>int</code>)           \u2013            <p>Input sequence length</p> </li> <li> <code>n_heads</code>               (<code>int</code>)           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>n_blocks</code>               (<code>int</code>)           \u2013            <p>Number of Transformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences.</p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token is included in the tokenized sequences. If present, the final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. NOTE: if included in the tokenized sequences it must be inserted as the first token in the sequences.</p> </li> <li> <code>with_pos_encoding</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if positional encoding will be used</p> </li> <li> <code>pos_encoding_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Positional encoding dropout</p> </li> <li> <code>pos_encoder</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>This model uses by default a standard positional encoding approach. However, any custom positional encoder can also be used and pass to the Transformer model via the 'pos_encoder' parameter</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embedding</code>               (<code>Module</code>)           \u2013            <p>Standard token embedding layer</p> </li> <li> <code>pos_encoder</code>               (<code>Module</code>)           \u2013            <p>Positional Encoder</p> </li> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of Transformer blocks</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Transformer\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/miscellaneous/basic_transformer.py</code> <pre><code>@alias(\"input_dim\", [\"embed_dim\", \"d_model\"])\n@alias(\"seq_length\", [\"max_length\", \"maxlen\"])\ndef __init__(\n    self,\n    vocab_size: int,\n    seq_length: int,\n    input_dim: int,\n    n_heads: int,\n    n_blocks: int,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    activation: str = \"gelu\",\n    use_linear_attention: bool = False,\n    use_flash_attention: bool = False,\n    padding_idx: int = 0,\n    with_cls_token: bool = False,\n    *,  # from here on pos encoding args\n    with_pos_encoding: bool = True,\n    pos_encoding_dropout: float = 0.1,\n    pos_encoder: Optional[nn.Module] = None,\n):\n    super().__init__()\n\n    self.input_dim = input_dim\n    self.seq_length = seq_length\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.activation = activation\n    self.use_linear_attention = use_linear_attention\n    self.use_flash_attention = use_flash_attention\n    self.padding_idx = padding_idx\n    self.with_cls_token = with_cls_token\n    self.with_pos_encoding = with_pos_encoding\n    self.pos_encoding_dropout = pos_encoding_dropout\n\n    self.embedding = nn.Embedding(\n        vocab_size, input_dim, padding_idx=self.padding_idx\n    )\n\n    if with_pos_encoding:\n        if pos_encoder is not None:\n            self.pos_encoder: Union[nn.Module, nn.Identity, PositionalEncoding] = (\n                pos_encoder\n            )\n        else:\n            self.pos_encoder = PositionalEncoding(\n                input_dim, pos_encoding_dropout, seq_length\n            )\n    else:\n        self.pos_encoder = nn.Identity()\n\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"transformer_block\" + str(i),\n            TransformerEncoder(\n                input_dim,\n                n_heads,\n                False,  # use_qkv_bias\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                activation,\n                use_linear_attention,\n                use_flash_attention,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel","title":"HFModel","text":"<pre><code>HFModel(\n    model_name,\n    use_cls_token=True,\n    trainable_parameters=None,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n    verbose=False,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around the Hugging Face transformers library. It can be used as the text component of a Wide &amp; Deep model or independently by itself.</p> <p>At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>use_cls_token</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the [CLS] token or the mean of the sequence of hidden states as the sentence embedding</p> </li> <li> <code>trainable_parameters</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the model parameters that will be trained. If None, none of the parameters will be trainable</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the head</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> <li> <code>verbose</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, it will print information about the model</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional kwargs to be passed to the model</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>head</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the transformer. This will only exists if <code>head_layers_dim</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n&gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code> <pre><code>@alias(\"use_cls_token\", [\"use_special_token\"])\ndef __init__(\n    self,\n    model_name: str,\n    use_cls_token: bool = True,\n    trainable_parameters: Optional[List[str]] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n    verbose: bool = False,\n    **kwargs,\n):\n    super().__init__()\n\n    # TO DO: add warning regarging ELECTRA as ELECTRA does not have a cls\n    # token.  Research what happens with ELECTRA\n    self.model_name = model_name\n    self.use_cls_token = use_cls_token\n    self.trainable_parameters = trainable_parameters\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n    self.verbose = verbose\n    self.kwargs = kwargs\n\n    if self.verbose and self.use_cls_token:\n        warnings.warn(\n            \"The model will use the [CLS] token. Make sure the tokenizer \"\n            \"was run with add_special_tokens=True\",\n            UserWarning,\n        )\n\n    self.model_class = get_model_class(model_name)\n\n    self.config, self.model = get_config_and_model(self.model_name)\n\n    self.output_attention_weights = kwargs.get(\"output_attentions\", False)\n\n    if self.trainable_parameters is not None:\n        for n, p in self.model.named_parameters():\n            p.requires_grad = any([tl in n for tl in self.trainable_parameters])\n\n    # FC-Head (Mlp). Note that the FC head will always be trainable\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.config.hidden_size] + self.head_hidden_dims\n        self.head = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight","title":"attention_weight  <code>property</code>","text":"<pre><code>attention_weight\n</code></pre> <p>Returns the attention weights if the model was created with the output_attention_weights=True argument. If not, it will raise an AttributeError.</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the sequence length.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision","title":"Vision","text":"<pre><code>Vision(\n    pretrained_model_setup=None,\n    n_trainable=None,\n    trainable_params=None,\n    channel_sizes=[64, 128, 256, 512],\n    kernel_sizes=[7, 3, 3, 3],\n    strides=[2, 1, 1, 1],\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=0.1,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Defines a standard image classifier/regressor using a pretrained network or a sequence of convolution layers that can be used as the <code>deepimage</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: this class represents the integration  between <code>pytorch-widedeep</code> and <code>torchvision</code>. New architectures will be  available as they are added to <code>torchvision</code>. In a distant future we aim  to bring transformer-based architectures as well. However, simple  CNN-based architectures (and even MLP-based) seem to produce SoTA  results. For the time being, we describe below the options available  through this class</p> <p>Parameters:</p> <ul> <li> <code>pretrained_model_setup</code>               (<code>Union[str, Dict[str, Union[str, WeightsEnum]]]</code>, default:                   <code>None</code> )           \u2013            <p>Name of the pretrained model. Should be a variant of the following architectures: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3', 'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'. if <code>pretrained_model_setup = None</code> a basic, fully trainable CNN will be used. Alternatively, since Torchvision 0.13 one can use pretrained models with different weigths. Therefore, <code>pretrained_model_setup</code> can also be dictionary with the name of the model and the weights (e.g. <code>{'resnet50': ResNet50_Weights.DEFAULT}</code> or <code>{'resnet50': \"IMAGENET1K_V2\"}</code>).  Aliased as <code>pretrained_model_name</code>.</p> </li> <li> <code>n_trainable</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of trainable layers starting from the layer closer to the output neuron(s). Note that this number DOES NOT take into account the so-called 'head' which is ALWAYS trainable. If <code>trainable_params</code> is not None this parameter will be ignored</p> </li> <li> <code>trainable_params</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List of strings containing the names (or substring within the name) of the parameters that will be trained. For example, if we use a 'resnet18' pretrained model and we set <code>trainable_params = ['layer4']</code> only the parameters of 'layer4' of the network (and the head, as mentioned before) will be trained. Note that setting this or the previous parameter involves some knowledge of the architecture used.</p> </li> <li> <code>channel_sizes</code>               (<code>List[int]</code>, default:                   <code>[64, 128, 256, 512]</code> )           \u2013            <p>List of integers with the channel sizes of a CNN in case we choose not to use a pretrained model</p> </li> <li> <code>kernel_sizes</code>               (<code>Union[int, List[int]]</code>, default:                   <code>[7, 3, 3, 3]</code> )           \u2013            <p>List of integers with the kernel sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> </li> <li> <code>strides</code>               (<code>Union[int, List[int]]</code>, default:                   <code>[2, 1, 1, 1]</code> )           \u2013            <p>List of integers with the stride sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the head. e.g: [64,32]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float indicating the dropout between the dense layers.</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>features</code>               (<code>Module</code>)           \u2013            <p>The pretrained model or Standard CNN plus the optional head</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Vision\n&gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n&gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n&gt;&gt;&gt; out = model(X_img)\n</code></pre> Source code in <code>pytorch_widedeep/models/image/vision.py</code> <pre><code>@alias(\"pretrained_model_setup\", [\"pretrained_model_name\"])\ndef __init__(\n    self,\n    pretrained_model_setup: Union[str, Dict[str, Union[str, WeightsEnum]]] = None,\n    n_trainable: Optional[int] = None,\n    trainable_params: Optional[List[str]] = None,\n    channel_sizes: List[int] = [64, 128, 256, 512],\n    kernel_sizes: Union[int, List[int]] = [7, 3, 3, 3],\n    strides: Union[int, List[int]] = [2, 1, 1, 1],\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Union[float, List[float]] = 0.1,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(Vision, self).__init__()\n\n    self._check_pretrained_model_setup(\n        pretrained_model_setup, n_trainable, trainable_params\n    )\n\n    self.pretrained_model_setup = pretrained_model_setup\n    self.n_trainable = n_trainable\n    self.trainable_params = trainable_params\n    self.channel_sizes = channel_sizes\n    self.kernel_sizes = kernel_sizes\n    self.strides = strides\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    self.features, self.backbone_output_dim = self._get_features()\n\n    if pretrained_model_setup is not None:\n        self._freeze(self.features)\n\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.backbone_output_dim] + self.head_hidden_dims\n        self.vision_mlp = MLP(\n            head_hidden_dims,\n            self.head_activation,\n            self.head_dropout,\n            self.head_batchnorm,\n            self.head_batchnorm_last,\n            self.head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser","title":"ModelFuser","text":"<pre><code>ModelFuser(\n    models,\n    *,\n    fusion_method,\n    projection_method=None,\n    custom_head=None,\n    head_hidden_dims=None,\n    head_activation=None,\n    head_dropout=None,\n    head_batchnorm=None,\n    head_batchnorm_last=None,\n    head_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around a list of models that are associated to the different text and/or image columns (and datasets) The class is designed to 'fuse' the models using a variety of methods.</p> <p>Parameters:</p> <ul> <li> <code>models</code>               (<code>List[BaseWDModelComponent]</code>)           \u2013            <p>List of models whose outputs will be fused</p> </li> <li> <code>fusion_method</code>               (<code>Union[Literal[concatenate, mean, max, sum, mult, head], List[Literal[concatenate, mean, max, sum, mult, head]]]</code>)           \u2013            <p>Method to fuse the output of the models. It can be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'head'] or a list of those. If a list is provided the output of the models will be fused using all the methods in the list and the final output will be the concatenation of the outputs of each method</p> </li> <li> <code>projection_method</code>               (<code>Optional[Literal[min, max, mean]]</code>, default:                   <code>None</code> )           \u2013            <p>If the fusion_method is not 'concatenate', this parameter will determine how to project the output of the models to a common dimension. It can be one of ['min', 'max', 'mean']. Default is None</p> </li> <li> <code>custom_head</code>               (<code>Optional[Union[BaseWDModelComponent, Module]]</code>, default:                   <code>None</code> )           \u2013            <p>Custom head to be used to fuse the output of the models. If provided, this will take precedence over head_hidden_dims. Also, if provided, 'projection_method' will be ignored.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per layer in the custom head. If custom_head is provided, this parameter will be ignored</p> </li> <li> <code>head_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function to be used in the custom head. Default is None</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout to be used in the custom head. Default is None</p> </li> <li> <code>head_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Whether to use batchnorm in the custom head. Default is None</p> </li> <li> <code>head_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>head_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>head</code>               (<code>Module or BaseWDModelComponent</code>)           \u2013            <p>Custom head to be used to fuse the output of the models. If custom_head is provided, this will take precedence over head_hidden_dims</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n... 'text_col2': ['goodbye world', 'this is another test']})\n&gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n...     text_col=\"text_col1\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n...     text_col=\"text_col2\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n&gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n&gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n&gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n&gt;&gt;&gt; rnn1 = BasicRNN(\n...     vocab_size=len(text_preprocessor_1.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; rnn2 = BasicRNN(\n...     vocab_size=len(text_preprocessor_2.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n&gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n</code></pre> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>def __init__(\n    self,\n    models: List[BaseWDModelComponent],\n    *,\n    fusion_method: Union[\n        Literal[\n            \"concatenate\",\n            \"mean\",\n            \"max\",\n            \"sum\",\n            \"mult\",\n            \"head\",\n        ],\n        List[Literal[\"concatenate\", \"mean\", \"max\", \"sum\", \"mult\", \"head\"]],\n    ],\n    projection_method: Optional[Literal[\"min\", \"max\", \"mean\"]] = None,\n    custom_head: Optional[Union[BaseWDModelComponent, nn.Module]] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: Optional[str] = None,\n    head_dropout: Optional[float] = None,\n    head_batchnorm: Optional[bool] = None,\n    head_batchnorm_last: Optional[bool] = None,\n    head_linear_first: Optional[bool] = None,\n) -&gt; None:\n    super(ModelFuser, self).__init__()\n\n    self.models = nn.ModuleList(models)\n    self.fusion_method = fusion_method\n    self.projection_method = projection_method\n\n    self.all_output_dim_equal = all(\n        model.output_dim == self.models[0].output_dim for model in self.models\n    )\n\n    self.check_input_parameters()\n\n    if self.fusion_method == \"head\":\n        assert (\n            head_hidden_dims is not None or custom_head is not None\n        ), \"When using 'head' as fusion_method, either head_hidden_dims or custom_head must be provided\"\n        if custom_head is not None:\n            # custom_head takes precedence over head_hidden_dims (in case\n            # both are provided)\n            assert hasattr(\n                custom_head, \"output_dim\"\n            ), \"custom_head must have an 'output_dim' property\"\n            self.head: Union[BaseWDModelComponent, nn.Module] = custom_head\n        else:\n            assert head_hidden_dims is not None\n            self.head_hidden_dims = head_hidden_dims\n            self.head_activation = head_activation\n            self.head_dropout = head_dropout\n            self.head_batchnorm = head_batchnorm\n            self.head_batchnorm_last = head_batchnorm_last\n            self.head_linear_first = head_linear_first\n\n            self.head = MLP(\n                d_hidden=[sum([model.output_dim for model in self.models])]\n                + self.head_hidden_dims,\n                activation=(\n                    \"relu\" if self.head_activation is None else self.head_activation\n                ),\n                dropout=0.0 if self.head_dropout is None else self.head_dropout,\n                batchnorm=(\n                    False if self.head_batchnorm is None else self.head_batchnorm\n                ),\n                batchnorm_last=(\n                    False\n                    if self.head_batchnorm_last is None\n                    else self.head_batchnorm_last\n                ),\n                linear_first=(\n                    True\n                    if self.head_linear_first is None\n                    else self.head_linear_first\n                ),\n            )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>Returns the output dimension of the model.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.project","title":"project","text":"<pre><code>project(X)\n</code></pre> <p>Projects the output of the models to a common dimension.</p> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>def project(self, X: List[Tensor]) -&gt; List[Tensor]:\n    r\"\"\"Projects the output of the models to a common dimension.\"\"\"\n\n    if self.all_output_dim_equal and self.projection_method is None:\n        return X\n\n    output_dims = [model.output_dim for model in self.models]\n\n    if self.projection_method == \"min\":\n        proj_dim = min(output_dims)\n        idx = output_dims.index(proj_dim)\n    elif self.projection_method == \"max\":\n        proj_dim = max(output_dims)\n        idx = output_dims.index(proj_dim)\n    elif self.projection_method == \"mean\":\n        proj_dim = int(sum(output_dims) / len(output_dims))\n        idx = None\n    else:\n        raise ValueError(\"projection_method must be one of ['min', 'max', 'mean']\")\n\n    x_proj: List[Tensor] = []\n    for i, x in enumerate(X):\n        if i == idx:\n            x_proj.append(x)\n        else:\n            x_proj.append(\n                nn.Linear(output_dims[i], proj_dim, bias=False, device=x.device)(x)\n            )\n\n    return x_proj\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.wide_deep.WideDeep","title":"WideDeep","text":"<pre><code>WideDeep(\n    wide=None,\n    deeptabular=None,\n    deeptext=None,\n    deepimage=None,\n    deephead=None,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=0.1,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=True,\n    enforce_positive=False,\n    enforce_positive_activation=\"softplus\",\n    pred_dim=1,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code> <code>deeptext</code> and <code>deepimage</code> models.</p> <p>Note that all models described so far in this library must be passed to the <code>WideDeep</code> class once constructed. This is because the models output the last layer before the prediction layer. Such prediction layer is added by the <code>WideDeep</code> class as it collects the components for every data mode.</p> <p>There are two options to combine these models that correspond to the two main architectures that <code>pytorch-widedeep</code> can build.</p> <ul> <li> <p>Directly connecting the output of the model components to an ouput neuron(s).</p> </li> <li> <p>Adding a <code>Fully-Connected Head</code> (FC-Head) on top of the deep models.   This FC-Head will combine the output form the <code>deeptabular</code>, <code>deeptext</code> and   <code>deepimage</code> and will be then connected to the output neuron(s).</p> </li> </ul> <p>Parameters:</p> <ul> <li> <code>wide</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p><code>Wide</code> model. This is a linear model where the non-linearities are captured via crossed-columns.</p> </li> <li> <code>deeptabular</code>               (<code>Optional[BaseWDModelComponent]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library implements a number of possible architectures for the <code>deeptabular</code> component. See the documenation of the package.</p> </li> <li> <code>deeptext</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library implements a number of possible architectures for the <code>deeptext</code> component. See the documenation of the package. Note that <code>deeptext</code> can be a list of models. This is useful when using multiple text inputs.</p> </li> <li> <code>deepimage</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library uses <code>torchvision</code> and implements a number of possible architectures for the <code>deepimage</code> component. See the documenation of the package. Note that <code>deepimage</code> can be a list of models. This is useful when using multiple image inputs.</p> </li> <li> <code>deephead</code>               (<code>Optional[BaseWDModelComponent]</code>, default:                   <code>None</code> )           \u2013            <p>Alternatively, the user can pass a custom model that will receive the output of the deep component. If <code>deephead</code> is not None all the previous fc-head parameters will be ignored</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently <code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p> </li> <li> <code>head_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the <code>'rnn_mlp'</code></p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> <li> <code>enforce_positive</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the output from the final layer must be positive. This is important if you are using loss functions with non-negative input restrictions, e.g. RMSLE, or if you know your predictions are bounded in between 0 and inf</p> </li> <li> <code>enforce_positive_activation</code>               (<code>str</code>, default:                   <code>'softplus'</code> )           \u2013            <p>Activation function to enforce that the final layer has a positive output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Size of the final wide and deep output layer containing the predictions. <code>1</code> for regression and binary classification or number of classes for multiclass classification.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n</code></pre> <p> NOTE: It is possible to use custom components to  build Wide &amp; Deep models. Simply, build them and pass them as the  corresponding parameters. Note that the custom models MUST return a last  layer of activations(i.e. not the final prediction) so that  these  activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with  the size of these last layers of activations. See for example  <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p> Source code in <code>pytorch_widedeep/models/wide_deep.py</code> <pre><code>@alias(  # noqa: C901\n    \"pred_dim\",\n    [\"num_class\", \"pred_size\"],\n)\ndef __init__(\n    self,\n    wide: Optional[nn.Module] = None,\n    deeptabular: Optional[BaseWDModelComponent] = None,\n    deeptext: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deepimage: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deephead: Optional[BaseWDModelComponent] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: float = 0.1,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = True,\n    enforce_positive: bool = False,\n    enforce_positive_activation: str = \"softplus\",\n    pred_dim: int = 1,\n):\n    super(WideDeep, self).__init__()\n\n    self._check_inputs(\n        wide,\n        deeptabular,\n        deeptext,\n        deepimage,\n        deephead,\n        head_hidden_dims,\n        pred_dim,\n    )\n\n    # this attribute will be eventually over-written by the Trainer's\n    # device. Acts here as a 'placeholder'.\n    self.wd_device: Optional[str] = None\n\n    # required as attribute just in case we pass a deephead\n    self.pred_dim = pred_dim\n\n    self.enforce_positive = enforce_positive\n\n    # The main 5 components of the wide and deep assemble: wide,\n    # deeptabular, deeptext, deepimage and deephead\n    self.with_deephead = deephead is not None or head_hidden_dims is not None\n    if deephead is None and head_hidden_dims is not None:\n        self.deephead = self._build_deephead(\n            deeptabular,\n            deeptext,\n            deepimage,\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    elif deephead is not None:\n        self.deephead = nn.Sequential(\n            deephead, nn.Linear(deephead.output_dim, self.pred_dim)\n        )\n    else:\n        # for consistency with other components we default to None\n        self.deephead = None\n\n    self.wide = wide\n    self.deeptabular, self.deeptext, self.deepimage = self._set_model_components(\n        deeptabular, deeptext, deepimage, self.with_deephead\n    )\n\n    if self.enforce_positive:\n        self.enf_pos = get_activation_fn(enforce_positive_activation)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html","title":"The <code>preprocessing</code> module","text":"<p>This module contains the classes that are used to prepare the data before being passed to the models. There is one Preprocessor per data mode or model component (<code>wide</code>, <code>deeptabular</code>, <code>deepimage</code> and <code>deeptext</code>) with the exception of the <code>deeptext</code> component. In this case, two processors are available: one for the case when no Hugging Face model is used (<code>TextPreprocessor</code>) and another one when a Hugging Face model is used (<code>HFPreprocessor</code>).</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor","title":"WidePreprocessor","text":"<pre><code>WidePreprocessor(wide_cols, crossed_cols=None)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>WidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> <ul> <li> <code>wide_cols</code>               (<code>List[str]</code>)           \u2013            <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> </li> <li> <code>crossed_cols</code>               (<code>Optional[List[Tuple[str, str]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_crossed_cols</code>               (<code>List</code>)           \u2013            <p>List with the names of all columns that will be label encoded</p> </li> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>the inverse encoding dictionary</p> </li> <li> <code>wide_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the wide model (i.e. dim of the linear layer)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n&gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n&gt;&gt;&gt; X_wide\narray([[1, 4],\n       [2, 5],\n       [3, 6]])\n&gt;&gt;&gt; wide_preprocessor.encoding_dict\n{'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n&gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n  color color_size\n0     r        r-s\n1     b        b-n\n2     g        g-l\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def __init__(\n    self, wide_cols: List[str], crossed_cols: Optional[List[Tuple[str, str]]] = None\n):\n    super(WidePreprocessor, self).__init__()\n\n    self.wide_cols = wide_cols\n    self.crossed_cols = crossed_cols\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>WidePreprocessor</code>           \u2013            <p><code>WidePreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    WidePreprocessor\n        `WidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(df)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n    glob_feature_list = self._make_global_feature_list(\n        df_wide[self.wide_crossed_cols]\n    )\n    # leave 0 for padding/\"unseen\" categories\n    self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n    self.wide_dim = len(self.encoding_dict)\n    self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n    self.inverse_encoding_dict[0] = \"unseen\"\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    r\"\"\"\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, attributes=[\"encoding_dict\"])\n    df_wide = self._prepare_wide(df)\n    encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n    for col_i, col in enumerate(self.wide_crossed_cols):\n        encoded[:, col_i] = df_wide[col].apply(\n            lambda x: (\n                self.encoding_dict[col + \"_\" + str(x)]\n                if col + \"_\" + str(x) in self.encoding_dict\n                else 0\n            )\n        )\n    return encoded.astype(\"int64\")\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> <ul> <li> <code>encoded</code>               (<code>ndarray</code>)           \u2013            <p>numpy array with the encoded values that are the output from the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original values</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        numpy array with the encoded values that are the output from the\n        `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n    if pd.__version__ &gt;= \"2.1.0\":\n        decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n    else:\n        decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n    for col in decoded.columns:\n        rm_str = \"\".join([col, \"_\"])\n        decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor","title":"TabPreprocessor","text":"<pre><code>TabPreprocessor(\n    cat_embed_cols=None,\n    continuous_cols=None,\n    quantization_setup=None,\n    cols_to_scale=None,\n    auto_embed_dim=True,\n    embedding_rule=\"fastai_new\",\n    default_embed_dim=16,\n    with_attention=False,\n    with_cls_token=False,\n    shared_embed=False,\n    verbose=1,\n    *,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> <ul> <li> <code>cat_embed_cols</code>               (<code>Optional[Union[List[str], List[Tuple[str, int]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the continuous cols</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used.</p> </li> <li> <code>cols_to_scale</code>               (<code>Optional[Union[List[str], str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> </li> <li> <code>auto_embed_dim</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> </li> <li> <code>embedding_rule</code>               (<code>Literal[google, fastai_old, fastai_new]</code>, default:                   <code>'fastai_new'</code> )           \u2013            <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6 \\times n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> </li> <li> <code>default_embed_dim</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical and/or continuous embeddings will be concatenated before being passed to the final MLP (if present).</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            </li> <li> <code>scale</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> </li> <li> <code>already_standard</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embed_dim</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are the embed cols and values are the embedding dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute is not generated during the <code>fit</code> process</p> </li> <li> <code>label_encoder</code>               (<code>LabelEncoder</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p> </li> <li> <code>cat_embed_input</code>               (<code>List</code>)           \u2013            <p>List of Tuples with the column name, number of individual values for that column and, If <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> </li> <li> <code>standardize_cols</code>               (<code>List</code>)           \u2013            <p>List of the columns that will be standarized</p> </li> <li> <code>scaler</code>               (<code>StandardScaler</code>)           \u2013            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code></p> </li> <li> <code>column_idx</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are column names and values are column indexes. This is neccesary to slice tensors</p> </li> <li> <code>quantizer</code>               (<code>Quantizer</code>)           \u2013            <p>an instance of <code>Quantizer</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n&gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n&gt;&gt;&gt; cont_cols = ['age']\n&gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n&gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n[('color', 5), ('size', 5)]\n&gt;&gt;&gt; deep_preprocessor.column_idx\n{'color': 0, 'size': 1, 'age': 2}\n&gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n&gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n&gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n&gt;&gt;&gt; # or...\n&gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n&gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n&gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\n@alias(\"cat_embed_cols\", [\"embed_cols\"])\n@alias(\"scale\", [\"scale_cont_cols\"])\n@alias(\"quantization_setup\", [\"cols_and_bins\"])\ndef __init__(\n    self,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    quantization_setup: Optional[\n        Union[int, Dict[str, Union[int, List[float]]]]\n    ] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    auto_embed_dim: bool = True,\n    embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n    default_embed_dim: int = 16,\n    with_attention: bool = False,\n    with_cls_token: bool = False,\n    shared_embed: bool = False,\n    verbose: int = 1,\n    *,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    super(TabPreprocessor, self).__init__()\n\n    self.continuous_cols = continuous_cols\n    self.quantization_setup = quantization_setup\n    self.cols_to_scale = cols_to_scale\n    self.scale = scale\n    self.already_standard = already_standard\n    self.auto_embed_dim = auto_embed_dim\n    self.embedding_rule = embedding_rule\n    self.default_embed_dim = default_embed_dim\n    self.with_attention = with_attention\n    self.with_cls_token = with_cls_token\n    self.shared_embed = shared_embed\n    self.verbose = verbose\n\n    self.quant_args = {\n        k: v for k, v in kwargs.items() if k in pd.cut.__code__.co_varnames\n    }\n    self.scale_args = {\n        k: v for k, v in kwargs.items() if k in StandardScaler().get_params()\n    }\n\n    self._check_inputs(cat_embed_cols)\n\n    if with_cls_token:\n        self.cat_embed_cols = (\n            [\"cls_token\"] + cat_embed_cols  # type: ignore[operator]\n            if cat_embed_cols is not None\n            else [\"cls_token\"]\n        )\n    else:\n        self.cat_embed_cols = cat_embed_cols  # type: ignore[assignment]\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>TabPreprocessor</code>           \u2013            <p><code>TabPreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n    \"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TabPreprocessor\n        `TabPreprocessor` fitted object\n    \"\"\"\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    self.column_idx: Dict[str, int] = {}\n\n    # Categorical embeddings logic\n    if self.cat_embed_cols is not None or self.quantization_setup is not None:\n        self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n            []\n        )\n\n    if self.cat_embed_cols is not None:\n        df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n        self.label_encoder = LabelEncoder(\n            columns_to_encode=df_cat.columns.tolist(),\n            shared_embed=self.shared_embed,\n            with_attention=self.with_attention,\n        )\n        self.label_encoder.fit(df_cat)\n\n        for k, v in self.label_encoder.encoding_dict.items():\n            if self.with_attention:\n                self.cat_embed_input.append((k, len(v)))\n            else:\n                self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n        self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n    # Continuous columns logic\n    if self.continuous_cols is not None:\n        df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n        # Standardization logic\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args).fit(\n                df_cont[self.standardize_cols].values\n            )\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # we do not run 'Quantizer.fit' here since in the wild case\n            # someone wants standardization and quantization for the same\n            # columns, the Quantizer will run on the scaled data\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n            if self.with_attention:\n                for col, n_cat, _ in cont_embed_dim:\n                    self.cat_embed_input.append((col, n_cat))\n            else:\n                self.cat_embed_input.extend(cont_embed_dim)\n\n        self.column_idx.update(\n            {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the processed <code>dataframe</code> as a np.ndarray</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n    \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, condition=self.is_fitted)\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    if self.cat_embed_cols is not None:\n        df_cat = df_adj[self.cat_cols]\n        df_cat = self.label_encoder.transform(df_cat)\n    if self.continuous_cols is not None:\n        df_cont = df_adj[self.continuous_cols]\n        # Standardization logic\n        if self.standardize_cols:\n            df_cont[self.standardize_cols] = self.scaler.transform(\n                df_cont[self.standardize_cols].values\n            )\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # Adjustment so I don't have to override the method\n            # in 'ChunkTabPreprocessor'\n            if self.quantizer.is_fitted:\n                df_cont = self.quantizer.transform(df_cont)\n            else:\n                df_cont = self.quantizer.fit_transform(df_cont)\n    try:\n        df_deep = pd.concat([df_cat, df_cont], axis=1)\n    except NameError:\n        try:\n            df_deep = df_cat.copy()\n        except NameError:\n            df_deep = df_cont.copy()\n\n    return df_deep.values\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> <ul> <li> <code>encoded</code>               (<code>ndarray</code>)           \u2013            <p>array with the output of the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original values</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n    # embeddings back to original category\n    if self.cat_embed_cols is not None:\n        decoded = self.label_encoder.inverse_transform(decoded)\n    if self.continuous_cols is not None:\n        # quantized cols to the mid point\n        if self.cols_and_bins is not None:\n            if self.verbose:\n                print(\n                    \"Note that quantized cols will be turned into the mid point of \"\n                    \"the corresponding bin\"\n                )\n            for k, v in self.quantizer.inversed_bins.items():\n                decoded[k] = decoded[k].map(v)\n        # continuous_cols back to non-standarised\n        try:\n            decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                decoded[self.standardize_cols]\n            )\n        except Exception:  # KeyError:\n            pass\n\n    if \"cls_token\" in decoded.columns:\n        decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer","title":"Quantizer","text":"<pre><code>Quantizer(quantization_setup, **kwargs)\n</code></pre> <p>Helper class to perform the quantization of continuous columns. It is included in this docs for completion, since depending on the value of the parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that class might have an attribute of type <code>Quantizer</code>. However, this class is designed to always run internally within the <code>TabPreprocessor</code> class.</p> <p>Parameters:</p> <ul> <li> <code>quantization_setup</code>               (<code>Dict[str, Union[int, List[float]]]</code>)           \u2013            <p>Dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges.</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    quantization_setup: Dict[str, Union[int, List[float]]],\n    **kwargs,\n):\n    self.quantization_setup = quantization_setup\n    self.quant_args = kwargs\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor","title":"TextPreprocessor","text":"<pre><code>TextPreprocessor(\n    text_col,\n    max_vocab=30000,\n    min_freq=5,\n    maxlen=80,\n    pad_first=True,\n    pad_idx=1,\n    already_processed=False,\n    word_vectors_path=None,\n    n_cpus=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> <ul> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>column in the input dataframe containing the texts</p> </li> <li> <code>max_vocab</code>               (<code>int</code>, default:                   <code>30000</code> )           \u2013            <p>Maximum number of tokens in the vocabulary</p> </li> <li> <code>min_freq</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>Minimum frequency for a token to be part of the vocabulary</p> </li> <li> <code>maxlen</code>               (<code>int</code>, default:                   <code>80</code> )           \u2013            <p>Maximum length of the tokenized sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> <li> <code>already_processed</code>               (<code>Optional[bool]</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the sequence of elements is already processed or prepared. If this is the case, this Preprocessor will simply tokenize and pad the sequence. </p> <pre><code>Param aliases: `not_text`. &lt;br/&gt;\n</code></pre> <p>This parameter is thought for those cases where the input sequences are already fully processed or are directly not text (e.g. IDs)</p> </li> <li> <code>word_vectors_path</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Path to the pretrained word vectors</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vocab</code>               (<code>Vocab</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p> </li> <li> <code>embedding_matrix</code>               (<code>ndarray</code>)           \u2013            <p>Array with the pretrained embeddings</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n&gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\nThe vocabulary contains 24 tokens\narray([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n&gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n&gt;&gt;&gt; text_preprocessor.transform(df_te)\narray([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>@alias(\"already_processed\", [\"not_text\"])\ndef __init__(\n    self,\n    text_col: str,\n    max_vocab: int = 30000,\n    min_freq: int = 5,\n    maxlen: int = 80,\n    pad_first: bool = True,\n    pad_idx: int = 1,\n    already_processed: Optional[bool] = False,\n    word_vectors_path: Optional[str] = None,\n    n_cpus: Optional[int] = None,\n    verbose: int = 1,\n):\n    super(TextPreprocessor, self).__init__()\n\n    self.text_col = text_col\n    self.max_vocab = max_vocab\n    self.min_freq = min_freq\n    self.maxlen = maxlen\n    self.pad_first = pad_first\n    self.pad_idx = pad_idx\n    self.already_processed = already_processed\n    self.word_vectors_path = word_vectors_path\n    self.verbose = verbose\n    self.n_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Builds the vocabulary</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>TextPreprocessor</code>           \u2013            <p><code>TextPreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n    \"\"\"Builds the vocabulary\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TextPreprocessor\n        `TextPreprocessor` fitted object\n    \"\"\"\n    texts = self._read_texts(df)\n\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n    self.vocab: TVocab = Vocab(\n        max_vocab=self.max_vocab,\n        min_freq=self.min_freq,\n        pad_idx=self.pad_idx,\n    ).fit(\n        tokens,\n    )\n\n    if self.verbose:\n        print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n    if self.word_vectors_path is not None:\n        self.embedding_matrix = build_embeddings_matrix(\n            self.vocab, self.word_vectors_path, self.min_freq\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the padded, 'numericalised' sequences</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequences\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    texts = self._read_texts(df)\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Returns the padded, 'numericalised' sequence</p> <p>Parameters:</p> <ul> <li> <code>text</code>               (<code>str</code>)           \u2013            <p>text to be tokenized and padded</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequence</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequence\n\n    Parameters\n    ----------\n    text: str\n        text to be tokenized and padded\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequence\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    tokens = get_texts([text], self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(padded_seq)\n</code></pre> <p>Returns the original text plus the added 'special' tokens</p> <p>Parameters:</p> <ul> <li> <code>padded_seq</code>               (<code>ndarray</code>)           \u2013            <p>array with the output of the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original text plus the added 'special' tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n    \"\"\"Returns the original text plus the added 'special' tokens\n\n    Parameters\n    ----------\n    padded_seq: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original text plus the added 'special' tokens\n    \"\"\"\n    texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n    return pd.DataFrame({self.text_col: texts})\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor","title":"HFPreprocessor","text":"<pre><code>HFPreprocessor(\n    model_name,\n    *,\n    use_fast_tokenizer=False,\n    text_col=None,\n    root_dir=None,\n    num_workers=None,\n    preprocessing_rules=None,\n    tokenizer_params=None,\n    encode_params=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this class is designed to be as flexible as possible. Therefore, it is coded so that the user can use it as one would use any HuggingFace tokenizers, or following the API call 'protocol' of the rest of the library.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>use_fast_tokenizer</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Whether to use the fast tokenizer from HuggingFace or not</p> </li> <li> <code>text_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>The column in the input dataframe containing the text data. If this tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this argument is mandatory. If the tokenizer is used via the <code>encode</code> method, this argument is not needed since the input text is passed directly to the <code>encode</code> method.</p> </li> <li> <code>num_workers</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> </li> <li> <code>preprocessing_rules</code>               (<code>Optional[List[Callable[[str], str]]]</code>, default:                   <code>None</code> )           \u2013            <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> </li> <li> <code>tokenizer_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code> can also be passed via the <code>**kwargs</code> argument</p> </li> <li> <code>encode_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code> methods are used, the <code>encode_params</code> dict parameter is mandatory. If the <code>encode</code> method is used, this parameter is not needed since the input text is passed directly to the <code>encode</code> method.</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional kwargs to be passed to the model, in particular to the <code>PreTrainedTokenizer</code> class.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>is_fitted</code>               (<code>bool</code>)           \u2013            <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n&gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n&gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n&gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n&gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n&gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    model_name: str,\n    *,\n    use_fast_tokenizer: bool = False,\n    text_col: Optional[str] = None,\n    root_dir: Optional[str] = None,\n    num_workers: Optional[int] = None,\n    preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n    tokenizer_params: Optional[Dict[str, Any]] = None,\n    encode_params: Optional[Dict[str, Any]] = None,\n    **kwargs,\n):\n    self.model_name = model_name\n    self.use_fast_tokenizer = use_fast_tokenizer\n    self.text_col = text_col\n    self.root_dir = root_dir\n    self.num_workers = num_workers\n    self.preprocessing_rules = preprocessing_rules\n    self.tokenizer_params = tokenizer_params if tokenizer_params is not None else {}\n    self.encode_params = encode_params if encode_params is not None else {}\n\n    self._multiprocessing = (\n        num_workers is not None and num_workers &gt; 1 and not use_fast_tokenizer\n    )\n\n    if kwargs:\n        self.tokenizer_params.update(kwargs)\n\n    self.tokenizer = get_tokenizer(\n        model_name=self.model_name,\n        use_fast_tokenizer=self.use_fast_tokenizer,\n        **self.tokenizer_params,\n    )\n\n    # A HuggingFace tokenizer is already trained, since we need this\n    # attribute elsewhere in the library, we simply set it to True\n    self.is_fitted = True\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode","title":"encode","text":"<pre><code>encode(texts, **kwargs)\n</code></pre> <p>Encodes a list of texts. The method is a wrapper around the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p> <p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p> <p>Parameters:</p> <ul> <li> <code>texts</code>               (<code>List[str]</code>)           \u2013            <p>List of texts to be encoded</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer. If the 'encode_params' dict was passed when instantiating the class, that dictionaly will be updated with the kwargs passed here.</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a list of texts. The method is a wrapper around the\n    `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n    if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n    Parameters\n    ----------\n    texts: List[str]\n        List of texts to be encoded\n    **kwargs\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n        when instantiating the class, that dictionaly will be updated with\n        the kwargs passed here.\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if kwargs:\n        self.encode_params.update(kwargs)\n\n    if self.preprocessing_rules:\n        if self._multiprocessing:\n            texts = self._process_text_parallel(texts)\n        else:\n            texts = [self._preprocess_text(text) for text in texts]\n\n    if self._multiprocessing:\n        input_ids = self._encode_paralell(texts, **self.encode_params)\n    else:\n        encoded_texts = self.tokenizer.batch_encode_plus(\n            texts,\n            **self.encode_params,\n        )\n        input_ids = encoded_texts.get(\"input_ids\")\n\n    self.is_fitted = True\n\n    try:\n        output = np.array(input_ids)\n    except ValueError:\n        warnings.warn(\n            \"Padding and Truncating parameters were not passed and all input arrays \"\n            \"do not have the same shape. Padding to the longest sequence. \"\n            \"Padding will be done with the index of the pad token for the model\",\n            UserWarning,\n        )\n        max_len = max([len(ids) for ids in input_ids])\n        output = np.array(\n            [\n                np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                for ids in input_ids\n            ]\n        )\n\n    return output\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode","title":"decode","text":"<pre><code>decode(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method is a wrapper around the <code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the HuggingFace's tokenizer.</p> <p>Parameters:</p> <ul> <li> <code>input_ids</code>               (<code>NDArray[int64]</code>)           \u2013            <p>The input_ids to be decoded</p> </li> <li> <code>skip_special_tokens</code>               (<code>bool</code>)           \u2013            <p>Whether to skip the special tokens or not</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>The decoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def decode(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method is a wrapper around the\n    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n    HuggingFace's tokenizer.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    texts = [\n        self.tokenizer.convert_tokens_to_string(\n            self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n        )\n        for i in range(input_ids.shape[0])\n    ]\n    return texts\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>This method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's tokenizers and models are already trained. Therefore, the 'fit' method here does nothing other than checking that the 'text_col' parameter is not <code>None</code>.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n    \"\"\"\n    This method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular. HuggingFace's\n    tokenizers and models are already trained. Therefore, the 'fit' method\n    here does nothing other than checking that the 'text_col' parameter is\n    not `None`.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe. This method simply calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method, this method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe. This method simply\n    calls the `encode` method under the hood. Similar to the `fit` method,\n    this method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n\n    texts = self._read_texts(df, self.root_dir)\n\n    return self.encode(texts)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Encodes a single text sample.</p> <p>Parameters:</p> <ul> <li> <code>text</code>               (<code>str</code>)           \u2013            <p>The text sample to be encoded</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded text</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a single text sample.\n\n    Parameters\n    ----------\n    text: str\n        The text sample to be encoded\n\n    Returns\n    -------\n    np.array\n        The encoded text\n    \"\"\"\n\n    if not self.is_fitted:\n        raise ValueError(\n            \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n        )\n    return self.encode([text])[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method under the hood.</p> <p>Parameters:</p> <ul> <li> <code>input_ids</code>               (<code>NDArray[int64]</code>)           \u2013            <p>The input_ids to be decoded</p> </li> <li> <code>skip_special_tokens</code>               (<code>bool</code>)           \u2013            <p>Whether to skip the special tokens or not</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>The decoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def inverse_transform(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method simply calls the `decode` method\n    under the hood.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    return self.decode(input_ids, skip_special_tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor","title":"ImagePreprocessor","text":"<pre><code>ImagePreprocessor(\n    img_col, img_path, width=224, height=224, verbose=1\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p> <p>The Preprocessing consists simply on resizing according to their aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>img_col</code>               (<code>str</code>)           \u2013            <p>name of the column with the images filenames</p> </li> <li> <code>img_path</code>               (<code>str</code>)           \u2013            <p>path to the dicrectory where the images are stored</p> </li> <li> <code>width</code>               (<code>int</code>, default:                   <code>224</code> )           \u2013            <p>width of the resulting processed image.</p> </li> <li> <code>height</code>               (<code>int</code>, default:                   <code>224</code> )           \u2013            <p>width of the resulting processed image.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>aap</code>               (<code>AspectAwarePreprocessor</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p> </li> <li> <code>spp</code>               (<code>SimplePreprocessor</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p> </li> <li> <code>normalise_metrics</code>               (<code>Dict</code>)           \u2013            <p>Dict containing the normalisation metrics of the image dataset, i.e. mean and std for the R, G and B channels</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n&gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n&gt;&gt;&gt;\n&gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n&gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n&gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n&gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n&gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n</code></pre> <p> NOTE: Normalising metrics will only be computed when the <code>fit_transform</code> method is run. Running <code>transform</code> only will not change the computed metrics and running <code>fit</code> only simply instantiates the resizing functions.</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    img_col: str,\n    img_path: str,\n    width: int = 224,\n    height: int = 224,\n    verbose: int = 1,\n):\n    super(ImagePreprocessor, self).__init__()\n\n    self.img_col = img_col\n    self.img_path = img_path\n    self.width = width\n    self.height = height\n    self.verbose = verbose\n\n    self.aap = AspectAwarePreprocessor(self.width, self.height)\n    self.spp = SimplePreprocessor(self.width, self.height)\n\n    self.compute_normalising_computed = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Resizes the images to the input height and width.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe with the <code>img_col</code></p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized images to the input height and width</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Resizes the images to the input height and width.\n\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe with the `img_col`\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    image_list = df[self.img_col].tolist()\n    if self.verbose:\n        print(\"Reading Images from {}\".format(self.img_path))\n    imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n    # finding images with different height and width\n    aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n    aspect_r = [a[0] / a[1] for a in aspect]\n    diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n    if self.verbose:\n        print(\"Resizing\")\n    resized_imgs = []\n    for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n        if i in diff_idx:\n            resized_imgs.append(self.aap.preprocess(img))\n        else:\n            # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n            resized_imgs.append(self.spp.preprocess(img))\n\n    if not self.compute_normalising_computed:\n        if self.verbose:\n            print(\"Computing normalisation metrics\")\n        # mean and std deviation will only be computed when the fit method\n        # is called\n        mean_R, mean_G, mean_B = [], [], []\n        std_R, std_G, std_B = [], [], []\n        for rsz_img in resized_imgs:\n            (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                rsz_img\n            )\n            mean_R.append(mean_r)\n            mean_G.append(mean_g)\n            mean_B.append(mean_b)\n            std_R.append(std_r)\n            std_G.append(std_g)\n            std_B.append(std_b)\n        self.normalise_metrics = dict(\n            mean={\n                \"R\": np.mean(mean_R) / 255.0,\n                \"G\": np.mean(mean_G) / 255.0,\n                \"B\": np.mean(mean_B) / 255.0,\n            },\n            std={\n                \"R\": np.mean(std_R) / 255.0,\n                \"G\": np.mean(std_G) / 255.0,\n                \"B\": np.mean(std_B) / 255.0,\n            },\n        )\n        self.compute_normalising_computed = True\n    return np.asarray(resized_imgs)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized images to the input height and width</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#chunked-versions","title":"Chunked versions","text":"<p>Chunked versions of the preprocessors are also available. These are useful when the data is too big to fit in memory. See also the <code>load_from_folder</code> module in the library and the corresponding section here in the documentation.</p> <p>Note that there is not a <code>ChunkImagePreprocessor</code>. This is because the processing of the images will occur inside the <code>ImageFromFolder</code> class in the <code>load_from_folder</code> module.</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor","title":"ChunkWidePreprocessor","text":"<pre><code>ChunkWidePreprocessor(\n    wide_cols, n_chunks, crossed_cols=None\n)\n</code></pre> <p>               Bases: <code>WidePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>ChunkWidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> <ul> <li> <code>wide_cols</code>               (<code>List[str]</code>)           \u2013            <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> </li> <li> <code>crossed_cols</code>               (<code>Optional[List[Tuple[str, str]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_crossed_cols</code>               (<code>List</code>)           \u2013            <p>List with the names of all columns that will be label encoded</p> </li> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>the inverse encoding dictionary</p> </li> <li> <code>wide_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the wide model (i.e. dim of the linear layer)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n&gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n... n_chunks=1)\n&gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    wide_cols: List[str],\n    n_chunks: int,\n    crossed_cols: Optional[List[Tuple[str, str]]] = None,\n):\n    super(ChunkWidePreprocessor, self).__init__(wide_cols, crossed_cols)\n\n    self.n_chunks = n_chunks\n\n    self.chunk_counter = 0\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(chunk)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>chunk</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ChunkWidePreprocessor</code>           \u2013            <p><code>ChunkWidePreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    chunk: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    ChunkWidePreprocessor\n        `ChunkWidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(chunk)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n\n    if self.chunk_counter == 0:\n        self.glob_feature_set = set(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n    else:\n        self.glob_feature_set.update(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n\n    self.chunk_counter += 1\n\n    if self.chunk_counter == self.n_chunks:\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Runs <code>partial_fit</code>. This is just to override the fit method in the base class. This class is not designed or thought to run fit</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    \"\"\"\n    Runs `partial_fit`. This is just to override the fit method in the base\n    class. This class is not designed or thought to run fit\n    \"\"\"\n    return self.partial_fit(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor","title":"ChunkTabPreprocessor","text":"<pre><code>ChunkTabPreprocessor(\n    n_chunks,\n    cat_embed_cols=None,\n    continuous_cols=None,\n    cols_and_bins=None,\n    cols_to_scale=None,\n    default_embed_dim=16,\n    with_attention=False,\n    with_cls_token=False,\n    shared_embed=False,\n    verbose=1,\n    *,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>TabPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> <ul> <li> <code>n_chunks</code>               (<code>int</code>)           \u2013            <p>Number of chunks that the tabular dataset is divided by.</p> </li> <li> <code>cat_embed_cols</code>               (<code>Optional[Union[List[str], List[Tuple[str, int]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the continuous cols</p> </li> <li> <code>cols_and_bins</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column names to quantize and the values are a list of scalars indicating the bin edges.</p> </li> <li> <code>cols_to_scale</code>               (<code>Optional[Union[List[str], str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code></p> </li> <li> <code>default_embed_dim</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Dimension for the embeddings if the embed_dim is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical (and continuous embeddings if present) will be concatenated before being passed to the final MLP (if present).</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            </li> <li> <code>scale</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> </li> <li> <code>already_standard</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embed_dim</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are the embed cols and values are the embedding dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute is not generated during the <code>fit</code> process</p> </li> <li> <code>label_encoder</code>               (<code>LabelEncoder</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p> </li> <li> <code>cat_embed_input</code>               (<code>List</code>)           \u2013            <p>List of Tuples with the column name, number of individual values for that column and, If <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> </li> <li> <code>standardize_cols</code>               (<code>List</code>)           \u2013            <p>List of the columns that will be standarized</p> </li> <li> <code>scaler</code>               (<code>StandardScaler</code>)           \u2013            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code> if 'cols_to_scale' is not None or 'scale' is 'True'</p> </li> <li> <code>column_idx</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are column names and values are column indexes. This is neccesary to slice tensors</p> </li> <li> <code>quantizer</code>               (<code>Quantizer</code>)           \u2013            <p>an instance of <code>Quantizer</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n&gt;&gt;&gt; np.random.seed(42)\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n... 'cont_col': np.random.uniform(1, 100, size=8)})\n&gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n&gt;&gt;&gt; cont_cols = ['cont_col']\n&gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n... )\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n&gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n[('cat_col', 4)]\n&gt;&gt;&gt; tab_preprocessor.column_idx\n{'cat_col': 0, 'cont_col': 1}\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\n@alias(\"cat_embed_cols\", [\"embed_cols\"])\n@alias(\"scale\", [\"scale_cont_cols\"])\n@alias(\"cols_and_bins\", [\"quantization_setup\"])\ndef __init__(\n    self,\n    n_chunks: int,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cols_and_bins: Optional[Dict[str, List[float]]] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    default_embed_dim: int = 16,\n    with_attention: bool = False,\n    with_cls_token: bool = False,\n    shared_embed: bool = False,\n    verbose: int = 1,\n    *,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    super(ChunkTabPreprocessor, self).__init__(\n        cat_embed_cols=cat_embed_cols,\n        continuous_cols=continuous_cols,\n        quantization_setup=None,\n        cols_to_scale=cols_to_scale,\n        auto_embed_dim=False,\n        embedding_rule=\"google\",  # does not matter, irrelevant\n        default_embed_dim=default_embed_dim,\n        with_attention=with_attention,\n        with_cls_token=with_cls_token,\n        shared_embed=shared_embed,\n        verbose=verbose,\n        scale=scale,\n        already_standard=already_standard,\n        **kwargs,\n    )\n\n    self.n_chunks = n_chunks\n    self.chunk_counter = 0\n\n    self.cols_and_bins = cols_and_bins  # type: ignore[assignment]\n    if self.cols_and_bins is not None:\n        self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n    self.embed_prepared = False\n    self.continuous_prepared = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor","title":"ChunkTextPreprocessor","text":"<pre><code>ChunkTextPreprocessor(\n    text_col,\n    n_chunks,\n    root_dir=None,\n    max_vocab=30000,\n    min_freq=5,\n    maxlen=80,\n    pad_first=True,\n    pad_idx=1,\n    already_processed=False,\n    word_vectors_path=None,\n    n_cpus=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>TextPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> <ul> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>column in the input dataframe containing either the texts or the filenames where the text documents are stored</p> </li> <li> <code>n_chunks</code>               (<code>int</code>)           \u2013            <p>Number of chunks that the text dataset is divided by.</p> </li> <li> <code>root_dir</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>If 'text_col' contains the filenames with the text documents, this is the path to the directory where those documents are stored.</p> </li> <li> <code>max_vocab</code>               (<code>int</code>, default:                   <code>30000</code> )           \u2013            <p>Maximum number of tokens in the vocabulary</p> </li> <li> <code>min_freq</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>Minimum frequency for a token to be part of the vocabulary</p> </li> <li> <code>maxlen</code>               (<code>int</code>, default:                   <code>80</code> )           \u2013            <p>Maximum length of the tokenized sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> <li> <code>word_vectors_path</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Path to the pretrained word vectors</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vocab</code>               (<code>Vocab</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p> </li> <li> <code>embedding_matrix</code>               (<code>ndarray</code>)           \u2013            <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n&gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    text_col: str,\n    n_chunks: int,\n    root_dir: Optional[str] = None,\n    max_vocab: int = 30000,\n    min_freq: int = 5,\n    maxlen: int = 80,\n    pad_first: bool = True,\n    pad_idx: int = 1,\n    already_processed: Optional[bool] = False,\n    word_vectors_path: Optional[str] = None,\n    n_cpus: Optional[int] = None,\n    verbose: int = 1,\n):\n    super(ChunkTextPreprocessor, self).__init__(\n        text_col=text_col,\n        max_vocab=max_vocab,\n        min_freq=min_freq,\n        maxlen=maxlen,\n        pad_first=pad_first,\n        pad_idx=pad_idx,\n        already_processed=already_processed,\n        word_vectors_path=word_vectors_path,\n        n_cpus=n_cpus,\n        verbose=verbose,\n    )\n\n    self.n_chunks = n_chunks\n    self.root_dir = root_dir\n\n    self.chunk_counter = 0\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor","title":"ChunkHFPreprocessor","text":"<pre><code>ChunkHFPreprocessor(\n    model_name,\n    *,\n    text_col,\n    root_dir=None,\n    use_fast_tokenizer=True,\n    num_workers=None,\n    preprocessing_rules=None,\n    tokenizer_params=None,\n    encode_params=None\n)\n</code></pre> <p>               Bases: <code>HFPreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Hugginface Tokenizer's are already 'trained'. Therefore, unlike the <code>ChunkTextPreprocessor</code> this is mostly identical to the <code>HFPreprocessor</code> with the only difference that the class needs a 'text_col' parameter to be passed. Also the parameter <code>encode_params</code> is not really optional when using this class. It must be passed containing at least the 'max_length' encoding parameter. This is because we need to ensure that  all sequences have the same length when encoding in chunks.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>The column in the input dataframe containing the text data. When using the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p> </li> <li> <code>root_dir</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>The root directory where the text files are located. This is only needed if the text data is stored in text files. If the text data is stored in a column in the input dataframe, this parameter is not needed.</p> </li> <li> <code>use_fast_tokenizer</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to use the fast tokenizer from HuggingFace or not</p> </li> <li> <code>num_workers</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> </li> <li> <code>preprocessing_rules</code>               (<code>Optional[List[Callable[[str], str]]]</code>, default:                   <code>None</code> )           \u2013            <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> </li> <li> <code>tokenizer_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>.</p> </li> <li> <code>encode_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the <code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It must be passed containing at least the 'max_length' encoding parameter</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>is_fitted</code>               (<code>bool</code>)           \u2013            <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    model_name: str,\n    *,\n    text_col: str,\n    root_dir: Optional[str] = None,\n    use_fast_tokenizer: bool = True,\n    num_workers: Optional[int] = None,\n    preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n    tokenizer_params: Optional[Dict[str, Any]] = None,\n    encode_params: Optional[Dict[str, Any]] = None,\n):\n    super().__init__(\n        model_name=model_name,\n        use_fast_tokenizer=use_fast_tokenizer,\n        text_col=text_col,\n        num_workers=num_workers,\n        preprocessing_rules=preprocessing_rules,\n        tokenizer_params=tokenizer_params,\n        encode_params=encode_params,\n    )\n\n    self.root_dir = root_dir\n\n    # when using in chunks encode_params is not really optional. I will\n    # review types in due time\n    if self.encode_params is None:\n        raise ValueError(\n            \"The 'encode_params' dict must be passed to the ChunkHFTokenizer \"\n            \"containing at least the 'max_length' encoding parameter\"\n        )\n\n    if \"padding\" not in self.encode_params or not self.encode_params[\"padding\"]:\n        self.encode_params[\"padding\"] = True\n\n    if (\n        \"truncation\" not in self.encode_params\n        or not self.encode_params[\"truncation\"]\n    ):\n        self.encode_params[\"truncation\"] = True\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html","title":"Self Supervised Pre-training for tabular data","text":"<p>In this library we have implemented two methods or routines that allow the user to use self-suerpvised pre-training for all tabular models in the library with the exception of the <code>TabPerceiver</code> (this is a particular model and self-supervised pre-training requires some adjustments that will be implemented in future versions). Please see the examples folder in the repo or the examples section in the docs for details on how to use self-supervised pre-training with this library.</p> <p>The two routines implemented are illustrated in the figures below. The first is from TabNet: Attentive Interpretable Tabular Learning. It is a 'standard' encoder-decoder architecture and and is designed here for models that do not use transformer-based architectures (or when the embeddings can all have different dimensions). The second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, it is based on Contrastive and Denoising learning and is designed for models that use transformer-based architectures (or when the embeddings all need to have the same dimension):</p> <p> </p> <p>Figure 1. Figure 2 in their paper. The caption of the original paper is included in case it is useful.</p> <p> </p> <p>Figure 2. Figure 1 in their paper. The caption of the original paper is included in case it is useful.</p> <p>Note that the self-supervised pre-trainers described below focus, of course, on the self-supervised pre-training phase, i.e. the left side in Figure 1 and the upper part in Figure 2. When combined with the <code>Trainer</code> described earlier in the documenation, one can reproduce the full process illustrated in the figures above.</p> <p>Also Note that it is beyond the scope of this docs to explain in detail these routines. In addition, to fully utilise the self-supervised trainers implemented in this library a minimum understanding of the processes as described in the papers is required. Therefore, we strongly encourage the users to have a look to the papers.</p>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer","title":"EncoderDecoderTrainer","text":"<pre><code>EncoderDecoderTrainer(\n    encoder,\n    decoder=None,\n    masked_prob=0.2,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseEncoderDecoderTrainer</code></p> <p>This class implements an Encoder-Decoder self-supervised 'routine' inspired by TabNet: Attentive Interpretable Tabular Learning. See Figure 1 above.</p> <p>Parameters:</p> <ul> <li> <code>encoder</code>               (<code>ModelWithoutAttention</code>)           \u2013            <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p> </li> <li> <code>decoder</code>               (<code>Optional[DecoderWithoutAttention]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or <code>TabNetDecoder</code> model. if <code>None</code> the decoder will be automatically built as a 'simetric' model to the Encoder</p> </li> <li> <code>masked_prob</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Indicates the fraction of elements in the embedding tensor that will be masked and hence used for reconstruction</p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def __init__(\n    self,\n    encoder: ModelWithoutAttention,\n    decoder: Optional[DecoderWithoutAttention] = None,\n    masked_prob: float = 0.2,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        encoder=encoder,\n        decoder=decoder,\n        masked_prob=masked_prob,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.ed_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer","title":"ContrastiveDenoisingTrainer","text":"<pre><code>ContrastiveDenoisingTrainer(\n    model,\n    preprocessor,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    loss_type=\"both\",\n    projection_head1_dims=None,\n    projection_head2_dims=None,\n    projection_heads_activation=\"relu\",\n    cat_mlp_type=\"multiple\",\n    cont_mlp_type=\"multiple\",\n    denoise_mlps_activation=\"relu\",\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseContrastiveDenoisingTrainer</code></p> <p>This class trains a Contrastive, Denoising Self Supervised 'routine' that is based on the one described in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their Figure 1.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>ModelWithAttention</code>)           \u2013            <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>, <code>TabFastFormer</code>, <code>TabPerceiver</code>, <code>ContextAttentionMLP</code> and <code>SelfAttentionMLP</code>.</p> </li> <li> <code>preprocessor</code>               (<code>TabPreprocessor</code>)           \u2013            <p>A fitted <code>TabPreprocessor</code> object. See <code>pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor</code></p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>loss_type</code>               (<code>Literal[contrastive, denoising, both]</code>, default:                   <code>'both'</code> )           \u2013            <p>One of 'contrastive', 'denoising' or 'both'. See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their figure (1) and their equation (5).</p> </li> <li> <code>projection_head1_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>The projection heads are simply MLPs. This parameter is a list of integers with the dimensions of the MLP hidden layers. See the paper for details. Note that setting up this parameter requires some knowledge of the architecture one is using. For example, if we are representing the features with embeddings of dim 32 (i.e. the so called dimension of the model is 32), then the first dimension of the projection head must be 32 (e.g. [32, 16])</p> </li> <li> <code>projection_head2_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>Same as 'projection_head1_dims' for the second head</p> </li> <li> <code>projection_heads_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the projection heads</p> </li> <li> <code>cat_mlp_type</code>               (<code>Literal[single, multiple]</code>, default:                   <code>'multiple'</code> )           \u2013            <p>If 'denoising' loss is used, one can choose two types of 'stacked' MLPs to process the output from the transformer-based encoder that receives 'corrupted' (cut-mixed and mixed-up) features. These are 'single' or 'multiple'. The former approach will apply a single MLP to all the categorical features while the latter will use one MLP per categorical feature</p> </li> <li> <code>cont_mlp_type</code>               (<code>Literal[single, multiple]</code>, default:                   <code>'multiple'</code> )           \u2013            <p>Same as 'cat_mlp_type' but for the continuous features</p> </li> <li> <code>denoise_mlps_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>activation function for the so called 'denoising mlps'.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def __init__(\n    self,\n    model: ModelWithAttention,\n    preprocessor: TabPreprocessor,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    loss_type: Literal[\"contrastive\", \"denoising\", \"both\"] = \"both\",\n    projection_head1_dims: Optional[List[int]] = None,\n    projection_head2_dims: Optional[List[int]] = None,\n    projection_heads_activation: str = \"relu\",\n    cat_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n    cont_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n    denoise_mlps_activation: str = \"relu\",\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        preprocessor=preprocessor,\n        loss_type=loss_type,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        projection_head1_dims=projection_head1_dims,\n        projection_head2_dims=projection_head2_dims,\n        projection_heads_activation=projection_heads_activation,\n        cat_mlp_type=cat_mlp_type,\n        cont_mlp_type=cont_mlp_type,\n        denoise_mlps_activation=denoise_mlps_activation,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data. Note that, although it is possible to use contrastive-denoising training with a validation set, such set must include feature values that are all seen in the training set in the case of the categorical columns. This is because the values of the columns themselves will be used as targets when computing the loss. Therefore, if a new category is present in the validation set that was not seen in training this will effectively be like trying to predict a new, never seen category (and Pytorch will throw an error)</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data. Note that, although it is possible to use\n        contrastive-denoising training with a validation set, such set\n        must include feature values that are _all_ seen in the training\n        set in the case of the categorical columns. This is because the\n        values of the columns themselves will be used as targets when\n        computing the loss. Therefore, if a new category is present in\n        the validation set that was not seen in training this will\n        effectively be like trying to predict a new, never seen category\n        (and Pytorch will throw an error)\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.cd_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html","title":"Tab2Vec","text":""},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec","title":"Tab2Vec","text":"<pre><code>Tab2Vec(\n    tab_preprocessor,\n    model,\n    return_dataframe=False,\n    verbose=False,\n)\n</code></pre> <p>Class to transform an input dataframe into vectorized form.</p> <p>This class will take an input dataframe in the form of the dataframe used for training, and it will turn it into a vectorised form based on the processing applied by the model to the categorical and continuous columns.</p> <p> NOTE: Currently this class is only implemented  for the deeptabular component. Therefore, if the input dataframe has a  text column or a column with the path to images, these will be ignored.  We will be adding these functionalities in future versions</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>Union[WideDeep, BayesianWide, BayesianTabMlp]</code>)           \u2013            <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p> </li> <li> <code>tab_preprocessor</code>               (<code>TabPreprocessor</code>)           \u2013            <p><code>TabPreprocessor</code> object. Must be fitted.</p> </li> <li> <code>return_dataframe</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating of the returned object(s) will be array(s) or pandas dataframe(s)</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vectorizer</code>               (<code>Module</code>)           \u2013            <p>Torch module with the categorical and continuous encoding process</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import string\n&gt;&gt;&gt; from random import choices\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n&gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n&gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n&gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n&gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # fit the TabPreprocessor\n&gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n&gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # define the model (and let's assume we train it)\n&gt;&gt;&gt; tabmlp = TabMlp(\n... column_idx=tab_preprocessor.column_idx,\n... cat_embed_input=tab_preprocessor.cat_embed_input,\n... continuous_cols=tab_preprocessor.continuous_cols,\n... mlp_hidden_dims=[8, 4])\n&gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n&gt;&gt;&gt; # ...train the model...\n&gt;&gt;&gt;\n&gt;&gt;&gt; # vectorise the dataframe\n&gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model)\n&gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n</code></pre> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def __init__(\n    self,\n    tab_preprocessor: TabPreprocessor,\n    model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n    return_dataframe: bool = False,\n    verbose: bool = False,\n):\n    super(Tab2Vec, self).__init__()\n\n    self._check_inputs(tab_preprocessor, model, verbose)\n\n    self.tab_preprocessor = tab_preprocessor\n    self.return_dataframe = return_dataframe\n    self.verbose = verbose\n\n    self.vectorizer = self._set_vectorizer(model)\n\n    self._set_dim_attributes(tab_preprocessor, model)\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit","title":"fit","text":"<pre><code>fit(df, target_col=None)\n</code></pre> <p>This is an empty method i.e. Returns the unchanged object itself. Is only included for consistency in case <code>Tab2Vec</code> is used as part of a Pipeline</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>Tab2Vec</code>           \u2013            </li> </ul> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n    r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n    only included for consistency in case `Tab2Vec` is used as part of a\n    Pipeline\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Tab2Vec\n    \"\"\"\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.transform","title":"transform","text":"<pre><code>transform(df, target_col=None)\n</code></pre> <p>Transforms the input dataframe into vectorized form. If a target column name is passed the target values will be returned separately in their corresponding type (np.ndarray or pd.DataFrame)</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code>           \u2013            <p>Returns eiter a numpy array with the vectorised values, or a Tuple of numpy arrays with the vectorised values and the target. The same applies to dataframes in case we choose to set <code>return_dataframe = True</code></p> </li> </ul> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def transform(\n    self,\n    df: pd.DataFrame,\n    target_col: Optional[str] = None,\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Transforms the input dataframe into vectorized form. If a target\n    column name is passed the target values will be returned separately\n    in their corresponding type (np.ndarray or pd.DataFrame)\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n        Returns eiter a numpy array with the vectorised values, or a Tuple\n        of numpy arrays with the vectorised values and the target. The\n        same applies to dataframes in case we choose to set\n        `return_dataframe = True`\n    \"\"\"\n\n    X_tab = self.tab_preprocessor.transform(df)\n    X = torch.from_numpy(X_tab.astype(\"float\")).to(device)\n\n    with torch.no_grad():\n        if self.is_tab_transformer:\n            x_vec, x_cont_not_embed = self.vectorizer(X)\n        else:\n            x_vec = self.vectorizer(X)\n            x_cont_not_embed = None\n\n    if self.tab_preprocessor.with_cls_token:\n        x_vec = x_vec[:, 1:, :]\n\n    if self.tab_preprocessor.with_attention:\n        x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n    if x_cont_not_embed is not None:\n        x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n    else:\n        x_vec = x_vec.detach().cpu().numpy()\n\n    if self.return_dataframe:\n        new_colnames = self._new_colnames()\n        if target_col:\n            return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n        else:\n            return pd.DataFrame(data=x_vec, columns=new_colnames)\n    else:\n        if target_col:\n            return x_vec, df[target_col].values\n        else:\n            return x_vec\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df, target_col=None)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit_transform(\n    self, df: pd.DataFrame, target_col: Optional[str] = None\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Combines `fit` and `transform`\"\"\"\n    return self.fit(df, target_col).transform(df, target_col)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html","title":"Training multimodal Deep Learning Models","text":"<p>Here is the documentation for the <code>Trainer</code> class, that will do all the heavy lifting.</p> <p>Trainer is also available from <code>pytorch-widedeep</code> directly, for example, one could do:</p> <pre><code>    from pytorch-widedeep.training import Trainer\n</code></pre> <p>or also:</p> <pre><code>    from pytorch-widedeep import Trainer\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer","title":"Trainer","text":"<pre><code>Trainer(\n    model,\n    objective,\n    custom_loss_function=None,\n    optimizers=None,\n    lr_schedulers=None,\n    initializers=None,\n    transforms=None,\n    callbacks=None,\n    metrics=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>WideDeep</code>)           \u2013            <p>An object of class <code>WideDeep</code></p> </li> <li> <code>objective</code>               (<code>str</code>)           \u2013            <p>Defines the objective, loss or cost function. </p> <p>Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code>. </p> <p>Possible values are:</p> <ul> <li> <p><code>binary</code>, aliases: <code>logistic</code>, <code>binary_logloss</code>, <code>binary_cross_entropy</code></p> </li> <li> <p><code>binary_focal_loss</code></p> </li> <li> <p><code>multiclass</code>, aliases: <code>multi_logloss</code>, <code>cross_entropy</code>, <code>categorical_cross_entropy</code>,</p> </li> <li> <p><code>multiclass_focal_loss</code></p> </li> <li> <p><code>regression</code>, aliases: <code>mse</code>, <code>l2</code>, <code>mean_squared_error</code></p> </li> <li> <p><code>mean_absolute_error</code>, aliases: <code>mae</code>, <code>l1</code></p> </li> <li> <p><code>mean_squared_log_error</code>, aliases: <code>msle</code></p> </li> <li> <p><code>root_mean_squared_error</code>, aliases:  <code>rmse</code></p> </li> <li> <p><code>root_mean_squared_log_error</code>, aliases: <code>rmsle</code></p> </li> <li> <p><code>zero_inflated_lognormal</code>, aliases: <code>ziln</code></p> </li> <li> <p><code>quantile</code></p> </li> <li> <p><code>tweedie</code></p> </li> <li> <p><code>multitarget</code>, aliases: <code>multi_target</code></p> </li> </ul> <p>NOTE: For <code>multitarget</code> a custom loss function must be passed</p> </li> <li> <code>custom_loss_function</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>It is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples section in this documentation or in the repo. Note that if <code>custom_loss_function</code> is not <code>None</code>, <code>objective</code> must be 'binary', 'multiclass' or 'regression', consistent with the loss function</p> </li> <li> <code>optimizers</code>               (<code>Optional[Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of Pytorch's <code>Optimizer</code> object   (e.g. <code>torch.optim.Adam()</code>) or</li> <li>a dictionary where there keys are the model components (i.e.   'wide', 'deeptabular', 'deeptext', 'deepimage'   and/or 'deephead')  and the values are the corresponding   optimizers or list of optimizers if multiple models are used for   the given data mode (e.g. two text columns/models for the deeptext   component). If multiple optimizers are used the   dictionary MUST contain an optimizer per model component.</li> </ul> <p>if no optimizers are passed it will default to <code>Adam</code> for all model components</p> </li> <li> <code>lr_schedulers</code>               (<code>Optional[Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of Pytorch's <code>LRScheduler</code> object (e.g   <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>) or</li> <li>a dictionary where there keys are the model componenst (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead') and the   values are the corresponding learning rate schedulers or list of     learning rate schedulers if multiple models are used for the given     data mode (e.g. two text columns/models for the deeptext component).</li> </ul> </li> <li> <code>initializers</code>               (<code>Optional[Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of an <code>Initializer</code> object see <code>pytorch-widedeep.initializers</code> or</li> <li>a dictionary where there keys are the model components (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead')   and the values are the corresponding initializers or list of     initializers if multiple models are used for the given data mode (e.g.     two text columns/models for the deeptext component).</li> </ul> </li> <li> <code>transforms</code>               (<code>Optional[List[Transforms]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>torchvision.transforms</code> to be applied to the image component of the model (i.e. <code>deepimage</code>) See torchvision transforms.</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. The <code>History</code> and the <code>LRShedulerCallback</code> callbacks are used by default. This can also be a custom callback as long as the object of type <code>Callback</code>. See <code>pytorch_widedeep.callbacks.Callback</code> or the examples folder in the repo.</p> </li> <li> <code>metrics</code>               (<code>Optional[Union[List[Metric], List[Metric]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as long   as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the examples folder in the   repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library   Examples.   This can also be a custom metric as long as   it is an object of type <code>Metric</code>. See   the instructions.</li> </ul> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Verbosity level. If set to 0 nothing will be printed during training</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train/test split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>lambda_sparse: <code>float</code>     lambda sparse parameter in case the <code>deeptabular</code> component is <code>TabNet</code></p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</li> </ul> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cyclic_lr</code>               (<code>bool</code>)           \u2013            <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See Pytorch schedulers.</p> </li> <li> <code>feature_importance</code>               (<code>dict</code>)           \u2013            <p>dict where the keys are the column names and the values are the corresponding feature importances. This attribute will only exist if the <code>deeptabular</code> component is a Tabnet model.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torchvision.transforms import ToTensor\n&gt;&gt;&gt;\n&gt;&gt;&gt; # wide deep imports\n&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # build the model\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set optimizers and schedulers\n&gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n&gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n&gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n&gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n&gt;&gt;&gt;\n&gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n&gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n&gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n&gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n&gt;&gt;&gt;\n&gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n&gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set initializers and callbacks\n&gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n&gt;&gt;&gt; transforms = [ToTensor]\n&gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set the trainer\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n</code></pre> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(  # noqa: C901\n    \"objective\",\n    [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n)\ndef __init__(\n    self,\n    model: WideDeep,\n    objective: str,\n    custom_loss_function: Optional[nn.Module] = None,\n    optimizers: Optional[\n        Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]\n    ] = None,\n    lr_schedulers: Optional[\n        Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]\n    ] = None,\n    initializers: Optional[\n        Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]\n    ] = None,\n    transforms: Optional[List[Transforms]] = None,\n    callbacks: Optional[List[Callback]] = None,\n    metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        objective=objective,\n        custom_loss_function=custom_loss_function,\n        optimizers=optimizers,\n        lr_schedulers=lr_schedulers,\n        initializers=initializers,\n        transforms=transforms,\n        callbacks=callbacks,\n        metrics=metrics,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.fit","title":"fit","text":"<pre><code>fit(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_train=None,\n    X_val=None,\n    val_split=None,\n    target=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    custom_dataloader=None,\n    feature_importance_sample_size=None,\n    finetune=False,\n    **kwargs\n)\n</code></pre> <p>Fit method.</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in dictionaries (<code>X_train</code> or <code>X_val</code>).</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code>. If multiple text columns/models are used, this should be a list of numpy arrays</p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code>. If multiple image columns/models are used, this should be a list of numpy arrays</p> </li> <li> <code>X_train</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The training dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> </li> <li> <code>X_val</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The validation dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>train/val split fraction</p> </li> <li> <code>target</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>target values</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> <li> <code>custom_dataloader</code>               (<code>Optional[DataLoader]</code>, default:                   <code>None</code> )           \u2013            <p>object of class <code>torch.utils.data.DataLoader</code>. Available predefined dataloaders are in <code>pytorch-widedeep.dataloaders</code>.If <code>None</code>, a standard torch <code>DataLoader</code> is used.</p> </li> <li> <code>finetune</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>fine-tune individual model components. This functionality can also be used to 'warm-up' (and hence the alias <code>warmup</code>) individual components before the joined training starts, and hence its alias. See the Examples folder in the repo for more details</p> <p><code>pytorch_widedeep</code> implements 3 fine-tune routines.</p> <ul> <li>fine-tune all trainable layers at once. This routine is   inspired by the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper. Using a   Slanted Triangular learing (see   Leslie N. Smith paper ) ,   the process is the following: i) the learning rate will   gradually increase for 10% of the training steps from max_lr/10   to max_lr. ii) It will then gradually decrease to max_lr/10   for the remaining 90% of the steps. The optimizer used in the   process is <code>Adam</code>.</li> </ul> <p>and two gradual fine-tune routines, where only certain layers are trained at a time.</p> <ul> <li>The so called <code>Felbo</code> gradual fine-tune rourine, based on the the   Felbo et al., 2017 DeepEmoji paper.</li> <li>The <code>Howard</code> routine based on the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper.</li> </ul> <p>For details on how these routines work, please see the Examples section in this documentation and the Examples folder in the repo.  Param Alias: <code>warmup</code></p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other keyword arguments are:</p> <ul> <li> <p>DataLoader related parameters:     For example,  <code>sampler</code>, <code>batch_sampler</code>, <code>collate_fn</code>, etc.     Please, see the pytorch     DataLoader docs     for details.</p> </li> <li> <p>Finetune related parameters:     see the source code at <code>pytorch_widedeep._finetune</code>. Namely, these are:</p> <ul> <li><code>finetune_epochs</code> (<code>int</code>):     number of epochs use for fine tuning</li> <li><code>finetune_max_lr</code> (<code>float</code>):    max lr during fine tuning</li> <li><code>routine</code> (<code>str</code>):    one of 'howard' or 'felbo'</li> <li><code>deeptabular_gradual</code> (<code>bool</code>):    boolean indicating if the <code>deeptabular</code> component will be fine tuned gradually</li> <li><code>deeptabular_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    List of pytorch modules indicating the layers of the    <code>deeptabular</code> that will be fine tuned</li> <li><code>deeptabular_max_lr</code> (<code>Union[float, List[float]]</code>):    max lr for the <code>deeptabular</code> componet during fine tuning</li> <li><code>deeptext_gradual</code> (<code>bool</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component</li> <li><code>deeptext_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component.    If there are multiple text columns/models, this should be a list of lists</li> <li><code>deeptext_max_lr</code> (<code>Union[float, List[float]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component    If there are multiple text columns/models, this should be a list of floats</li> <li><code>deepimage_gradual</code> (<code>bool</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_max_lr</code> (<code>Union[float, List[float]]</code>):     same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> </ul> </li> </ul> </li> </ul> <p>Examples:</p> <p>For a series of comprehensive examples on how to use the <code>fit</code> method, please see the Examples folder in the repo</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(\"finetune\", [\"warmup\"])\ndef fit(  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[np.ndarray] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    val_split: Optional[float] = None,\n    target: Optional[np.ndarray] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    custom_dataloader: Optional[DataLoader] = None,\n    feature_importance_sample_size: Optional[int] = None,\n    finetune: bool = False,\n    **kwargs,\n):\n    r\"\"\"Fit method.\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    dictionaries (`X_train` or `X_val`).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n        If multiple text columns/models are used, this should be a list of\n        numpy arrays\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n        If multiple image columns/models are used, this should be a list of\n        numpy arrays\n    X_train: Dict, Optional. default=None\n        The training dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices. Note that of multiple text or image\n        columns/models are used, the corresponding values should be lists\n        of numpy arrays\n    X_val: Dict, Optional. default=None\n        The validation dataset can also be passed in a dictionary. Keys\n        are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n        Values are the corresponding matrices. Note that of multiple text\n        or image columns/models are used, the corresponding values should\n        be lists of numpy arrays\n    val_split: float, Optional. default=None\n        train/val split fraction\n    target: np.ndarray, Optional. default=None\n        target values\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    custom_dataloader: `DataLoader`, Optional, default=None\n        object of class `torch.utils.data.DataLoader`. Available\n        predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n        `None`, a standard torch `DataLoader` is used.\n    finetune: bool, default=False\n        fine-tune individual model components. This functionality can also\n        be used to 'warm-up' (and hence the alias `warmup`) individual\n        components before the joined training starts, and hence its\n        alias. See the Examples folder in the repo for more details\n\n        `pytorch_widedeep` implements 3 fine-tune routines.\n\n        - fine-tune all trainable layers at once. This routine is\n          inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n          Slanted Triangular learing (see\n          [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n          the process is the following: *i*) the learning rate will\n          gradually increase for 10% of the training steps from max_lr/10\n          to max_lr. *ii*) It will then gradually decrease to max_lr/10\n          for the remaining 90% of the steps. The optimizer used in the\n          process is `Adam`.\n\n        and two gradual fine-tune routines, where only certain layers are\n        trained at a time.\n\n        - The so called `Felbo` gradual fine-tune rourine, based on the the\n          Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n        - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n        For details on how these routines work, please see the Examples\n        section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n        Param Alias: `warmup`\n\n    Other Parameters\n    ----------------\n    **kwargs:\n        Other keyword arguments are:\n\n        - **DataLoader related parameters**:&lt;br/&gt;\n            For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n            Please, see the pytorch\n            [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n            for details.\n\n        - **Finetune related parameters**:&lt;br/&gt;\n            see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n            - `finetune_epochs` (`int`):\n                number of epochs use for fine tuning\n            - `finetune_max_lr` (`float`):\n               max lr during fine tuning\n            - `routine` (`str`):\n               one of _'howard'_ or _'felbo'_\n            - `deeptabular_gradual` (`bool`):\n               boolean indicating if the `deeptabular` component will be fine tuned gradually\n            - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               List of pytorch modules indicating the layers of the\n               `deeptabular` that will be fine tuned\n            - `deeptabular_max_lr` (`Union[float, List[float]]`):\n               max lr for the `deeptabular` componet during fine tuning\n            - `deeptext_gradual` (`bool`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n            - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component.\n               If there are multiple text columns/models, this should be a list of lists\n            - `deeptext_max_lr` (`Union[float, List[float]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n               If there are multiple text columns/models, this should be a list of floats\n            - `deepimage_gradual` (`bool`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_max_lr` (`Union[float, List[float]]`):\n                same as `deeptext_layers` but for the `deepimage` component\n\n    Examples\n    --------\n\n    For a series of comprehensive examples on how to use the `fit` method, please see the\n    [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n    folder in the repo\n    \"\"\"\n\n    dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = wd_train_val_split(\n        self.seed,\n        self.method,  # type: ignore\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_train,\n        X_val,\n        val_split,\n        target,\n        self.transforms,\n    )\n    if custom_dataloader is not None:\n        # make sure is callable (and HAS to be an subclass of DataLoader)\n        assert isinstance(custom_dataloader, type)\n        train_loader = custom_dataloader(  # type: ignore[misc]\n            dataset=train_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            **dataloader_args,\n        )\n    else:\n        train_loader = DataLoader(\n            dataset=train_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            **dataloader_args,\n        )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    if finetune:\n        self.with_finetuning: bool = True\n        self._finetune(train_loader, **finetune_args)\n        if self.verbose:\n            print(\n                \"Fine-tuning (or warmup) of individual components completed. \"\n                \"Training the whole model for {} epochs\".format(n_epochs)\n            )\n    else:\n        self.with_finetuning = False\n\n    self.callback_container.on_train_begin(\n        {\"batch_size\": batch_size, \"train_steps\": train_steps, \"n_epochs\": n_epochs}\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (data, targett) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(data, targett, batch_idx)\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (data, targett) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(data, targett, i)\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            # self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n\n    if feature_importance_sample_size is not None:\n        self.feature_importance = FeatureImportance(\n            self.device, feature_importance_sample_size\n        ).feature_importance(train_loader, self.model)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict","title":"predict","text":"<pre><code>predict(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predictions</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <p>array with the predictions</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict(  # type: ignore[override, return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[np.ndarray] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray, Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"regression\":\n        return np.vstack(preds_l).squeeze(1)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        return (preds &gt; 0.5).astype(\"int\")\n    if self.method == \"qregression\":\n        return np.vstack(preds_l)\n    if self.method == \"multiclass\":\n        preds = np.vstack(preds_l)\n        return np.argmax(preds, 1)  # type: ignore[return-value]\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_uncertainty","title":"predict_uncertainty","text":"<pre><code>predict_uncertainty(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n    uncertainty_granularity=1000,\n)\n</code></pre> <p>Returns the predicted ucnertainty of the model for the test dataset using a Monte Carlo method during which dropout layers are activated in the evaluation/prediction phase and each sample is predicted N times (<code>uncertainty_granularity</code> times).</p> <p>This is based on Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning.</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> <li> <code>uncertainty_granularity</code>           \u2013            <p>number of times the model does prediction for each sample</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <ul> <li>if <code>method = regression</code>, it will return an array with <code>(max, min, mean, stdev)</code>   values for each sample.</li> <li>if <code>method = binary</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)</code> for each sample.</li> <li>if <code>method = multiclass</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)</code>   values for each sample.</li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_uncertainty(  # type: ignore[return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[np.ndarray] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n    uncertainty_granularity=1000,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n    using a Monte Carlo method during which dropout layers are activated\n    in the evaluation/prediction phase and each sample is predicted N\n    times (`uncertainty_granularity` times).\n\n    This is based on\n    [Dropout as a Bayesian Approximation: Representing\n    Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray, Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n    uncertainty_granularity: int default = 1000\n        number of times the model does prediction for each sample\n\n    Returns\n    -------\n    np.ndarray:\n        - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n          values for each sample.\n        - if `method = binary` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n        - if `method = multiclass` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n          values for each sample.\n\n    \"\"\"\n    preds_l = self._predict(\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_test,\n        batch_size,\n        uncertainty_granularity,\n        uncertainty=True,\n    )\n    preds = np.vstack(preds_l)\n    samples_num = int(preds.shape[0] / uncertainty_granularity)\n    if self.method == \"regression\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        return np.array(\n            (\n                preds.max(axis=0),\n                preds.min(axis=0),\n                preds.mean(axis=0),\n                preds.std(axis=0),\n            )\n        ).T\n    if self.method == \"qregression\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for qregression method\"\n        )\n    if self.method == \"binary\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        preds = preds.mean(axis=0)\n        probs = np.zeros([preds.shape[0], 3])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n        preds = preds.mean(axis=0)\n        preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n        return preds\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predicted probabilities for the test dataset for  binary and multiclass methods</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>array with the probabilities per class</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_proba(  # type: ignore[override, return]  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[np.ndarray] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n    and multiclass methods\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray, Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        probs = np.zeros([preds.shape[0], 2])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    save_optimizer=False,\n    model_filename=\"wd_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history, and the <code>feature_importance</code> attribute (if the <code>deeptabular</code> component is a Tabnet model) to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers, learning rate schedulers, initializers, etc) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>The exception is Tabnet. If the <code>deeptabular</code> component is a Tabnet model, an attribute (a dict) called <code>feature_importance</code> will be created at the end of the training process. Therefore, a <code>save</code> method was created that will save the feature importance dictionary to a json file and, since we are here, the model weights, training history and learning rate history.</p> <p>Parameters:</p> <ul> <li> <code>path</code>               (<code>str</code>)           \u2013            <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save directly the model (and optimizer) or the model's (and optimizer's) state dictionary</p> </li> <li> <code>save_optimizer</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save the optimizer</p> </li> <li> <code>model_filename</code>               (<code>str</code>, default:                   <code>'wd_model.pt'</code> )           \u2013            <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    save_optimizer: bool = False,\n    model_filename: str = \"wd_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history, and the\n    `feature_importance` attribute (if the `deeptabular` component is a\n    Tabnet model) to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers,\n    learning rate schedulers, initializers, etc) have to be defined\n    externally and then passed to the `Trainer`. As a result, the\n    `Trainer` does not generate any attribute or additional data\n    products that need to be saved other than the `model` object itself,\n    which can be saved as any other torch model (e.g. `torch.save(model,\n    path)`).\n\n    The exception is Tabnet. If the `deeptabular` component is a Tabnet\n    model, an attribute (a dict) called `feature_importance` will be\n    created at the end of the training process. Therefore, a `save`\n    method was created that will save the feature importance dictionary\n    to a json file and, since we are here, the model weights, training\n    history and learning rate history.\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model\n        (and optimizer) or the model's (and optimizer's) state\n        dictionary\n    save_optimizer: bool, default = False\n        Boolean indicating whether to save the optimizer\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    self._save_history(path)\n\n    self._save_model_and_optimizer(\n        path, save_state_dict, save_optimizer, model_filename\n    )\n\n    if self.model.is_tabnet:\n        with open(Path(path) / \"feature_importance.json\", \"w\") as fi:\n            json.dump(self.feature_importance, fi)\n</code></pre>"},{"location":"pytorch-widedeep/utils/index.html","title":"The <code>utils</code> module","text":"<p>These are a series of utilities that might be useful for a number of preprocessing tasks, even not directly related to <code>pytorch-widedeep</code>. All the classes and functions discussed here are available directly from the <code>utils</code> module. For example, the <code>LabelEncoder</code> within the <code>deeptabular_utils</code> submodule can be imported as:</p> <pre><code>from pytorch_widedeep.utils import LabelEncoder\n</code></pre> <p>These are classes and functions that are internally used in the library. We include them here in case the user finds them useful for other purposes.</p>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html","title":"deeptabular utils","text":""},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder","title":"LabelEncoder","text":"<pre><code>LabelEncoder(\n    columns_to_encode=None,\n    with_attention=False,\n    shared_embed=False,\n)\n</code></pre> <p>Label Encode categorical values for multiple columns at once</p> <p> NOTE: LabelEncoder reserves 0 for <code>unseen</code> new categories. This is convenient when defining the embedding layers, since we can just set padding idx to 0.</p> <p>Parameters:</p> <ul> <li> <code>columns_to_encode</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List of strings containing the names of the columns to encode. If <code>None</code> all columns of type <code>object</code> in the dataframe will be label encoded.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model. Aliased as <code>for_transformer</code>.</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary containing the encoding mappings in the format, e.g. :  <code>{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}</code></p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary containing the inverse encoding mappings in the format, e.g. :  <code>{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}</code></p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\ndef __init__(\n    self,\n    columns_to_encode: Optional[List[str]] = None,\n    with_attention: bool = False,\n    shared_embed: bool = False,\n):\n    self.columns_to_encode = columns_to_encode\n\n    self.shared_embed = shared_embed\n    self.with_attention = with_attention\n\n    self.reset_embed_idx = not self.with_attention or self.shared_embed\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(df)\n</code></pre> <p>Main method. Creates encoding attributes.</p> <p>Returns:</p> <ul> <li> <code>LabelEncoder</code>           \u2013            <p><code>LabelEncoder</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n    \"\"\"Main method. Creates encoding attributes.\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # here df is a chunk of the data. this is meant to be run when the\n    # data is large and we pass a chunk at a time. Therefore, we do not\n    # copy the input chunk as mutating a chunk is ok\n    if self.columns_to_encode is None:\n        self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n    else:\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:\n            df[col] = df[col].astype(\"O\")\n\n    unique_column_vals: Dict[str, List[str]] = {}\n    for c in self.columns_to_encode:\n        unique_column_vals[c] = df[c].unique().tolist()\n\n    if not hasattr(self, \"encoding_dict\"):\n        # we run the method 'partial_fit' for the 1st time\n        self.encoding_dict: Dict[str, Dict[str, int]] = {}\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n            del unique_column_vals[\"cls_token\"]\n\n        # leave 0 for padding/\"unseen\" categories. Also we need an\n        # attribute to keep track of the encoding in case we use\n        # attention and we do not re-start the index/counter\n        self.cum_idx: int = 1\n        for k, v in unique_column_vals.items():\n            self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n            self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n    else:\n        # the 'partial_fit' method has already run.\n        # \"cls_token\" will have been added already\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            del unique_column_vals[\"cls_token\"]\n\n        # Classes in the new df/chunk of the dataset that have not been seen\n        # before\n        unseen_classes: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unseen_classes[c] = list(\n                np.setdiff1d(\n                    unique_column_vals[c], list(self.encoding_dict[c].keys())\n                )\n            )\n\n        # leave 0 for padding/\"unseen\" categories\n        for k, v in unique_column_vals.items():\n            # if we use attention we need to start encoding from the\n            # last 'overall' encoding index. Otherwise, we use the max\n            # encoding index per categorical col\n            _idx = (\n                max(self.encoding_dict[k].values()) + 1\n                if self.reset_embed_idx\n                else self.cum_idx\n            )\n            if len(unseen_classes[k]) != 0:\n                for i, o in enumerate(unseen_classes[k]):\n                    if o not in self.encoding_dict[k]:\n                        self.encoding_dict[k][o] = i + _idx\n                # if self.reset_embed_idx is True it will be 1 anyway\n                self.cum_idx = (\n                    1\n                    if self.reset_embed_idx\n                    else self.cum_idx + len(unseen_classes[k])\n                )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p> <p>Returns:</p> <ul> <li> <code>LabelEncoder</code>           \u2013            <p><code>LabelEncoder</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n    \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # this is meant to be run when the data fits in memory and therefore,\n    # we do not want to mutate the original df, so we copy it\n    self.partial_fit(df.copy())\n\n    self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Label Encoded the categories in <code>columns_to_encode</code></p> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>label-encoded dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Label Encoded the categories in `columns_to_encode`\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    try:\n        self.encoding_dict\n    except AttributeError:\n        raise NotFittedError(\n            \"This LabelEncoder instance is not fitted yet. \"\n            \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n        )\n\n    df_inp = df.copy()\n    # sanity check to make sure all categorical columns are in an adequate\n    # format\n    for col in self.columns_to_encode:  # type: ignore\n        df_inp[col] = df_inp[col].astype(\"O\")\n\n    for k, v in self.encoding_dict.items():\n        df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n    return df_inp\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; encoder.fit_transform(df)\n   col1  col2\n0     1     1\n1     2     2\n2     3     3\n&gt;&gt;&gt; encoder.encoding_dict\n{'col2': {'me': 1, 'you': 2, 'him': 3}}\n</code></pre> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>label-encoded dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Combines `fit` and `transform`\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; encoder.fit_transform(df)\n       col1  col2\n    0     1     1\n    1     2     2\n    2     3     3\n    &gt;&gt;&gt; encoder.encoding_dict\n    {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(df)\n</code></pre> <p>Returns the original categories</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n&gt;&gt;&gt; encoder.inverse_transform(df_enc)\n   col1 col2\n0     1   me\n1     2  you\n2     3  him\n</code></pre> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>DataFrame with original categories</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Returns the original categories\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n    &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n       col1 col2\n    0     1   me\n    1     2  you\n    2     3  him\n\n    Returns\n    -------\n    pd.DataFrame\n        DataFrame with original categories\n    \"\"\"\n\n    if not hasattr(self, \"inverse_encoding_dict\"):\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    for k, v in self.inverse_encoding_dict.items():\n        df[k] = df[k].apply(lambda x: v[x])\n\n    return df\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html","title":"Fastai transforms","text":"<p>I directly copied and pasted part of the <code>transforms.py</code> module from the <code>fastai</code> library (from an old version). The reason to do such a thing is because <code>pytorch_widedeep</code> only needs the <code>Tokenizer</code> and the <code>Vocab</code> classes there. This way I avoid extra dependencies. Credit for all the code in the <code>fastai_transforms</code> module in this <code>pytorch-widedeep</code> package goes to Jeremy Howard and the <code>fastai</code> team. I only include the documentation here for completion, but I strongly advise the user to read the <code>fastai</code> documentation.</p>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer","title":"Tokenizer","text":"<pre><code>Tokenizer(\n    tok_func=SpacyTokenizer,\n    lang=\"en\",\n    pre_rules=None,\n    post_rules=None,\n    special_cases=None,\n    n_cpus=None,\n)\n</code></pre> <p>Class to combine a series of rules and a tokenizer function to tokenize text with multiprocessing.</p> <p>Setting some of the parameters of this class require perhaps some familiarity with the source code.</p> <p>Parameters:</p> <ul> <li> <code>tok_func</code>               (<code>Callable</code>, default:                   <code>SpacyTokenizer</code> )           \u2013            <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p> </li> <li> <code>lang</code>               (<code>str</code>, default:                   <code>'en'</code> )           \u2013            <p>Text's Language</p> </li> <li> <code>pre_rules</code>               (<code>Optional[ListRules]</code>, default:                   <code>None</code> )           \u2013            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the text (str) directly as <code>rule(tok)</code> before being tokenized.</p> </li> <li> <code>post_rules</code>               (<code>Optional[ListRules]</code>, default:                   <code>None</code> )           \u2013            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the tokens as <code>rule(tokens)</code> after the text has been tokenized.</p> </li> <li> <code>special_cases</code>               (<code>Optional[Collection[str]]</code>, default:                   <code>None</code> )           \u2013            <p>special cases to be added to the tokenizer via <code>Spacy</code>'s <code>add_special_case</code> method</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def __init__(\n    self,\n    tok_func: Callable = SpacyTokenizer,\n    lang: str = \"en\",\n    pre_rules: Optional[ListRules] = None,\n    post_rules: Optional[ListRules] = None,\n    special_cases: Optional[Collection[str]] = None,\n    n_cpus: Optional[int] = None,\n):\n    self.tok_func, self.lang, self.special_cases = tok_func, lang, special_cases\n    self.pre_rules = ifnone(pre_rules, defaults.text_pre_rules)\n    self.post_rules = ifnone(post_rules, defaults.text_post_rules)\n    self.special_cases = (\n        special_cases if special_cases is not None else defaults.text_spec_tok\n    )\n    self.n_cpus = ifnone(n_cpus, defaults.cpus)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text","title":"process_text","text":"<pre><code>process_text(t, tok)\n</code></pre> <p>Process and tokenize one text <code>t</code> with tokenizer <code>tok</code>.</p> <p>Parameters:</p> <ul> <li> <code>t</code>               (<code>str</code>)           \u2013            <p>text to be processed and tokenized</p> </li> <li> <code>tok</code>               (<code>BaseTokenizer</code>)           \u2013            <p>Instance of <code>BaseTokenizer</code>. See <code>pytorch_widedeep.utils.fastai_transforms.BaseTokenizer</code></p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List of tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n    r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n    Parameters\n    ----------\n    t: str\n        text to be processed and tokenized\n    tok: ``BaseTokenizer``\n        Instance of `BaseTokenizer`. See\n        `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    for rule in self.pre_rules:\n        t = rule(t)\n    toks = tok.tokenizer(t)\n    for rule in self.post_rules:\n        toks = rule(toks)\n    return toks\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all","title":"process_all","text":"<pre><code>process_all(texts)\n</code></pre> <p>Process a list of texts. Parallel execution of <code>process_text</code>.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tok = Tokenizer()\n&gt;&gt;&gt; tok.process_all(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p> NOTE: Note the token <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> docs.</p> <p>Returns:</p> <ul> <li> <code>List[List[str]]</code>           \u2013            <p>List containing lists of tokens. One list per \"document\"</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n    r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tok = Tokenizer()\n    &gt;&gt;&gt; tok.process_all(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    :information_source: **NOTE**:\n    Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n    next word begins with a capital in the original text. For more\n    details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    List[List[str]]\n        List containing lists of tokens. One list per \"_document_\"\n\n    \"\"\"\n\n    if self.n_cpus &lt;= 1:\n        return self._process_all_1(texts)\n    with ProcessPoolExecutor(self.n_cpus) as e:\n        return sum(\n            e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []\n        )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab","title":"Vocab","text":"<pre><code>Vocab(max_vocab, min_freq, pad_idx=None)\n</code></pre> <p>Contains the correspondence between numbers and tokens.</p> <p>Parameters:</p> <ul> <li> <code>max_vocab</code>               (<code>int</code>)           \u2013            <p>maximum vocabulary size</p> </li> <li> <code>min_freq</code>               (<code>int</code>)           \u2013            <p>minimum frequency for a token to be considereds</p> </li> <li> <code>pad_idx</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index for the unknown token ('xxunk') and defaults to 1 for the padding token ('xxpad').</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>itos</code>               (<code>Collection</code>)           \u2013            <p><code>index to str</code>. Collection of strings that are the tokens of the vocabulary</p> </li> <li> <code>stoi</code>               (<code>defaultdict</code>)           \u2013            <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and their corresponding index</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def __init__(\n    self,\n    max_vocab: int,\n    min_freq: int,\n    pad_idx: Optional[int] = None,\n):\n    self.max_vocab = max_vocab\n    self.min_freq = min_freq\n    self.pad_idx = pad_idx\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.create","title":"create","text":"<pre><code>create(tokens)\n</code></pre> <p>Create a vocabulary object from a set of tokens.</p> <p>Parameters:</p> <ul> <li> <code>tokens</code>               (<code>Tokens</code>)           \u2013            <p>Custom type: <code>Collection[Collection[str]]</code>  see <code>pytorch_widedeep.wdtypes</code>. Collection of collection of strings (e.g. list of tokenized sentences)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n&gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n&gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n[10, 11, 9, 12]\n&gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n'machine learning is great'\n</code></pre> <p> NOTE: Note the many special tokens that <code>fastai</code>'s' tokenizer adds. These are particularly useful when building Language models and/or in classification/Regression tasks. Please see the <code>fastai</code> docs.</p> <p>Returns:</p> <ul> <li> <code>Vocab</code>           \u2013            <p>An instance of a <code>Vocab</code> object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def create(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    r\"\"\"Create a vocabulary object from a set of tokens.\n\n    Parameters\n    ----------\n    tokens: Tokens\n        Custom type: ``Collection[Collection[str]]``  see\n        `pytorch_widedeep.wdtypes`. Collection of collection of\n        strings (e.g. list of tokenized sentences)\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n    &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n    &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n    [10, 11, 9, 12]\n    &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n    'machine learning is great'\n\n    :information_source: **NOTE**:\n    Note the many special tokens that ``fastai``'s' tokenizer adds. These\n    are particularly useful when building Language models and/or in\n    classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    Vocab\n        An instance of a `Vocab` object\n    \"\"\"\n\n    freq = Counter(p for o in tokens for p in o)\n    itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n    for o in reversed(defaults.text_spec_tok):\n        if o in itos:\n            itos.remove(o)\n        itos.insert(0, o)\n\n    if self.pad_idx is not None and self.pad_idx != 1:\n        itos.remove(PAD)\n        itos.insert(self.pad_idx, PAD)\n        # get the new 'xxunk' index\n        xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n    else:\n        xxunk_idx = 0\n\n    itos = itos[: self.max_vocab]\n    if (\n        len(itos) &lt; self.max_vocab\n    ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n        while len(itos) % 8 != 0:\n            itos.append(\"xxfake\")\n\n    self.itos = itos\n    self.stoi = defaultdict(\n        lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n    )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.fit","title":"fit","text":"<pre><code>fit(tokens)\n</code></pre> <p>Calls the <code>create</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a fit method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def fit(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    \"\"\"\n    Calls the `create` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including a fit method\n    \"\"\"\n    return self.create(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize","title":"numericalize","text":"<pre><code>numericalize(t)\n</code></pre> <p>Convert a list of tokens <code>t</code> to their ids.</p> <p>Returns:</p> <ul> <li> <code>List[int]</code>           \u2013            <p>List of 'numericalsed' tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def numericalize(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"Convert a list of tokens ``t`` to their ids.\n\n    Returns\n    -------\n    List[int]\n        List of '_numericalsed_' tokens\n    \"\"\"\n    return [self.stoi[w] for w in t]\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.transform","title":"transform","text":"<pre><code>transform(t)\n</code></pre> <p>Calls the <code>numericalize</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def transform(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"\n    Calls the `numericalize` method. I simply want to honor fast ai naming,\n    but for consistency with the rest of the library I am including a\n    transform method\n    \"\"\"\n    return self.numericalize(t)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.textify","title":"textify","text":"<pre><code>textify(nums, sep=' ')\n</code></pre> <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List of tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n    \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    return (\n        sep.join([self.itos[i] for i in nums])\n        if sep is not None\n        else [self.itos[i] for i in nums]\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(nums, sep=' ')\n</code></pre> <p>Calls the <code>textify</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including an inverse_transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def inverse_transform(\n    self, nums: Collection[int], sep=\" \"\n) -&gt; Union[str, List[str]]:\n    \"\"\"\n    Calls the `textify` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including an\n    inverse_transform method\n    \"\"\"\n    # I simply want to honor fast ai naming, but for consistency with the\n    # rest of the library I am including an inverse_transform method\n    return self.textify(nums, sep)\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html","title":"Image utils","text":"<p>SimplePreprocessor and AspectAwarePreprocessor are directly taked from the great series of Books `Deep Learning for Computer Vision by Adrian. Therefore, all credit for the code in the <code>image_utils</code> module goes to Adrian Rosebrock.</p>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor","title":"AspectAwarePreprocessor","text":"<pre><code>AspectAwarePreprocessor(\n    width, height, inter=cv2.INTER_AREA\n)\n</code></pre> <p>Class to resize an image to a certain width and height taking into account the image aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>width</code>               (<code>int</code>)           \u2013            <p>output width</p> </li> <li> <code>height</code>               (<code>int</code>)           \u2013            <p>output height</p> </li> <li> <code>inter</code>           \u2013            <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n    self.width = width\n    self.height = height\n    self.inter = inter\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image taking into account the image aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>image</code>               (<code>ndarray</code>)           \u2013            <p>Input image to be resized</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import cv2\n&gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n&gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n&gt;&gt;&gt; img.shape\n(694, 890, 3)\n&gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n&gt;&gt;&gt; resized_img = app.preprocess(img)\n&gt;&gt;&gt; resized_img.shape\n(224, 224, 3)\n</code></pre> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized image according to its original image aspect ratio</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Examples\n    --------\n    &gt;&gt;&gt; import cv2\n    &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n    &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n    &gt;&gt;&gt; img.shape\n    (694, 890, 3)\n    &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n    &gt;&gt;&gt; resized_img = app.preprocess(img)\n    &gt;&gt;&gt; resized_img.shape\n    (224, 224, 3)\n\n    Returns\n    -------\n    np.ndarray\n        Resized image according to its original image aspect ratio\n    \"\"\"\n    (h, w) = image.shape[:2]\n    dW = 0\n    dH = 0\n\n    if w &lt; h:\n        image = imutils.resize(image, width=self.width, inter=self.inter)\n        dH = int((image.shape[0] - self.height) / 2.0)\n    else:\n        image = imutils.resize(image, height=self.height, inter=self.inter)\n        dW = int((image.shape[1] - self.width) / 2.0)\n\n    (h, w) = image.shape[:2]\n    image = image[dH : h - dH, dW : w - dW]\n\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor","title":"SimplePreprocessor","text":"<pre><code>SimplePreprocessor(width, height, inter=cv2.INTER_AREA)\n</code></pre> <p>Class to resize an image to a certain width and height</p> <p>Parameters:</p> <ul> <li> <code>width</code>               (<code>int</code>)           \u2013            <p>output width</p> </li> <li> <code>height</code>               (<code>int</code>)           \u2013            <p>output height</p> </li> <li> <code>inter</code>           \u2013            <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n    self.width = width\n    self.height = height\n    self.inter = inter\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image</p> <p>Parameters:</p> <ul> <li> <code>image</code>               (<code>ndarray</code>)           \u2013            <p>Input image to be resized</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized image</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Returns\n    -------\n    np.ndarray\n        Resized image\n\n    \"\"\"\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html","title":"Text utils","text":"<p>Collection of helper function that facilitate processing text.</p>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.simple_preprocess","title":"simple_preprocess","text":"<pre><code>simple_preprocess(\n    doc, lower=False, deacc=False, min_len=2, max_len=15\n)\n</code></pre> <p>This is <code>Gensim</code>'s <code>simple_preprocess</code> with a <code>lower</code> param to indicate wether or not to lower case all the token in the doc</p> <p>For more information see: <code>Gensim</code> utils module</p> <p>Parameters:</p> <ul> <li> <code>doc</code>               (<code>str</code>)           \u2013            <p>Input document.</p> </li> <li> <code>lower</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Lower case tokens in the input doc</p> </li> <li> <code>deacc</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p> </li> <li> <code>min_len</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p> </li> <li> <code>max_len</code>               (<code>int</code>, default:                   <code>15</code> )           \u2013            <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n&gt;&gt;&gt; simple_preprocess('Machine learning is great')\n['Machine', 'learning', 'is', 'great']\n</code></pre> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List with the processed tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def simple_preprocess(\n    doc: str,\n    lower: bool = False,\n    deacc: bool = False,\n    min_len: int = 2,\n    max_len: int = 15,\n) -&gt; List[str]:\n    r\"\"\"\n    This is `Gensim`'s `simple_preprocess` with a `lower` param to\n    indicate wether or not to lower case all the token in the doc\n\n    For more information see: `Gensim` [utils module](https://radimrehurek.com/gensim/utils.html)\n\n    Parameters\n    ----------\n    doc: str\n        Input document.\n    lower: bool, default = False\n        Lower case tokens in the input doc\n    deacc: bool, default = False\n        Remove accent marks from tokens using `Gensim`'s `deaccent`\n    min_len: int, default = 2\n        Minimum length of token (inclusive). Shorter tokens are discarded.\n    max_len: int, default = 15\n        Maximum length of token in result (inclusive). Longer tokens are discarded.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n    &gt;&gt;&gt; simple_preprocess('Machine learning is great')\n    ['Machine', 'learning', 'is', 'great']\n\n    Returns\n    -------\n    List[str]\n        List with the processed tokens\n    \"\"\"\n    tokens = [\n        token\n        for token in tokenize(doc, lower=lower, deacc=deacc, errors=\"ignore\")\n        if min_len &lt;= len(token) &lt;= max_len and not token.startswith(\"_\")\n    ]\n    return tokens\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.get_texts","title":"get_texts","text":"<pre><code>get_texts(texts, already_processed=False, n_cpus=None)\n</code></pre> <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a series of very convenients things during the tokenization process</p> <p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p> <p>Parameters:</p> <ul> <li> <code>texts</code>               (<code>List[str]</code>)           \u2013            <p>List of str with the texts (or documents). One str per document</p> </li> <li> <code>already_processed</code>               (<code>Optional[bool]</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the text is already processed and we simply want to tokenize it. This parameter is thought for those cases where the input sequences might not be text (but IDs, or anything else) and we just want to tokenize it</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; get_texts(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p>Returns:</p> <ul> <li> <code>List[List[str]]</code>           \u2013            <p>List of lists, one list per 'document' containing its corresponding tokens</p> </li> <li> <code>information_source: **NOTE**:</code>           \u2013            </li> <li> <code>`get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</code>           \u2013            </li> <li> <code>Such tokenizer uses a series of convenient processing steps, including</code>           \u2013            </li> <li> <code>the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</code>           \u2013            </li> <li> <code>indicate the next word begins with a capital in the original text. For more</code>           \u2013            </li> <li> <code>details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</code>           \u2013            </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def get_texts(\n    texts: List[str],\n    already_processed: Optional[bool] = False,\n    n_cpus: Optional[int] = None,\n) -&gt; List[List[str]]:\n    r\"\"\"Tokenization using `Fastai`'s `Tokenizer` because it does a\n    series of very convenients things during the tokenization process\n\n    See `pytorch_widedeep.utils.fastai_utils.Tokenizer`\n\n    Parameters\n    ----------\n    texts: List\n        List of str with the texts (or documents). One str per document\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the text is already processed and we simply want\n        to tokenize it. This parameter is thought for those cases where the\n        input sequences might not be text (but IDs, or anything else) and we\n        just want to tokenize it\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; get_texts(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    Returns\n    -------\n    List[List[str]]\n        List of lists, one list per '_document_' containing its corresponding tokens\n\n    :information_source: **NOTE**:\n    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.\n    Such tokenizer uses a series of convenient processing steps, including\n    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to\n    indicate the next word begins with a capital in the original text. For more\n    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)\n    \"\"\"\n\n    num_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    if not already_processed:\n        processed_texts = [\" \".join(simple_preprocess(t)) for t in texts]\n    else:\n        processed_texts = texts\n    tok = Tokenizer(n_cpus=num_cpus).process_all(processed_texts)\n    return tok\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.pad_sequences","title":"pad_sequences","text":"<pre><code>pad_sequences(seq, maxlen, pad_first=True, pad_idx=1)\n</code></pre> <p>Given a List of tokenized and <code>numericalised</code> sequences it will return padded sequences according to the input parameters.</p> <p>Parameters:</p> <ul> <li> <code>seq</code>               (<code>List[int]</code>)           \u2013            <p>List of int with the <code>numericalised</code> tokens</p> </li> <li> <code>maxlen</code>               (<code>int</code>)           \u2013            <p>Maximum length of the padded sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n&gt;&gt;&gt; seq = [1,2,3]\n&gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\narray([0, 0, 1, 2, 3], dtype=int32)\n</code></pre> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>numpy array with the padded sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def pad_sequences(\n    seq: List[int], maxlen: int, pad_first: bool = True, pad_idx: int = 1\n) -&gt; np.ndarray:\n    r\"\"\"\n    Given a List of tokenized and `numericalised` sequences it will return\n    padded sequences according to the input parameters.\n\n    Parameters\n    ----------\n    seq: List\n        List of int with the `numericalised` tokens\n    maxlen: int\n        Maximum length of the padded sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n    &gt;&gt;&gt; seq = [1,2,3]\n    &gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\n    array([0, 0, 1, 2, 3], dtype=int32)\n\n    Returns\n    -------\n    np.ndarray\n        numpy array with the padded sequences\n    \"\"\"\n    if len(seq) == 0:\n        return np.zeros(maxlen, dtype=\"int32\") + pad_idx\n    elif len(seq) &gt;= maxlen:\n        res = np.array(seq[-maxlen:]).astype(\"int32\")\n        return res\n    else:\n        res = np.zeros(maxlen, dtype=\"int32\") + pad_idx\n        if pad_first:\n            res[-len(seq) :] = seq\n        else:\n            res[: len(seq) :] = seq\n        return res\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.build_embeddings_matrix","title":"build_embeddings_matrix","text":"<pre><code>build_embeddings_matrix(\n    vocab, word_vectors_path, min_freq, verbose=1\n)\n</code></pre> <p>Build the embedding matrix using pretrained word vectors.</p> <p>Returns pretrained word embeddings. If a word in our vocabulary is not among the pretrained embeddings it will be assigned the mean pretrained word-embeddings vector</p> <p>Parameters:</p> <ul> <li> <code>vocab</code>               (<code>Union[Vocab, ChunkVocab]</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p> </li> <li> <code>word_vectors_path</code>               (<code>str</code>)           \u2013            <p>path to the pretrained word embeddings</p> </li> <li> <code>min_freq</code>               (<code>int</code>)           \u2013            <p>minimum frequency required for a word to be in the vocabulary</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>level of verbosity. Set to 0 for no verbosity</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Pretrained word embeddings</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def build_embeddings_matrix(\n    vocab: Union[Vocab, ChunkVocab],\n    word_vectors_path: str,\n    min_freq: int,\n    verbose: int = 1,\n) -&gt; np.ndarray:  # pragma: no cover\n    r\"\"\"Build the embedding matrix using pretrained word vectors.\n\n    Returns pretrained word embeddings. If a word in our vocabulary is not\n    among the pretrained embeddings it will be assigned the mean pretrained\n    word-embeddings vector\n\n    Parameters\n    ----------\n    vocab: Vocab\n        see `pytorch_widedeep.utils.fastai_utils.Vocab`\n    word_vectors_path: str\n        path to the pretrained word embeddings\n    min_freq: int\n        minimum frequency required for a word to be in the vocabulary\n    verbose: int,  default=1\n        level of verbosity. Set to 0 for no verbosity\n\n    Returns\n    -------\n    np.ndarray\n        Pretrained word embeddings\n    \"\"\"\n    if not os.path.isfile(word_vectors_path):\n        raise FileNotFoundError(\"{} not found\".format(word_vectors_path))\n    if verbose:\n        print(\"Indexing word vectors...\")\n\n    embeddings_index = {}\n    f = open(word_vectors_path)\n    for line in f:\n        values = line.split()\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype=\"float32\")\n        embeddings_index[word] = coefs\n    f.close()\n\n    if verbose:\n        print(\"Loaded {} word vectors\".format(len(embeddings_index)))\n        print(\"Preparing embeddings matrix...\")\n\n    mean_word_vector = np.mean(list(embeddings_index.values()), axis=0)  # type: ignore[arg-type]\n    embedding_dim = len(list(embeddings_index.values())[0])\n    num_words = len(vocab.itos)\n    embedding_matrix = np.zeros((num_words, embedding_dim))\n    found_words = 0\n    for i, word in enumerate(vocab.itos):\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n            found_words += 1\n        else:\n            embedding_matrix[i] = mean_word_vector\n\n    if verbose:\n        print(\n            \"{} words in the vocabulary had {} vectors and appear more than {} times\".format(\n                found_words, word_vectors_path, min_freq\n            )\n        )\n\n    return embedding_matrix.astype(\"float32\")\n</code></pre>"}]}
\ No newline at end of file
diff --git a/mkdocs/site/sitemap.xml b/mkdocs/site/sitemap.xml
index d2da60fa..41bfb97a 100644
--- a/mkdocs/site/sitemap.xml
+++ b/mkdocs/site/sitemap.xml
@@ -2,222 +2,217 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/index.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/contributing.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/installation.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/quick_start.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/01_preprocessors_and_utils.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/02_model_components.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/03_binary_classification_with_defaults.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/04_regression_with_images_and_text.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/05_save_and_load_model_and_artifacts.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/06_finetune_and_warmup.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/07_custom_components.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/08_custom_dataLoader_imbalanced_dataset.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/09_extracting_embeddings.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/10_3rd_party_integration-RayTune_WnB.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/11_auc_multiclass.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/13_model_uncertainty_prediction.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/14_bayesian_models.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
-         <loc>https://pytorch-widedeep.readthedocs.io/examples/15_DIR-LDS_and_FDS.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <loc>https://pytorch-widedeep.readthedocs.io/examples/15_Self_Supervised_Pretraning_pt1.html</loc>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
-         <loc>https://pytorch-widedeep.readthedocs.io/examples/16_Self_Supervised_Pretraning_pt1.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <loc>https://pytorch-widedeep.readthedocs.io/examples/15_Self_Supervised_Pretraning_pt2.html</loc>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
-         <loc>https://pytorch-widedeep.readthedocs.io/examples/16_Self_Supervised_Pretraning_pt2.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <loc>https://pytorch-widedeep.readthedocs.io/examples/16_Usign_a_custom_hugging_face_model.html</loc>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
-         <loc>https://pytorch-widedeep.readthedocs.io/examples/17_Usign_a_custom_hugging_face_model.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <loc>https://pytorch-widedeep.readthedocs.io/examples/17_feature_importance_via_attention_weights.html</loc>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
-         <loc>https://pytorch-widedeep.readthedocs.io/examples/18_feature_importance_via_attention_weights.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <loc>https://pytorch-widedeep.readthedocs.io/examples/18_wide_and_deep_for_recsys_pt1.html</loc>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
-         <loc>https://pytorch-widedeep.readthedocs.io/examples/19_wide_and_deep_for_recsys_pt1.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <loc>https://pytorch-widedeep.readthedocs.io/examples/18_wide_and_deep_for_recsys_pt2.html</loc>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
-         <loc>https://pytorch-widedeep.readthedocs.io/examples/19_wide_and_deep_for_recsys_pt2.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <loc>https://pytorch-widedeep.readthedocs.io/examples/19_load_from_folder_functionality.html</loc>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
-         <loc>https://pytorch-widedeep.readthedocs.io/examples/20_load_from_folder_functionality.html</loc>
-         <lastmod>2024-06-15</lastmod>
-         <changefreq>daily</changefreq>
-    </url>
-    <url>
-         <loc>https://pytorch-widedeep.readthedocs.io/examples/21_Using_huggingface_within_widedeep.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <loc>https://pytorch-widedeep.readthedocs.io/examples/20_Using_huggingface_within_widedeep.html</loc>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/bayesian_models.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/bayesian_trainer.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/callbacks.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/dataloaders.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/load_from_folder.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/losses.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/metrics.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/model_components.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/preprocessing.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/self_supervised_pretraining.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/tab2vec.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/trainer.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/index.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/deeptabular_utils.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/fastai_transforms.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/image_utils.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/text_utils.html</loc>
-         <lastmod>2024-06-15</lastmod>
+         <lastmod>2024-08-18</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/mkdocs/site/sitemap.xml.gz b/mkdocs/site/sitemap.xml.gz
index 1e78a779..a77c868a 100644
Binary files a/mkdocs/site/sitemap.xml.gz and b/mkdocs/site/sitemap.xml.gz differ
diff --git a/mkdocs/sources/examples/15_DIR-LDS_and_FDS.ipynb b/mkdocs/sources/examples/15_DIR-LDS_and_FDS.ipynb
deleted file mode 100644
index 09770d99..00000000
--- a/mkdocs/sources/examples/15_DIR-LDS_and_FDS.ipynb
+++ /dev/null
@@ -1,847 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "01a4f48f",
-   "metadata": {},
-   "source": [
-    "# Label and Feature Distribution Smoothing for Deep Imbalanced Regression "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "92c81bca",
-   "metadata": {},
-   "source": [
-    "## Initial imports"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "2e7b9854",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "import torch\n",
-    "from torch.optim import SGD, lr_scheduler\n",
-    "\n",
-    "from pytorch_widedeep import Trainer\n",
-    "from pytorch_widedeep.preprocessing import TabPreprocessor\n",
-    "from pytorch_widedeep.models import TabMlp, WideDeep\n",
-    "from sklearn.metrics import mean_squared_error\n",
-    "from pytorch_widedeep.initializers import XavierNormal\n",
-    "from pytorch_widedeep.datasets import load_california_housing\n",
-    "\n",
-    "from sklearn.model_selection import train_test_split\n",
-    "\n",
-    "import matplotlib.pyplot as plt\n",
-    "from scipy.ndimage import convolve1d\n",
-    "from scipy.ndimage import gaussian_filter1d\n",
-    "from scipy.signal.windows import triang\n",
-    "from pytorch_widedeep.utils.deeptabular_utils import get_kernel_window, find_bin\n",
-    "from pytorch_widedeep.models import fds_layer\n",
-    "\n",
-    "# increase displayed columns in jupyter notebook\n",
-    "pd.set_option(\"display.max_columns\", 200)\n",
-    "pd.set_option(\"display.max_rows\", 300)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "3c0fc233",
-   "metadata": {},
-   "source": [
-    "# Load dataset"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "934cca51",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>MedInc</th>\n",
-       "      <th>HouseAge</th>\n",
-       "      <th>AveRooms</th>\n",
-       "      <th>AveBedrms</th>\n",
-       "      <th>Population</th>\n",
-       "      <th>AveOccup</th>\n",
-       "      <th>Latitude</th>\n",
-       "      <th>Longitude</th>\n",
-       "      <th>MedHouseVal</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>8.3252</td>\n",
-       "      <td>41.0</td>\n",
-       "      <td>6.984127</td>\n",
-       "      <td>1.023810</td>\n",
-       "      <td>322.0</td>\n",
-       "      <td>2.555556</td>\n",
-       "      <td>37.88</td>\n",
-       "      <td>-122.23</td>\n",
-       "      <td>4.526</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>8.3014</td>\n",
-       "      <td>21.0</td>\n",
-       "      <td>6.238137</td>\n",
-       "      <td>0.971880</td>\n",
-       "      <td>2401.0</td>\n",
-       "      <td>2.109842</td>\n",
-       "      <td>37.86</td>\n",
-       "      <td>-122.22</td>\n",
-       "      <td>3.585</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>7.2574</td>\n",
-       "      <td>52.0</td>\n",
-       "      <td>8.288136</td>\n",
-       "      <td>1.073446</td>\n",
-       "      <td>496.0</td>\n",
-       "      <td>2.802260</td>\n",
-       "      <td>37.85</td>\n",
-       "      <td>-122.24</td>\n",
-       "      <td>3.521</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>5.6431</td>\n",
-       "      <td>52.0</td>\n",
-       "      <td>5.817352</td>\n",
-       "      <td>1.073059</td>\n",
-       "      <td>558.0</td>\n",
-       "      <td>2.547945</td>\n",
-       "      <td>37.85</td>\n",
-       "      <td>-122.25</td>\n",
-       "      <td>3.413</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>3.8462</td>\n",
-       "      <td>52.0</td>\n",
-       "      <td>6.281853</td>\n",
-       "      <td>1.081081</td>\n",
-       "      <td>565.0</td>\n",
-       "      <td>2.181467</td>\n",
-       "      <td>37.85</td>\n",
-       "      <td>-122.25</td>\n",
-       "      <td>3.422</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "   MedInc  HouseAge  AveRooms  AveBedrms  Population  AveOccup  Latitude  \\\n",
-       "0  8.3252      41.0  6.984127   1.023810       322.0  2.555556     37.88   \n",
-       "1  8.3014      21.0  6.238137   0.971880      2401.0  2.109842     37.86   \n",
-       "2  7.2574      52.0  8.288136   1.073446       496.0  2.802260     37.85   \n",
-       "3  5.6431      52.0  5.817352   1.073059       558.0  2.547945     37.85   \n",
-       "4  3.8462      52.0  6.281853   1.081081       565.0  2.181467     37.85   \n",
-       "\n",
-       "   Longitude  MedHouseVal  \n",
-       "0    -122.23        4.526  \n",
-       "1    -122.22        3.585  \n",
-       "2    -122.24        3.521  \n",
-       "3    -122.25        3.413  \n",
-       "4    -122.25        3.422  "
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df = load_california_housing(as_frame=True)\n",
-    "df.head()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "753c111d",
-   "metadata": {},
-   "source": [
-    "# Effects of ks and sigma paramaters on kernel function"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "38837284",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[<matplotlib.lines.Line2D at 0x16a0a43a0>]"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    },
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGdCAYAAADqsoKGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcPUlEQVR4nO3deViU5cIG8HtmYGbYF5FVZBEVc0W2NEtLFM06erKOmqVSaostRpnynS/tfJ4OuBwz07RjxyVbtE7ZLmok2oKCIOaKoijIKiLDJgzMvN8fY9MhNwaBZ2a4f9c115XDM+/cb+Mwt++87/PIJEmSQERERGTh5KIDEBEREbUFlhoiIiKyCiw1REREZBVYaoiIiMgqsNQQERGRVWCpISIiIqvAUkNERERWgaWGiIiIrIKN6AAdRa/Xo6ioCE5OTpDJZKLjEBERUQtIkoTq6mr4+vpCLr/5sZhOU2qKiorg7+8vOgYRERG1QkFBAbp163bTMZ2m1Dg5OQEw/E9xdnYWnIaIiIhaoqqqCv7+/sbP8ZvpNKXmt6+cnJ2dWWqIiIgsTEtOHeGJwkRERGQVWGqIiIjIKrDUEBERkVVgqSEiIiKrwFJDREREVoGlhoiIiKwCSw0RERFZBZYaIiIisgosNURERGQVWGqIiIjIKrDUEBERkVVgqSEiIiKrwFJDRBYvt6wG7/14FjUNTaKjEJFAnWaVbiKyTmcv1uCRdb/gcl0j9uSUYeOMKCht+O81os6I73wisljlNQ2YsTEDl+saAQA/517Cgs9/hSRJgpMRkQgsNURkka5odXhy80HkV9Shm5sdlj8yEAq5DJ9nFeLN3adExyMiAfj1ExFZHJ1ewgtbD+FwQSVc7GyxKS4KIZ6OaNLpseDzI1j1Qy58Xe0wOaq76KhE1IF4pIaILIokSfjb18ew+3gplDZyvDc9AiGejgCAyVHd8fx9IQCAv35xFKk5ZSKjElEHY6khIouy/sezeD/tPABgxV8GIjLQvdnP40f1wkNhftDpJcz5MAtHCzUiYhKRACw1RGQxvj5chH98dxIA8Nf7++CBAb7XjJHJZEiaOAB3hXRBrVaHuE0ZuHC5rqOjEpEALDVEZBHS8yrw8ieHAQAzhgZi5t1BNxyrtJFj7WPh6O3lhIvVhiukNFevkCIi68VSQ0RmL7esGrPePwitTo/Rd3jhtQfugEwmu+ljnNW22BgXCW9nNXLLajB7y0E0NOk6KDERicBSQ0Rmray6HtM3ZEBzpRFh3V3x1uQwKOQ3LzS/8XW1w8a4SDiqbHAgrwLzPv0Vej3nsCGyViw1RGS2ahua8MSmDBRWXkFAF3u8Ny0CdkqFSdvo4+OMtY8Nho1chq8OF2HZrpx2SktEorHUEJFZatLp8dxHWThaWAV3ByU2xUWhi6OqVdu6u2dXJD7UHwCwNvUMPth/vi2jEpGZYKkhIrMjSRJe+/IY9uRchOrqXDRBHg63tc1HIvzxUkwvAMDCL4/i++OlbRGViMwISw0RmZ13Us/g4/R8yGTAW5PDMLi7W5ts94WRIZgU4Q+9BDz/sWFGYiKyHiw1RGRWvjhUiGU7Dee9LHrgDozp591m25bJZPj7n/vhnl5dcaVRhyc3ZyD/EuewIbIWLDVEZDZ+yS3HvP8Y5qKZOSwIM+668Vw0rWWrkOOdqYNxh48zymu0mLExHZdrtW3+PETU8VhqiMgs5JRU46kPMtGokzCuvw/+5/4+7fZcjiobbIyLhJ+rHc6W12LW+wdR38g5bIgsHUsNEQlXWlWPuI3pqK5vQkSAG/75l4GQt3AumtbyclZjY1wknNQ2OHj+Ml7+5DDnsCGycK0qNWvWrEFgYCDUajWio6ORnp5+w7Hr16/H3XffDTc3N7i5uSEmJuaa8Z9//jlGjx6NLl26QCaTITs7+5rtjBgxAjKZrNnt6aefbk18IjIj1fWNmLExA0WaegR3dcD6aRFQ25o2F01r9fJywruPh8NWIcO3R4rxj+9OdMjzElH7MLnUbNu2DfHx8Vi0aBGysrIwcOBAxMbGoqys7LrjU1NTMWXKFOzZswdpaWnw9/fH6NGjUVhYaBxTW1uLYcOGYcmSJTd97lmzZqG4uNh4W7p0qanxiciMNOr0ePbDLJworoKHoxKb46Lg5qDs0AxDe3hg+SMDAQDv/ZSHjT/ndejzE1HbkUmSZNLx1ujoaERGRmL16tUAAL1eD39/fzz//PNYsGDBLR+v0+ng5uaG1atXY9q0ac1+du7cOQQFBeHQoUMYNGhQs5+NGDECgwYNwsqVK02Ja1RVVQUXFxdoNBo4Ozu3ahtE1HYkScKr//kVn2ZegJ2tAtueuhMDurkKy/NOai6WJudAJgPWTg1v06uuiKj1TPn8NulIjVarRWZmJmJiYn7fgFyOmJgYpKWltWgbdXV1aGxshLu7uylPDQD48MMP4eHhgX79+iEhIQF1dTe+FLOhoQFVVVXNbkRkPt5KOY1PMy9ALgNWPxomtNAAwDPDe+DR6O6QJODFrYeQef6y0DxEZDqTSk15eTl0Oh28vLya3e/l5YWSkpIWbWP+/Pnw9fVtVoxa4tFHH8UHH3yAPXv2ICEhAVu2bMFjjz12w/GJiYlwcXEx3vz9/U16PiJqP58eLMDK708DAP5vfD+M7ON1i0e0P5lMhv/7U1+MDPVEQ5MeMzdnIK+8VnQsIjJBh179lJSUhK1bt2L79u1Qq9UmPXb27NmIjY1F//79MXXqVLz//vvYvn07zpw5c93xCQkJ0Gg0xltBQUFb7AIR3aZ9py4i4fMjAIBnRvTAY3cGCE70OxuFHG8/Gob+fi64XNeIGRvTcammQXQsImohk0qNh4cHFAoFSkubr5lSWloKb++bf/+8fPlyJCUlYdeuXRgwYIDpSf8gOjoaAJCbm3vdn6tUKjg7Oze7EZFYx4uq8OyHWWjSS/jTQF/MG91bdKRr2Ctt8O8ZEejmZofzl+rw5OaDuKLlHDZElsCkUqNUKhEeHo6UlBTjfXq9HikpKRgyZMgNH7d06VIsXrwYycnJiIiIaH3a//LbZd8+Pj5tsj0ial9FlVcQtykdNQ1NuDPYHcseGdDuc9G0lqeTGpviouBiZ4vsgkq8sPUQdJzDhsjsmfz1U3x8PNavX4/NmzfjxIkTeOaZZ1BbW4u4uDgAwLRp05CQkGAcv2TJErz22mvYsGEDAgMDUVJSgpKSEtTU1BjHVFRUIDs7G8ePHwcA5OTkIDs723iezpkzZ7B48WJkZmbi3Llz+OqrrzBt2jTcc889bXLUh4jal+aK4auc0qoG9PR0xLuPRUBl0zFz0bRWiKcj3pseAaWNHLuPl+L/vj4GEy8WJaIOZnKpmTRpEpYvX46FCxdi0KBByM7ORnJysvHk4fz8fBQXFxvHr127FlqtFg8//DB8fHyMt+XLlxvHfPXVVwgLC8O4ceMAAJMnT0ZYWBjWrVsHwHCE6Pvvv8fo0aMRGhqKl19+GRMnTsTXX399WztPRO1P26TH01sycaq0Bl2dVNgYFwkXe1vRsVokMtAdK/5imMNmc9p5vPcj57AhMmcmz1NjqThPDVHHkyQJ8Z8cxvZDhXBQKrDtqSHo5+ciOpbJ1u87izeuzja8+tEwPDDAV3Aios6j3eapISIyxT93ncL2Q4VQyGVYM3WwRRYaAJh5dxBmDA0EAMRvO4z0vAqxgYjoulhqiKhdfHQgH6v3GK5O/Mef+2FEb0/BiVpPJpPhtQfuwOg7vKDV6THr/YPILau59QOJqEOx1BBRm9tzsgyvfXkUAPDCfSGYFNldcKLbp5DL8NbkMIR1dzWe+FxWXS86FhH9F5YaImpTRy5oMOejLOj0Eh4a7IeXRvUSHanN2CkVeG9aBAK62OPC5St4ctNB1GmbRMcioqtYaoiozRRU1CFuUwbqtDoMC/FA0kMDIJOZ51w0rdXFUYVNcVFwd1DiSKEGz310CE06vehYRASWGiJqI5V1WszYmI7ymgaEejvhnccGQ2ljnb9igjwc8N70CKhs5PjhZBkWfsU5bIjMgXX+xiGiDtXQpMPsLZk4c7EW3s5qbIyLhLPaMuaiaa3B3d3w1uQwyGSGk6LX7r3+OnRE1HFYaojotuj1El7+xHCZs5PKBhvjIuHjYic6VocY088bCx+4AwCwNDkHX2YXCk5E1Lmx1BDRbVmSfBLf/FoMG7kM6x4PRx+fzjW5ZdxdQZg5LAgA8Mqnh/HLmXLBiYg6L5YaImq199PO4d19ZwEASyYOwF0hHoITifE/9/fBuP4+aNRJeGpLJk6VVouORNQpsdQQUavsPl6K1786BgB4eVQvTAzvJjiROHK5DP/8y0BEBLihur4JMzako7SKc9gQdTSWGiIy2aH8y3j+4yzoJWBypD+euy9EdCTh1LYKrJ8WgeCuDijS1CNuYwZqGjiHDVFHYqkhIpOcv1SLmZsPor5Rj+G9umLxhH5WNxdNa7k5KLE5LgoejkocL67Csx9moZFz2BB1GJYaImqxilotZmzMwKVaLfr6OmPN1MGwVfDXyH/zd7fHhhmRsLNVYN+pi/jr9iOcw4aog/C3ERG1SH2jDjM3ZyCvvBZ+rnbYOCMSjiob0bHM0oBurlj9aBjkMuCTgxewKiVXdCSiToGlhohuSaeXMHdrNrLyK+GstsGmuEh4OqtFxzJrI/t44f/G9wMAvPn9Kfwn84LgRETWj6WGiG7pjW9PIPlYCZQKOf41LQI9vZxER7IIj90ZgGdG9AAALPjsV/x4+qLgRETWjaWGiG7q3z/lYcPPeQCAZY8MwJ3BXQQnsizzRvfGnwb6okkv4ZkPsnC8qEp0JCKrxVJDRDe040gx/v7tcQDA/DGhGD/IT3AiyyOXy66WQXfUNDThiU0ZKNZcER2LyCqx1BDRdWWer8DcbdmQJOCxO7vj6eHBoiNZLJWNAu8+FoGeno4oqarHjA0ZqKpvFB2LyOqw1BDRNc5erMHMzQfR0KTHyFBPvP5gX85Fc5tc7G2xMS4SXZ1UyCmtxjMfZELbxDlsiNoSSw0RNVNe04AZGzNwua4RA7u54O1Hw2DDuWjaRDc3e2ycEQkHpQI/517Cgs9+5Rw2RG2Iv6mIyKhO24QnNx9EfkUd/N3t8N70SNgrORdNW+rn54I1UwdDIZfh80OFWLH7lOhIRFaDpYaIABjmonnh42wcLqiEq70tNsVFoauTSnQsqzSityf+8WfDHDZv/5CLj9PzBScisg4sNUQESZLw+lfH8P2JUiht5HhvWgR6dHUUHcuqTYrsjheuLgT6v18cxZ6cMsGJiCwfSw0R4V/7zmLL/vOQyYCVkwYhItBddKRO4aVRvfDQYD/o9BLmfJiFo4Ua0ZGILBpLDVEn99XhIiTuOAkA+Ov9fXB/fx/BiToPmUyGpIcGYFiIB+q0OsRtykBBRZ3oWEQWi6WGqBPbf/YSXvnkMABgxtBAPDksSHCizkdpI8c7jw1GqLcTLlY3IG5TBjR1nMOGqDVYaog6qdyyasx+/yC0Oj1i+3rhtQfu4Fw0gjirDXPYeDurkVtWg1lbDqKhSSc6FpHFYakh6oTKquoxfUMGquqbENbdFW9NDoNCzkIjko+LHTbGRcJJZYP0vAq88umv0Os5hw2RKVhqiDqZ2oYmPLE5A4WVVxDYxR7vTYuA2lYhOhYB6OPjjHWPh8NGLsPXh4uwZOdJ0ZGILApLDVEn0qTTY85HWThaWAV3ByU2xUWhiyPnojEnd4V4YMnEAQCAd/eexZa0c2IDEVkQlhqiTkKSJLz25VGk5lyE2laO96ZHINDDQXQsuo6J4d0QP6oXAGDRV8ew+3ip4EREloGlhqiTWLMnFx+nF0AmA96aHIbB3d1ER6KbeP6+EEyO9IdeAp7/OAvZBZWiIxGZPZYaok5g+6ELWL7LsMbQ6w/2RWxfb8GJ6FZkMhkWT+iH4b26or5Rjyc3ZSD/EuewIboZlhoiK/dLbjle/c+vAIDZ9wRj+tBAsYGoxWwVcqyZOhh9fZ1xqVaLGRvTcblWKzoWkdliqSGyYidLqvDUlkw06iSMG+CDBWNCRUciEzmqbLBxRiT8XO1wtrwWM98/iPpGzmFDdD0sNURWqkRTj7iNGahuaEJUoDv++chAyDkXjUXydFZjU1wknNU2yDx/GS9ty+YcNkTXwVJDZIWq6xsxY2M6ijX1CO7qgH9NC+dcNBaup5cT/jUtAkqFHDuOluCN706IjkRkdlhqiKxMo06PZz/MwsmSang4qrA5Lgqu9krRsagN3BncBcseMcxh8++f8rDhpzzBiYjMC0sNkRWRJAkJnx/Bj6fLYWerwIYZEfB3txcdi9rQ+EF+mH/13KjF3x5H8tFiwYmIzAdLDZEVWfn9afwn8wLkMmDN1DAM6OYqOhK1g6eHB+OxO7tDkoAXt2Yj83yF6EhEZoGlhshKfJJRgLdSTgMAFk/oh/tCvQQnovYik8nw+oN9MTLUEw1NeszcfBBnL9aIjkUkHEsNkRXYe+oiErYfAQA8O6IHpkYHCE5E7c1GIcfbj4ZhYDcXXK5rxIyNGSivaRAdi0ioVpWaNWvWIDAwEGq1GtHR0UhPT7/h2PXr1+Puu++Gm5sb3NzcEBMTc834zz//HKNHj0aXLl0gk8mQnZ19zXbq6+sxZ84cdOnSBY6Ojpg4cSJKS7keCtGxIg2e/SATOr2E8YN8MS+2t+hI1EHslTZ4b3ok/N3tkF9Rhyc3H8QVLeewoc7L5FKzbds2xMfHY9GiRcjKysLAgQMRGxuLsrKy645PTU3FlClTsGfPHqSlpcHf3x+jR49GYWGhcUxtbS2GDRuGJUuW3PB5X3rpJXz99df49NNPsXfvXhQVFeGhhx4yNT6RVSmsvIK4jRmo1epwZ7A7lj48ADIZ56LpTLo6qbApLgqu9rY4XFCJF7Yego5z2FAnJZMkyaS//dHR0YiMjMTq1asBAHq9Hv7+/nj++eexYMGCWz5ep9PBzc0Nq1evxrRp05r97Ny5cwgKCsKhQ4cwaNAg4/0ajQZdu3bFRx99hIcffhgAcPLkSfTp0wdpaWm48847b/m8VVVVcHFxgUajgbOzswl7TGSeNFca8ci6X3CqtAa9vBzx6dND4WJnKzoWCXLwXAUefe8AtE16TBsSgL/9qS8LLlkFUz6/TTpSo9VqkZmZiZiYmN83IJcjJiYGaWlpLdpGXV0dGhsb4e7u3uLnzczMRGNjY7PnDQ0NRffu3W/4vA0NDaiqqmp2I7IWDU06PL0lE6dKa+DppMLGuCgWmk4uItAdKycNgkwGvJ92Hut/PCs6ElGHM6nUlJeXQ6fTwcur+VUVXl5eKCkpadE25s+fD19f32YF5VZKSkqgVCrh6ura4udNTEyEi4uL8ebv79/i5yMyZ5IkYf5/fkXa2UtwUCqwMc6wLhDR/f198Nf7+wAA/vHdSXx9uEhwIqKO1aFXPyUlJWHr1q3Yvn071Gp1uz5XQkICNBqN8VZQUNCuz0fUUZbvysEX2UVQyGV457Fw9PV1ER2JzMiTw4Iw4+pK7C9/chjpeZzDhjoPk0qNh4cHFArFNVcdlZaWwtvb+6aPXb58OZKSkrBr1y4MGDDApJDe3t7QarWorKxs8fOqVCo4Ozs3uxFZuo8O5GPNnjMAgMQ/98fwXl0FJyJzI5PJ8NoDdyC2rxe0Oj1mvX8QuWXVomMRdQiTSo1SqUR4eDhSUlKM9+n1eqSkpGDIkCE3fNzSpUuxePFiJCcnIyIiwuSQ4eHhsLW1bfa8OTk5yM/Pv+nzElmTH06W4n+/MMxF88LInvhLJL9SpetTyGV4a3IYwrq7QnOlEdM3ZKCsul50LKJ2Z2PqA+Lj4zF9+nREREQgKioKK1euRG1tLeLi4gAA06ZNg5+fHxITEwEAS5YswcKFC/HRRx8hMDDQeA6Mo6MjHB0dAQAVFRXIz89HUZHh+9+cnBwAhiM03t7ecHFxwZNPPon4+Hi4u7vD2dkZzz//PIYMGdKiK5+ILN2vFyox58ND0EvAw+Hd8FJMT9GRyMypbRV4b1oEJq79Becu1eGJTRnYNnsIHFQm/9onshgmn1MzadIkLF++HAsXLsSgQYOQnZ2N5ORk48nD+fn5KC7+fYG1tWvXQqvV4uGHH4aPj4/xtnz5cuOYr776CmFhYRg3bhwAYPLkyQgLC8O6deuMY95880088MADmDhxIu655x54e3vj888/b/WOE1mKggrDB9KVRh3u7umBxIf681JdapEujoY5bNwdlDhaWIXnPspCk04vOhZRuzF5nhpLxXlqyBJV1mnx0NpfcPZiLUK9nfDp00PgpOal22SarPzLeHT9ftQ36jElqjv+8ed+LMZkMdptnhoi6jj1jTrMev8gzl6shY+LGpviolhoqFUGd3fDW5PDIJMBH6fn453UM6IjEbULlhoiM6TXS3j508PIOHcZTiobbIyLhLdL+06DQNYttq83Xn+wLwBg2c4cbD90QXAiorbHUkNkhpKST+LbX4thq5Dh3cfDEerNr0zp9k0fGojZ9wQDAF79z6/4JbdccCKitsVSQ2RmNv9yDv/aZ5jifsnEARga4iE4EVmTBWNCMW6ADxp1Ep7akomcEs5hQ9aDpYbIjOw6VoLXvz4GAHhldC88NLib4ERkbeRyGf75yEBEBbqjuqEJMzamo0TDOWzIOrDUEJmJQ/mX8cLWQ5AkYEqUP+bcGyI6Elkpta0C/5oWjuCuDijW1CNuUwaq6xtFxyK6bSw1RGbgXHktntx8EPWNeozo3RWLx/OSW2pfrvZKbI6LgoejCieKq/Dsh1lo5Bw2ZOFYaogEq6jVYsbGdFTUatHPzxlrHh0MGwXfmtT+/N3tsWFGBOxsFfjxdDn+5/Mj6CRTl5GV4m9OIoHqG3WYuTkD5y7Vwc/VDhtmRHIae+pQA7q5Ys3UMMhlwKeZF/BWymnRkYhajaWGSBCdXsKLWw8hK78SzmobbH4iEp5OnIuGOt59oV5YPKEfAGDl96fxycECwYmIWoelhkiQv397HDuPlUKpkGP9tAiEeDqJjkSd2NToADw7ogcA4H8+P4J9py4KTkRkOpYaIgHe+/EsNv58DgCw/C8DER3cRWwgIgDzYntj/CBfNOklPPthFo4XVYmORGQSlhqiDvbdkWK88d0JAMCCsaH400BfwYmIDGQyGZY+PAB3BrujpqEJcZvSUVR5RXQsohZjqSHqQAfPVWDutmxIEvD4nQF46uqU9UTmQmWjwLuPR6CXlyNKqxowY2M6NFc4hw1ZBpYaog5y5mINZr5/ENomPWL6eOL1P/XlXDRkllzsbLExLgqeTiqcKq3B01syoW3iHDZk/lhqiDrAxWrDv3gr6xoxsJsLVk0Jg0LOQkPmy8/VDhvjIuGgVCDt7CXM/+xXzmFDZo+lhqid1WmbMHNzBgoqrqC7uz3+PSMS9krORUPmr6+vC955LBwKuQzbDxXin7tOiY5EdFMsNUTtqEmnxwsfH8LhCxq42ttiU1wkPBxVomMRtdjwXl2R+Of+AIDVe3Lx0YF8wYmIboylhqidSJKE178+hu9PlEFpI8d70yIQ3NVRdCwik/0l0h8vjOwJAHjty6PYc7JMcCKi62OpIWon7+47iw/250MmA96aNAgRge6iIxG12ksxPfFweDfo9BLmfJSFIxc0oiMRXYOlhqgdfJldiKQdJwEA/zvuDozt7yM4EdHtkclkSHyoP+7u6YE6rQ5xmzJQUFEnOhZRMyw1RG1s/9lLmPfprwCAuLsC8eSwIMGJiNqGrUKOd6YORqi3E8prfruiTys6FpERSw1RGzpdWo3Z7x+EVqfHmL7e+N9xd4iORNSmnNS22BQXBR8XNc5crMXsLZloaNKJjkUEgKWGqM2UVdVjxsYMVNU3YXB3V6ycPIhz0ZBV8nZRY2NcJJxUNkjPq8DLnxyGXs85bEg8lhqiNmBYJycDhZVXEOThgPemR0JtqxAdi6jdhHo7Y93j4bBVyPDNr8VYknxSdCQilhqi29Wo02POh1k4VlSFLg5KbIqLhLuDUnQsonZ3V4gHlkwcAMBwtd/7aefEBqJOj6WG6DZIkoTXvjiKvacuQm0rx79nRCKgi4PoWEQd5qHB3fDK6F4AgNe/OoZdx0oEJ6LOjKWG6Das/iEXWzMKIJcBb08ZjEH+rqIjEXW4OfeGYEqUP/QS8MLWQziUf1l0JOqkWGqIWumzzAv4527DWjiv/6kvRt3hJTgRkRgymQyLx/fDiN5dUd+ox8zNB3H+Uq3oWNQJsdQQtcLPueWY/5lhLpqn7gnGtCGBYgMRCWajkGPNo4PRz88Zl2q1mLExAxW1nMOGOhZLDZGJTpZU4ektmWjSS3hggA/mjwkVHYnILDiobLBhRiT8XO2QV16LmZszUN/IOWyo47DUEJmgWHMFMzZkoLqhCVFB7lj+yEDIORcNkZGnkxqbn4iEs9oGWfmVmLs1GzrOYUMdhKWGqIWq6hsRtzEDJVX16NHVAf96PJxz0RBdR4inE9ZPi4BSIUfysRK88e0J0ZGok2CpIWoBbZMez36QhZMl1fBwVGFTXBRc7TkXDdGNRAd3wfK/DAQAbPg5D//+KU9wIuoMWGqIbkGSJCz4/Ff8lFsOe6UCG2dEwt/dXnQsIrP3p4G+WDDWcM7Z3789jh1HigUnImvHUkN0C29+fxqfZxVCIZdhzaOD0b+bi+hIRBbjqXuC8fidAZAk4MVt2cg8XyE6Elkxlhqim9iWkY9VKacBAIvH98O9oZ6CExFZFplMhtf/1BcxfTyhbTLMYXP2Yo3oWGSlWGqIbiA1pwz/s/0oAGDOvT3waHR3wYmILJNCLsOqKWEY2M0Fl+saMWNjBsprGkTHIivEUkN0HUcLNZjzYRZ0egl/DvPDK6N7i45EZNHslTb494xIdHe3R35FHZ7clIE6bZPoWGRlWGqI/qCw8gqe2JSBWq0OQ4K7YMnEAZDJOBcN0e0yXDkYCVd7Wxy+oMELH3MOG2pbLDVE/0VT14gZG9JRVt2AXl6OWPd4OJQ2fJsQtZXgro54b1oElDZyfH+iFK9/dQySxGJDbYO/rYmuamjS4akPDuJ0WQ28nA1z0bjY2YqORWR1IgLd8dakQZDJgC37z+Nf+86KjkRWgqWGCIBeL+HV//yK/Wcr4KiywcYZUfB1tRMdi8hqje3vg/8ddwcAIHHHSXx1uEhwIrIGrSo1a9asQWBgINRqNaKjo5Genn7DsevXr8fdd98NNzc3uLm5ISYm5prxkiRh4cKF8PHxgZ2dHWJiYnD69OlmYwIDAyGTyZrdkpKSWhOf6BrLduXgy+wi2MhleGfqYNzh6yw6EpHVe3JYEOLuCgQAvPLJYew/e0lsILJ4Jpeabdu2IT4+HosWLUJWVhYGDhyI2NhYlJWVXXd8amoqpkyZgj179iAtLQ3+/v4YPXo0CgsLjWOWLl2KVatWYd26dThw4AAcHBwQGxuL+vr6Ztv6v//7PxQXFxtvzz//vKnxia6x40gx1qaeAQAkPtQf9/TqKjgRUefxv+PuwJi+3tDq9HhqSyYv9abbIpNMPEMrOjoakZGRWL16NQBAr9fD398fzz//PBYsWHDLx+t0Ori5uWH16tWYNm0aJEmCr68vXn75ZbzyyisAAI1GAy8vL2zatAmTJ08GYDhSM3fuXMydO9fEXTSoqqqCi4sLNBoNnJ35r3Ay0DbpEbNiL/Ir6vDUPcFIuL+P6EhEnU59ow4PvfMLjhdXYfqQAPxtfD/RkciMmPL5bdKRGq1Wi8zMTMTExPy+AbkcMTExSEtLa9E26urq0NjYCHd3dwBAXl4eSkpKmm3TxcUF0dHR12wzKSkJXbp0QVhYGJYtW4amJs5xQLfnwwPnkV9RBw9HFV4Y2VN0HKJOSW2rwP+OM/yD4sMD+ThXXis4EVkqG1MGl5eXQ6fTwcvLq9n9Xl5eOHnyZIu2MX/+fPj6+hpLTElJiXEbf9zmbz8DgBdeeAGDBw+Gu7s7fvnlFyQkJKC4uBgrVqy47vM0NDSgoeH3w5hVVVUtykedR3V9I97+IRcAMDemJxxUJr0diKgNDQ3xwPBeXbH31EUs25WDNY8OFh2JLFCH/hZPSkrC1q1bkZqaCrVabdJj4+Pjjf89YMAAKJVKPPXUU0hMTIRKpbpmfGJiIv72t7/ddmayXu/uPYuKWi2CPRwwKdJfdByiTm/B2FDsO30R3/5ajNl3V2Kgv6voSGRhTPr6ycPDAwqFAqWlpc3uLy0thbe3900fu3z5ciQlJWHXrl0YMGCA8f7fHmfqNqOjo9HU1IRz585d9+cJCQnQaDTGW0FBwU3zUedSWlWP934yzI3x6pjesFVwdgMi0fr4OOPPYX4AgMQdJzgpH5nMpN/kSqUS4eHhSElJMd6n1+uRkpKCIUOG3PBxS5cuxeLFi5GcnIyIiIhmPwsKCoK3t3ezbVZVVeHAgQM33WZ2djbkcjk8Pa+/arJKpYKzs3OzG9FvVn5/CvWNegzu7orYvjcv5ETUcV4e3RtKGzn2n61Aas5F0XHIwpj89VN8fDymT5+OiIgIREVFYeXKlaitrUVcXBwAYNq0afDz80NiYiIAYMmSJVi4cCE++ugjBAYGGs+TcXR0hKOjI2QyGebOnYu///3v6NmzJ4KCgvDaa6/B19cXEyZMAACkpaXhwIEDuPfee+Hk5IS0tDS89NJLeOyxx+Dm5tZG/yuos8gtq8a2DMORu4T7+3BdJyIz4udqh7ihgXh331kk7TiJe3p1hULO9yi1jMmlZtKkSbh48SIWLlyIkpISDBo0CMnJycYTffPz8yGX/34AaO3atdBqtXj44YebbWfRokV4/fXXAQCvvvoqamtrMXv2bFRWVmLYsGFITk42nnejUqmwdetWvP7662hoaEBQUBBeeumlZufZELXUkuQc6CVg1B1eiAx0Fx2HiP7g2REh2JpRgJzSanyedQGPRPCcN2oZk+epsVScp4YA4OC5Cjy8Lg1yGbDrpXsQ4ukkOhIRXce/9p3BP747CR8XNfa8MgJqW4XoSCRIu81TQ2TJJEnCP747AQCYFOnPQkNkxqYNCYSfqx2KNfXY9Ms50XHIQrDUUKex81gpsvIrobaVY25ML9FxiOgm1LYKxI8yvE/f2ZOLyjqt4ERkCVhqqFNo0umxdKdhgsiZw4Lh5WzaPElE1PEmhPkh1NsJVfVNWLMnV3QcsgAsNdQpbDtYgLMXa+HuoMRTw4NFxyGiFlDIZVgwNhQAsPmX87hwuU5wIjJ3LDVk9eq0TVj5/WkAwPP3hcBJbSs4ERG11PBeXTG0RxdodXqs2HVKdBwycyw1ZPXe+zEPF6sb0N3dHlOjA0THISITyGQyJIw1LHa5PbsQx4u4jh/dGEsNWbXymga8u/cMAOCVWMNMpURkWfp3c8GDA30hSUBScssWT6bOib/hyaq9nXIatVod+vu54IH+PqLjEFErzRvdG7YKGfaduoifTpeLjkNmiqWGrNa58lp8eCAfAJAwNhRyTrVOZLG6d/n96+PEHSeg13eKeWPJRCw1ZLWW7cxBk14ynGgY4iE6DhHdpufvC4GjygbHiqrw9a9FouOQGWKpIauUXVCJb48UQyaD8ZJQIrJsXRxVePrqlAzLduagoUknOBGZG5YasjqSJCHx6nIIfw7zQx8frvVFZC2eGBYETycVLly+gg/254uOQ2aGpYaszp6cMhzIq4DSRo6XR/cWHYeI2pC90gYvXV0+YfUPp1FV3yg4EZkTlhqyKjq9hCU7cgAAM4YaFsQjIuvySHg39OjqgMt1jViXekZ0HDIjLDVkVT7LuoCc0mo4q23w7IgeouMQUTuwUcgxf4zhXLkNP+ehRFMvOBGZC5Yashr1jTq8udswjfqce0Pgaq8UnIiI2suoO7wQEeCG+ka98X1PxFJDVmPjz+dQrKmHr4sa04cGio5DRO1IJpMh4X7D0ZpPMwtwurRacCIyByw1ZBUu12rxTmouACB+dG+obRWCExFRewsPcEdsXy/oJWAJl08gsNSQlVizJxfV9U0I9XbCn8P8RMchog7y6phQKOQyfH+iDOl5FaLjkGAsNWTxCirq8H7aeQDA/LGGX3BE1Dn06OqISZH+AAzLJ0gSl0/ozFhqyOKt2H0KWp0eQ4K7YESvrqLjEFEHmzuyJ+xsFTiUX4nkoyWi45BALDVk0Y4VafBFdiEAIOH+UMhkPEpD1Nl4Oqsx6+4gAMDSnTlo1OkFJyJRWGrIoiXtOAlJAh4Y4IMB3VxFxyEiQWYP74EuDkrklddia0aB6DgkCEsNWayfTpfjx9PlsFXIMC+WyyEQdWaOKhu8MLInAOCt70+jtqFJcCISgaWGLJJeLyFxh2HRyqnRAQjo4iA4ERGJNiWqOwK62KO8pgHrfzwrOg4JwFJDFunrX4twrKgKjiobPH9fiOg4RGQGlDZyvBprmJDvX/vO4mJ1g+BE1NFYasjiNDTpsGynYdHKp4cHo4ujSnAiIjIX9/f3xkB/V9RpdViVclp0HOpgLDVkcT7Yn48Ll6/A00mFJ4YFiY5DRGZEJpMhYazhaM3H6fnIK68VnIg6EksNWZSq+kas/sHwr6+XRvWCvdJGcCIiMjd3BnfBfaGeaNJLWLaTyyd0Jiw1ZFHWpZ7B5bpG9OjqgEfCu4mOQ0Rmav6YUMhlwHdHSnAo/7LoONRBWGrIYpRo6rHh5zwAhl9YNgr+9SWi6+vt7YSJgw3/8EnccZLLJ3QS/FQgi/Hm7lOob9QjIsANo+7wEh2HiMxc/OheUNnIkZ5XgR9OlomOQx2ApYYswunSanyaaZgllMshEFFL+LjYIe4uw8UESTtOoonLJ1g9lhqyCEuST0IvAbF9vRAe4C46DhFZiGdG9ICrvS1Ol9Xgs6wLouNQO2OpIbOXnleB70+UQSGX4dUxoaLjEJEFcbGzxXP3GiboXLH7FK5odYITUXtiqSGzJkm/L4cwKdIfPbo6Ck5ERJbm8SEB8HO1Q2lVg/FiA7JOLDVk1pKPluBQfiXsbBWYe3WxOiIiU6hsFHglthcAw7QQFbVawYmovbDUkNlq1Omx9OpyCLPuDoKns1pwIiKyVOMH+uEOH2dUNzRh9Q+5ouNQO2GpIbO19eoU510clJg9vIfoOERkweRyGRZcXT5hy/5zKKioE5yI2gNLDZmlmoYmvHV1MboXRvaEo4rLIRDR7bmnV1cMC/FAo07C8l05ouNQO2CpIbO0ft9ZlNdoEdDFHlOiuouOQ0RW4rejNV9mF+FooUZwGmprLDVkdsqq67H+x7MAgHmxvaG04V9TImob/fxcMH6QLwDDhHxkXfhpQWZnVcpp1Gl1GNjNBeP6+4iOQ0RW5pXRvaFUyPFTbjn2nbooOg61IZYaMitnL9bg43TDcggLxvbhcghE1Ob83e3x2J0BAAxHa/R6LnZpLVpVatasWYPAwECo1WpER0cjPT39hmPXr1+Pu+++G25ubnBzc0NMTMw14yVJwsKFC+Hj4wM7OzvExMTg9OnTzcZUVFRg6tSpcHZ2hqurK5588knU1NS0Jj6ZsWU7c6DTS7i3d1cM6dFFdBwislLP3RcCJ5UNjhdX4cvDhaLjUBsxudRs27YN8fHxWLRoEbKysjBw4EDExsairOz6K6CmpqZiypQp2LNnD9LS0uDv74/Ro0ejsPD3v0RLly7FqlWrsG7dOhw4cAAODg6IjY1FfX29cczUqVNx7Ngx7N69G9988w327duH2bNnt2KXyVxl5V/GjqMlkMmA+WO5HAIRtR93ByWeHmGYKmL5zlOob+TyCVZBMlFUVJQ0Z84c4591Op3k6+srJSYmtujxTU1NkpOTk7R582ZJkiRJr9dL3t7e0rJly4xjKisrJZVKJX388ceSJEnS8ePHJQBSRkaGccyOHTskmUwmFRYWtuh5NRqNBEDSaDQtGk8dS6/XS4+s/UUKmP+N9PIn2aLjEFEnUNfQJEW/8b0UMP8baf2+M6Lj0A2Y8vlt0pEarVaLzMxMxMTEGO+Ty+WIiYlBWlpai7ZRV1eHxsZGuLsbVlrOy8tDSUlJs226uLggOjrauM20tDS4uroiIiLCOCYmJgZyuRwHDhy47vM0NDSgqqqq2Y3MV8qJMqSfq4DKRo74Ub1ExyGiTsBOqcBLowzLr6zekwvNlUbBieh2mVRqysvLodPp4OXl1ex+Ly8vlJSUtGgb8+fPh6+vr7HE/Pa4m22zpKQEnp6ezX5uY2MDd3f3Gz5vYmIiXFxcjDd/f/8W5aOO16TTY0my4dLKGXcFwtfVTnAiIuosJg7uhp6ejqisa8Ta1DOi49Bt6tCrn5KSkrB161Zs374danX7ruOTkJAAjUZjvBUUFLTr81HrfZZ1AafLauBiZ4tnh4eIjkNEnYiNQo75Ywzn8G38OQ9FlVcEJ6LbYVKp8fDwgEKhQGlpabP7S0tL4e3tfdPHLl++HElJSdi1axcGDBhgvP+3x91sm97e3teciNzU1ISKioobPq9KpYKzs3OzG5mfK1odVuw+BQB47t4QuNjbCk5ERJ3NyD6eiAp0R0OTHm9e/X1ElsmkUqNUKhEeHo6UlBTjfXq9HikpKRgyZMgNH7d06VIsXrwYycnJzc6LAYCgoCB4e3s322ZVVRUOHDhg3OaQIUNQWVmJzMxM45gffvgBer0e0dHRpuwCmZkNP+ehtKoBfq52eHxIgOg4RNQJyWQyLLjfcLTms6wLyCmpFpyIWsvkr5/i4+Oxfv16bN68GSdOnMAzzzyD2tpaxMXFAQCmTZuGhIQE4/glS5bgtddew4YNGxAYGIiSkhKUlJQY55iRyWSYO3cu/v73v+Orr77CkSNHMG3aNPj6+mLChAkAgD59+mDMmDGYNWsW0tPT8fPPP+O5557D5MmT4evr2wb/G0iEilot1l39Dvvl0b2gtlUITkREndXg7m4Y288begnGc/zI8pi89PGkSZNw8eJFLFy4ECUlJRg0aBCSk5ONJ/rm5+dDLv+9K61duxZarRYPP/xws+0sWrQIr7/+OgDg1VdfRW1tLWbPno3KykoMGzYMycnJzc67+fDDD/Hcc89h5MiRkMvlmDhxIlatWtWafSYzsfqHXFQ3NKGPjzMmDPITHYeIOrl5sb2x+3gpfjhZhv1nL+HOYE4AamlkkiR1ivmhq6qq4OLiAo1Gw/NrzEBBRR3u+2cqGnUS3n8iCvf06io6EhERXvviKLbsP4+B/q744tmhXKrFDJjy+c21n0iI5bty0KiTMCzEg4WGiMzGCyN7wkGpwOGCSnx3pGVTlZD5YKmhDne0UIMvs4sAAAu4HAIRmZGuTirMuicYALBs50lom/SCE5EpWGqowyXtMJyEN36QL/r5uQhOQ0TU3Ky7g+HhqMK5S3X4OD1fdBwyAUsNdah9py7ip9xyKBVyvDK6t+g4RETXcFDZ4MUYw/IJq1JOo7qeyydYCpYa6jB6vWQ8SvPYnQHwd7cXnIiI6PomR/oj2MMBl2q1WL/vrOg41EIsNdRhvjxciOPFVXBS2eC5+7gcAhGZL1uFHPNiDUeT1/+Yh7KqesGJqCVYaqhD1DfqsHynYfrxp0f0gLuDUnAiIqKbG9PPG2HdXXGlUYeVKadFx6EWYKmhDvHB/vMorLwCb2c1nrgrSHQcIqJbkslkSBjbBwCwLaMAZy7WCE5Et8JSQ+1Oc6URq/fkAgBeGtUTdkouh0BEliEqyB0xfTyh00tYyuUTzB5LDbW7talnUFnXiJ6ejpg4uJvoOEREJpk/JhRyGbDzWCkyz1eIjkM3wVJD7aqo8go2/pwHwPCLwUbBv3JEZFl6ejnhkXB/AEDidyfRSVYXskj8hKF2tWL3KTQ06REV6I6RfTxFxyEiapWXRvWC2laOg+cvY/fxUtFx6AZYaqjdnCypwmdZFwAAC+4P5cJwRGSxvF1+v8hhSfJJNOm4fII5YqmhdrNkx0lIEjC2nzcGd3cTHYeI6LY8PaIH3OxtceZiLT7NvCA6Dl0HSw21i7Qzl7An5yIUcplxAisiIkvmrLbFc/cZlk94c/cp1GmbBCeiP2KpoTYnSRKSdpwAAEyJ8kdwV0fBiYiI2sZjd3ZHNzc7lFU3YMNPeaLj0B+w1FCb+/ZIMQ5f0MBeqcCLI3uJjkNE1GZUNgrj0ed1e8/iUk2D4ET031hqqE1pm/RYtjMHADDr7mB0dVIJTkRE1LYeHOCLfn7OqGlowts/5IqOQ/+FpYba1Mfp+Th/qQ4ejkrMuidYdBwiojYnl8uwYIxh+YQPD5xH/qU6wYnoNyw11Gaq6xux6uqiby+O7AlHlY3gRERE7WNYTw/c3dMDjToJy3bliI5DV7HUUJtZv+8sLtVqEeThgMlR3UXHISJqVwvGhkImA74+XIRfL1SKjkNgqaE2UlZVj/U/Gq4EmBfbG7ZcDoGIrFxfXxdMGOQHAEjaweUTzAE/eahNrEw5jSuNOgzyd8XYft6i4xARdYj4Ub2gVMjxy5lL2Hvqoug4nR5LDd22MxdrsC2jAACQMJbLIRBR5+Hvbo9pQwIAGI7W6PQ8WiMSSw3dtqXJhjfyyFBPRAd3ER2HiKhDzbk3BE5qG5wsqcYXhwpFx+nUWGrotmSer8DOY6WQy4D5Y0NFxyEi6nBuDko8OyIEALBi9ynUN+oEJ+q8WGqo1SRJQuJ3JwEAj4T7o5eXk+BERERixN0VCB8XNQorr+D9tHOi43RaLDXUaruPl+Lg+ctQ28rx0iguh0BEnZfaVoH4q78HV/+Qi8o6reBEnRNLDbVKk06PJcmGozRP3BUEbxe14ERERGI9NLgbQr2dUFXfhHdSz4iO0ymx1FCrfJp5AWcu1sLN3hZPj+ghOg4RkXAKuQzzxxjOLdz0yzkUVl4RnKjzYakhk9Vpm/Dm7lMAgOfu6wlnta3gRERE5mFE7664M9gd2iY9/snlEzocSw2ZbMNPeSirbkA3Nzs8dieXQyAi+o1MJkPCWMNil9sPFeJ4UZXgRJ0LSw2Z5FJNA9btPQvAsByCykYhOBERkXkZ6O+KcQN8IEkwnntIHYOlhkzy9g+5qGloQj8/Zzw4wFd0HCIiszRvdG/YyGXYe+oifsktFx2n02CpoRbLv1SHDw+cBwAsGNMHcjmXQyAiup5ADwdMjTZ8PZ+44yT0XD6hQ7DUUIst25WDRp2Eu3t6YFhPD9FxiIjM2vMje8JBqcCRQg2+OVIsOk6nwFJDLfLrhUp8fbgIMhmwgMshEBHdkoejCk8NN0x5sXxnDrRNesGJrB9LDd2SJElI2mE42W3CID/09XURnIiIyDLMvDsIXZ1UyK/4/et7aj8sNXRLe09dxC9nLkGpkBunAScioluzV9pgbkxPAIYLLarrGwUnsm4sNXRTOv3vR2mmDQmAv7u94ERERJZlUoQ/grs6oKJWi3evTolB7YOlhm5q+6FCnCyphpPaBnPuDREdh4jI4tgo5Hg11nAu4ns/nUVpVb3gRNaLpYZuqL5RhxVXp/l+dkQI3ByUghMREVmm2L5eCA9wQ32jHiu/PyU6jtViqaEb2vzLORRp6uHjokbcXYGi4xARWSzD8gmGozXbMgqQW1YtOJF1Yqmh66qs02LNnlwAwEujekFty+UQiIhuR0SgO0bd4QW9BCxJ5mKX7aFVpWbNmjUIDAyEWq1GdHQ00tPTbzj22LFjmDhxIgIDAyGTybBy5cprxlRXV2Pu3LkICAiAnZ0dhg4dioyMjGZjZsyYAZlM1uw2ZsyY1sSnFngn9Qyq6pvQ28sJEwd3Ex2HiMgqzB/TG3IZsPt4KQ6eqxAdx+qYXGq2bduG+Ph4LFq0CFlZWRg4cCBiY2NRVlZ23fF1dXUIDg5GUlISvL29rztm5syZ2L17N7Zs2YIjR45g9OjRiImJQWFhYbNxY8aMQXFxsfH28ccfmxqfWqCw8go2/XIOADB/bG8ouBwCEVGbCPF0wqRIfwDAP747AUni8gltyeRSs2LFCsyaNQtxcXG44447sG7dOtjb22PDhg3XHR8ZGYlly5Zh8uTJUKlU1/z8ypUr+Oyzz7B06VLcc889CAkJweuvv46QkBCsXbu22ViVSgVvb2/jzc3NzdT41AL/3GWY+TI6yB339vYUHYeIyKrMjekFta0cWfmV2HmsVHQcq2JSqdFqtcjMzERMTMzvG5DLERMTg7S0tFYFaGpqgk6ng1qtbna/nZ0dfvrpp2b3paamwtPTE71798YzzzyDS5cu3XC7DQ0NqKqqanajWzteVIXthwxHyBLu7wOZjEdpiIjakpezGjOHBQMAlu48iSYdl09oKyaVmvLycuh0Onh5eTW738vLCyUlJa0K4OTkhCFDhmDx4sUoKiqCTqfDBx98gLS0NBQX/74A2JgxY/D+++8jJSUFS5Yswd69ezF27FjodLrrbjcxMREuLi7Gm7+/f6vydTZLkk9CkoBx/X0wyN9VdBwiIqv01PBguDsocfZiLbYdLBAdx2qYxdVPW7ZsgSRJ8PPzg0qlwqpVqzBlyhTI5b/Hmzx5Mv70pz+hf//+mDBhAr755htkZGQgNTX1uttMSEiARqMx3goK+JfmVn7JLcfeUxdhI5dhXmxv0XGIiKyWk9oWz99nmNB05fenUadtEpzIOphUajw8PKBQKFBa2vw7wNLS0hueBNwSPXr0wN69e1FTU4OCggKkp6ejsbERwcHBN3xMcHAwPDw8kJube92fq1QqODs7N7vRjen1EhKvLofwaHR3BHo4CE5ERGTdpkYHoLu7PS5WN+C9H/NEx7EKJpUapVKJ8PBwpKSkGO/T6/VISUnBkCFDbjuMg4MDfHx8cPnyZezcuRPjx4+/4dgLFy7g0qVL8PHxue3nJeCbI8U4UqiBg1KBF0b2FB2HiMjqKW3keOXqUfF3955BeU2D4ESWz+Svn+Lj47F+/Xps3rwZJ06cwDPPPIPa2lrExcUBAKZNm4aEhATjeK1Wi+zsbGRnZ0Or1aKwsBDZ2dnNjrDs3LkTycnJyMvLw+7du3HvvfciNDTUuM2amhrMmzcP+/fvx7lz55CSkoLx48cjJCQEsbGxt/v/oNPTNumxfKdhIqjZ9/SAh+O1V6kREVHbe6C/D/r7uaBWq8OqlNOi41g8k0vNpEmTsHz5cixcuBCDBg1CdnY2kpOTjScP5+fnNzvBt6ioCGFhYQgLC0NxcTGWL1+OsLAwzJw50zhGo9Fgzpw5CA0NxbRp0zBs2DDs3LkTtra2AACFQoFff/0Vf/rTn9CrVy88+eSTCA8Px48//njdy8TJNB8eOI/8ijp4OKow8+4g0XGIiDoNufz35RM+OpCPvPJawYksm0zqJDP/VFVVwcXFBRqNhufX/Jfq+kYMX5aKilot3vhzP0yNDhAdiYio05mxMR2pORcxrr8P1kwdLDqOWTHl89ssrn4icd7dexYVtVoEd3XApAhe9k5EJML8MaGQyYBvjxQju6BSdByLxVLTiZVW1eO9n84CAF6NDYWNgn8diIhE6OPjjIfCDOvsJXL5hFbjp1gntvL7U6hv1CM8wA2xfb1u/QAiImo38aN7QWkjx4G8CuzJuf56inRzLDWdVG5ZNbZlGCYkTBgbyuUQiIgE83O1Q9zQQADAkh050Ol5tMZULDWd1JLkHOglYNQdXogIdBcdh4iIADw7IgQudrbIKa3GZ1kXRMexOCw1ndDBcxXYfbwUchkwfwyXQyAiMhcu9raYc28PAMCbu0+hvvH66xvS9bHUdDKSJOEf350AAEyK9EeIp5PgRERE9N+mDQmEn6sdijX12PjzOdFxLApLTSez81gpsvIrobaVY25ML9FxiIjoD9S2CsSPMvx+fic1F5drtYITWQ6Wmk6kSafH0p2GRStnDguGl7NacCIiIrqeCWF+CPV2QnV9E9bsuf7CzXQtlppOZNvBApy9WAt3ByWeGn7jFdCJiEgshVyGBVeXT3g/7TwKKuoEJ7IMLDWdRJ22CSu/NyyW9vx9IXBS2wpORERENzO8V1cM7dEFWp0eK3afEh3HIrDUdBLv/ZiHi9UN6O5uz/WdiIgsgEwmQ8LYPgCAL7ILcaxIIziR+WOp6QTKaxrw7t4zAIBXYntDacOXnYjIEvTv5oIHB/pCkoCkHSdFxzF7/HTrBFalnEatVof+fi54oL+P6DhERGSCeaN7w1Yhw4+ny/HT6XLRccwaS42VyyuvxUcH8gEYlkOQy7kcAhGRJene5ffTBhJ3nICeyyfcEEuNlVu+MwdNeslwwlmIh+g4RETUCs/fFwJHlQ2OFVXh61+LRMcxWyw1Viy7oBLfHimGTAbjpYFERGR5ujiq8PTVqTiW7cxBQxOXT7gelhorJUkSEq8uh/DnMD/08XEWnIiIiG7HE8OC4OmkwoXLV/DB/nzRccwSS42V2pNThgN5FVDayPHyaC5aSURk6eyVNnjp6vIJq384jar6RsGJzA9LjRXS6SUs2ZEDAJgx1LAwGhERWb5HwruhR1cHXK5rxLrUM6LjmB2WGiv0WdYF5JRWw1ltg2dH9BAdh4iI2oiNQo75YwznSG74OQ8lmnrBicwLS42VqW/U4c2r02nPuTcErvZKwYmIiKgtjbrDCxEBbqhv1Bt/35MBS42V2fjzORRr6uHrosb0oYGi4xARURuTyWRIuN9wtObTzAKcKq0WnMh8sNRYkcu1WryTaliiPn50b6htFYITERFRewgPcEdsXy/oJWAJl08wYqmxImv25KK6vgmh3k74c5if6DhERNSOXh0TCoVchpSTZThw9pLoOGaBpcZKFFTU4f208wCA+WMNf9GJiMh69ejqiEmR/gCAxB0nIUlcPoGlxkqs2H0KWp0eQ4K7YESvrqLjEBFRB5g7sifsbBXILqjEjqMlouMIx1JjBY4VafBFdiEAIOH+UMhkPEpDRNQZeDqrMevuIACG5RMadXrBicRiqbECSTtOQpKABwf6YkA3V9FxiIioA80e3gNdHJTIK6/F1vTOvXwCS42F++l0OX48XQ5bhQzzuBwCEVGn46iywYsxPQEAb6WcRk1Dk+BE4rDUWDC9XkLiDsOilVOjA9C9i73gREREJMKUqO4I7GKP8hot1u87KzqOMCw1FuzrX4twrKgKjiobPH9fiOg4REQkiK1Cjnmxhgn51v94FmXVnXP5BJYaC9XQpMOynYZFK58eHowujirBiYiISKT7+3tjoL8r6rQ6rEo5LTqOECw1FuqD/fm4cPkKPJ1UeGJYkOg4REQkmEwmQ8JYw9Gaj9MLcPZijeBEHY+lxgJV1Tdi9Q+GFv7SqF6wV9oITkRERObgzuAuuC/UEzq9ZDya35mw1FigdalncLmuET26OuCR8G6i4xARkRmZPyYUchmw42gJsvIvi47ToVhqLEyJph4bfs4DYPiLa6PgS0hERL/r7e2EiYMN/+BN+q5zLZ/AT0QL8+buU6hv1CMiwA2j7vASHYeIiMxQ/OheUNnIkX6uAiknykTH6TAsNRbkdGk1Ps0sAMDlEIiI6MZ8XOwQd5fhIpIlySfR1EmWT2CpsSBLkk9CLwGxfb0QHuAuOg4REZmxZ0b0gKu9LU6X1eCzrAui43QIlhoLkZ5Xge9PlEEhl+HVMaGi4xARkZlzsbPFc/caJmZdsfsUrmh1ghO1P5YaCyBJEv7xnWE5hEmR/ujR1VFwIiIisgSPDwmAn6sdSqsajBeZWDOWGguw42gJsgsqYWerwNyRPUXHISIiC6GyUeCV2F4ADNOBVNRqBSdqX60qNWvWrEFgYCDUajWio6ORnp5+w7HHjh3DxIkTERgYCJlMhpUrV14zprq6GnPnzkVAQADs7OwwdOhQZGRkNBsjSRIWLlwIHx8f2NnZISYmBqdPW/800I06vXECpVl3B8HTWS04ERERWZLxA/1wh48zqhuasPqHXNFx2pXJpWbbtm2Ij4/HokWLkJWVhYEDByI2NhZlZde/ZKyurg7BwcFISkqCt7f3dcfMnDkTu3fvxpYtW3DkyBGMHj0aMTExKCwsNI5ZunQpVq1ahXXr1uHAgQNwcHBAbGws6uute9Guren5yCuvRRcHJWYP7yE6DhERWRi5XIYFV5dP2LL/HAoq6gQnakeSiaKioqQ5c+YY/6zT6SRfX18pMTHxlo8NCAiQ3nzzzWb31dXVSQqFQvrmm2+a3T948GDpr3/9qyRJkqTX6yVvb29p2bJlxp9XVlZKKpVK+vjjj1uUW6PRSAAkjUbTovHmoLq+UQpfvEsKmP+NtOnnPNFxiIjIgk1dv18KmP+N9MLHWaKjmMSUz2+TjtRotVpkZmYiJibGeJ9cLkdMTAzS0tJaVaqampqg0+mgVjf/WsXOzg4//fQTACAvLw8lJSXNntfFxQXR0dGtfl5LsH7fWZTXaBHQxR5TorqLjkNERBbst6M1X2YX4WihRnCa9mFSqSkvL4dOp4OXV/OZbL28vFBSUtKqAE5OThgyZAgWL16MoqIi6HQ6fPDBB0hLS0NxcTEAGLdtyvM2NDSgqqqq2c2SlFXXY/2PZwEA82J7Q2nDc7qJiKj1+vm5YPwgXwBA4o4TVrl8gll8Um7ZsgWSJMHPzw8qlQqrVq3ClClTIJe3Pl5iYiJcXFyMN39//zZM3P5WpZxGnVaHgd1cMK6/j+g4RERkBV4Z3RtKhRw/517CvtPlouO0OZNag4eHBxQKBUpLS5vdX1paesOTgFuiR48e2Lt3L2pqalBQUID09HQ0NjYiODgYAIzbNuV5ExISoNFojLeCgoJW5+toZy/W4ON0Q94FY/twOQQiImoT/u72eOzOAABA0o6T0Out62iNSaVGqVQiPDwcKSkpxvv0ej1SUlIwZMiQ2w7j4OAAHx8fXL58GTt37sT48eMBAEFBQfD29m72vFVVVThw4MANn1elUsHZ2bnZzVIs25kDnV7Cvb27YkiPLqLjEBGRFXnuvhA4qWxworgKX2QX3voBFsTk73fi4+Oxfv16bN68GSdOnMAzzzyD2tpaxMXFAQCmTZuGhIQE43itVovs7GxkZ2dDq9WisLAQ2dnZyM39/Vr5nTt3Ijk5GXl5edi9ezfuvfdehIaGGrcpk8kwd+5c/P3vf8dXX32FI0eOYNq0afD19cWECRNu83+BecnKv4wdR0sgkwHzx3I5BCIialvuDko8PcIwRcg/d51CfaP1LJ9gY+oDJk2ahIsXL2LhwoUoKSnBoEGDkJycbDyJNz8/v9m5MEVFRQgLCzP+efny5Vi+fDmGDx+O1NRUAIBGo0FCQgIuXLgAd3d3TJw4EW+88QZsbW2Nj3v11VdRW1uL2bNno7KyEsOGDUNycvI1V01ZMkmSkPTdSQDAxMHdEOptOUeXiIjIcjxxVxC2pJ1HYeUVbEk7j1n3BIuO1CZkkjWe/nwdVVVVcHFxgUajMduvor4/XoqZ7x+EykaOPa+MgK+rnehIRERkpbZl5GP+Z0fgYmeLffPuhYu97a0fJIApn99mcfUTAU06PZYkG47SzLgrkIWGiIja1cTB3dDT0xGaK414Z691LJ/AUmMmPsu6gNNlNXCxs8Wzw0NExyEiIitno5Bj/hjDuZsbfz6HosorghPdPpYaM3BFq8OK3acAAM/fF2K2hwCJiMi6jOzjiaggd2ib9MbPIUvGUmMGNvych9KqBvi52uHxIQGi4xARUSchk8mQcPVK28+yLuBkiWXNvv9HLDWCVdRqsS71DADgldheUNkoBCciIqLOJKy7G+7v7w1JApbsOCk6zm1hqRFs9Q+5qG5owh0+zhg/0E90HCIi6oTmxYbCRi7DnpyLSDtzSXScVmOpEaigog5b9p8DYFg9VS7ncghERNTxgjwcMCWqOwAgyYIXu2SpEWj5rhw06iQMC/HAPb26io5DRESd2Asje8JBqcDhCxp8e6RYdJxWYakR5GihBl9mFwEwHKUhIiISqauTyjiz8LKdOdA26QUnMh1LjSBJV0/GGj/IF/38XASnISIiAmbdHQwPRxXOX6rDx+n5ouOYjKVGgH2nLuKn3HIoFXK8Mrq36DhEREQAAAeVDV6M6QkAWJVyGtX1jYITmYalpoPp9ZLxKM1jdwbA391ecCIiIqLfTY70R7CHAy7VarF+31nRcUzCUtPBvjxciOPFVXBS2eC5+7gcAhERmRdbhRzzYg3fIqz/MQ9lVfWCE7UcS00Hqm/UYflOwzTUT4/oAXcHpeBERERE1xrTzxth3V1xpVGHlSmnRcdpMZaaDvTB/vMorLwCb2c1nrgrSHQcIiKi6zIsn9AHALAtowBnLtYITtQyLDUdRHOlEav3GJZ2f2lUT9gpuRwCERGZr6ggd8T08YROL2FpsmUsn8BS00HWpp5BZV0jeno6YuLgbqLjEBER3dL8MaGQy4Cdx0qReb5CdJxbYqnpAEWVV7Dh5zwAhr8gNgr+byciIvPX08sJj4T7AwASvztp9ssn8NO1A6zYfQraJj2iAt0xso+n6DhEREQt9tKoXlDbynHw/GXsPl4qOs5NsdS0s5MlVfgs6wIAYMH9oZDJuGglERFZDm+X3y9uWZJ8Ek06810+gaWmnS3ZcRKSBIzt543B3d1ExyEiIjLZ0yN6wM3eFmcu1uKTgxdEx7khlpp2lHbmEvbkXIRCLjNOZERERGRpnNW2eO4+w/IJb35/CnXaJsGJro+lpp1IkoSkHScAAFOi/BHc1VFwIiIiotZ77M7u6OZmh4vVDfj3j3mi41wXS007+fZIMQ5f0MBeqcCLI3uJjkNERHRbVDYK47cO7+47i0s1DYITXYulph1om/RYtjMHgGEZ965OKsGJiIiIbt+DA3zRz88ZNQ1NePuHXNFxrsFS0w4+Ts/H+Ut18HBUYtY9waLjEBERtQm5XIYFYwzLJ3x44DzOX6oVnKg5lpo2Vl3fiFVXF/96cWRPOKpsBCciIiJqO8N6euDunh5o1EnGbyXMBUtNG1u/7ywu1WoR5OGAyVHdRcchIiJqcwvGhkImA775tRiHCypFxzFiqWlDZVX1WH/1jPB5sb1hy+UQiIjICvX1dcGEQX4AgKQd5rN8Aj9129DKlNO40qjDIH9XjO3nLToOERFRu4kf1QtKhRxpZy8h9dRF0XEAsNS0mTMXa7AtowAAkDCWyyEQEZF183e3x/ShAQAMs+fr9OKP1rDUtJGlyYYXNKaPJ6KDu4iOQ0RE1O7m3BsCZ7UNTpZUY/uhQtFxWGraQub5Cuw8Vgq5DJg/JlR0HCIiog7haq/Es/eGAABW7MpBfaNOaB6WmtskSRISvzsJAHgk3B89vZwEJyIiIuo4M4YGwsdFjSJNPTb/ck5oFpaa27QnpwwHz1+G2laOl0ZxOQQiIupc1LYKxF/9/FuzJxdV9Y3CsnBmuNs0tIcH/np/HzTq9fB2UYuOQ0RE1OEeGtwN+06XY+JgPzgJnHRWJpnLxeXtrKqqCi4uLtBoNHB2dhYdh4iIiFrAlM9vfv1EREREVoGlhoiIiKwCSw0RERFZBZYaIiIisgosNURERGQVWGqIiIjIKrDUEBERkVVoValZs2YNAgMDoVarER0djfT09BuOPXbsGCZOnIjAwEDIZDKsXLnymjE6nQ6vvfYagoKCYGdnhx49emDx4sX47yl0ZsyYAZlM1uw2ZsyY1sQnIiIiK2TytH/btm1DfHw81q1bh+joaKxcuRKxsbHIycmBp6fnNePr6uoQHByMRx55BC+99NJ1t7lkyRKsXbsWmzdvRt++fXHw4EHExcXBxcUFL7zwgnHcmDFjsHHjRuOfVSqVqfGJiIjISplcalasWIFZs2YhLi4OALBu3Tp8++232LBhAxYsWHDN+MjISERGRgLAdX8OAL/88gvGjx+PcePGAQACAwPx8ccfX3MESKVSwdvb29TIRERE1AmY9PWTVqtFZmYmYmJift+AXI6YmBikpaW1OsTQoUORkpKCU6dOAQAOHz6Mn376CWPHjm02LjU1FZ6enujduzeeeeYZXLp06YbbbGhoQFVVVbMbERERWS+TjtSUl5dDp9PBy8ur2f1eXl44efJkq0MsWLAAVVVVCA0NhUKhgE6nwxtvvIGpU6cax4wZMwYPPfQQgoKCcObMGfzP//wPxo4di7S0NCgUimu2mZiYiL/97W+tzkRERESWxSxW6f7kk0/w4Ycf4qOPPkLfvn2RnZ2NuXPnwtfXF9OnTwcATJ482Ti+f//+GDBgAHr06IHU1FSMHDnymm0mJCQgPj7e+Oeqqir4+/u3/84QERGRECaVGg8PDygUCpSWlja7v7S09LbOdZk3bx4WLFhgLC79+/fH+fPnkZiYaCw1fxQcHAwPDw/k5uZet9SoVKpmJxL/diUVv4YiIiKyHL99bv/3FdE3YlKpUSqVCA8PR0pKCiZMmAAA0Ov1SElJwXPPPWd60qvq6uoglzc/vUehUECv19/wMRcuXMClS5fg4+PToueorq4GAB6tISIiskDV1dVwcXG56RiTv36Kj4/H9OnTERERgaioKKxcuRK1tbXGq6GmTZsGPz8/JCYmAjCcXHz8+HHjfxcWFiI7OxuOjo4ICQkBADz44IN444030L17d/Tt2xeHDh3CihUr8MQTTwAAampq8Le//Q0TJ06Et7c3zpw5g1dffRUhISGIjY1tUW5fX18UFBTAyckJMpnM1N2+qd++2iooKICzs3ObbtscWPv+Ada/j9w/y2ft+8j9s3zttY+SJKG6uhq+vr4tGmyyt99+W+revbukVCqlqKgoaf/+/cafDR8+XJo+fbrxz3l5eRKAa27Dhw83jqmqqpJefPFFqXv37pJarZaCg4Olv/71r1JDQ4MkSZJUV1cnjR49Wuratatka2srBQQESLNmzZJKSkpaE7/NaTQaCYCk0WhER2kX1r5/kmT9+8j9s3zWvo/cP8tnDvsok6QWfElFN1VVVQUXFxdoNBqrbODWvn+A9e8j98/yWfs+cv8snznsI9d+IiIiIqvAUtMGVCoVFi1aZLXLNlj7/gHWv4/cP8tn7fvI/bN85rCP/PqJiIiIrAKP1BAREZFVYKkhIiIiq8BSQ0RERFaBpYaIiIisAktNC61ZswaBgYFQq9WIjo5Genr6Tcd/+umnCA0NhVqtRv/+/fHdd991UNLWMWX/Nm3aBJlM1uymVqs7MK1p9u3bhwcffBC+vr6QyWT44osvbvmY1NRUDB48GCqVCiEhIdi0aVO757wdpu5jamrqNa+hTCZDSUlJxwQ2QWJiIiIjI+Hk5ARPT09MmDABOTk5t3ycJb0HW7OPlvQ+XLt2LQYMGABnZ2c4OztjyJAh2LFjx00fY0mvH2D6PlrS63c9SUlJkMlkmDt37k3HdfTryFLTAtu2bUN8fDwWLVqErKwsDBw4ELGxsSgrK7vu+F9++QVTpkzBk08+iUOHDmHChAmYMGECjh492sHJW8bU/QMAZ2dnFBcXG2/nz5/vwMSmqa2txcCBA7FmzZoWjc/Ly8O4ceNw7733GleMnzlzJnbu3NnOSVvP1H38TU5OTrPX0dPTs50Stt7evXsxZ84c7N+/H7t370ZjYyNGjx6N2traGz7G0t6DrdlHwHLeh926dUNSUhIyMzNx8OBB3HfffRg/fjyOHTt23fGW9voBpu8jYDmv3x9lZGTg3XffxYABA246TsjrKGwuYwsSFRUlzZkzx/hnnU4n+fr6SomJidcd/5e//EUaN25cs/uio6Olp556ql1ztpap+7dx40bJxcWlg9K1LQDS9u3bbzrm1Vdflfr27dvsvkmTJkmxsbHtmKzttGQf9+zZIwGQLl++3CGZ2lJZWZkEQNq7d+8Nx1jae/CPWrKPlvw+lCRJcnNzk957773r/szSX7/f3GwfLfX1q66ulnr27Cnt3r1bGj58uPTiiy/ecKyI15FHam5Bq9UiMzMTMTExxvvkcjliYmKQlpZ23cekpaU1Gw8AsbGxNxwvUmv2DzAsMhoQEAB/f/9b/mvE0ljS63e7Bg0aBB8fH4waNQo///yz6DgtotFoAADu7u43HGPpr2FL9hGwzPehTqfD1q1bUVtbiyFDhlx3jKW/fi3ZR8AyX785c+Zg3Lhx17w+1yPidWSpuYXy8nLodDp4eXk1u9/Ly+uG5x+UlJSYNF6k1uxf7969sWHDBnz55Zf44IMPoNfrMXToUFy4cKEjIre7G71+VVVVuHLliqBUbcvHxwfr1q3DZ599hs8++wz+/v4YMWIEsrKyREe7Kb1ej7lz5+Kuu+5Cv379bjjOkt6Df9TSfbS09+GRI0fg6OgIlUqFp59+Gtu3b8cdd9xx3bGW+vqZso+W9voBwNatW5GVlYXExMQWjRfxOtq025bJag0ZMqTZvz6GDh2KPn364N1338XixYsFJqOW6t27N3r37m3889ChQ3HmzBm8+eab2LJli8BkNzdnzhwcPXoUP/30k+go7aal+2hp78PevXsjOzsbGo0G//nPfzB9+nTs3bv3hh/6lsiUfbS016+goAAvvvgidu/ebdYnNLPU3IKHhwcUCgVKS0ub3V9aWgpvb+/rPsbb29uk8SK1Zv/+yNbWFmFhYcjNzW2PiB3uRq+fs7Mz7OzsBKVqf1FRUWZdFp577jl888032LdvH7p163bTsZb0HvxvpuzjH5n7+1CpVCIkJAQAEB4ejoyMDLz11lt49913rxlrqa+fKfv4R+b++mVmZqKsrAyDBw823qfT6bBv3z6sXr0aDQ0NUCgUzR4j4nXk10+3oFQqER4ejpSUFON9er0eKSkpN/yudMiQIc3GA8Du3btv+t2qKK3Zvz/S6XQ4cuQIfHx82itmh7Kk168tZWdnm+VrKEkSnnvuOWzfvh0//PADgoKCbvkYS3sNW7OPf2Rp70O9Xo+Ghobr/szSXr8budk+/pG5v34jR47EkSNHkJ2dbbxFRERg6tSpyM7OvqbQAIJex3Y7BdmKbN26VVKpVNKmTZuk48ePS7Nnz5ZcXV2lkpISSZIk6fHHH5cWLFhgHP/zzz9LNjY20vLly6UTJ05IixYtkmxtbaUjR46I2oWbMnX//va3v0k7d+6Uzpw5I2VmZkqTJ0+W1Gq1dOzYMVG7cFPV1dXSoUOHpEOHDkkApBUrVkiHDh2Szp8/L0mSJC1YsEB6/PHHjePPnj0r2dvbS/PmzZNOnDghrVmzRlIoFFJycrKoXbglU/fxzTfflL744gvp9OnT0pEjR6QXX3xRksvl0vfffy9qF27omWeekVxcXKTU1FSpuLjYeKurqzOOsfT3YGv20ZLehwsWLJD27t0r5eXlSb/++qu0YMECSSaTSbt27ZIkyfJfP0kyfR8t6fW7kT9e/WQOryNLTQu9/fbbUvfu3SWlUilFRUVJ+/fvN/5s+PDh0vTp05uN/+STT6RevXpJSqVS6tu3r/Ttt992cGLTmLJ/c+fONY718vKS7r//fikrK0tA6pb57fLlP95+26fp06dLw4cPv+YxgwYNkpRKpRQcHCxt3Lixw3ObwtR9XLJkidSjRw9JrVZL7u7u0ogRI6QffvhBTPhbuN5+AWj2mlj6e7A1+2hJ78MnnnhCCggIkJRKpdS1a1dp5MiRxg97SbL810+STN9HS3r9buSPpcYcXkeZJElS+x0HIiIiIuoYPKeGiIiIrAJLDREREVkFlhoiIiKyCiw1REREZBVYaoiIiMgqsNQQERGRVWCpISIiIqvAUkNERERWgaWGiIiIrAJLDREREVkFlhoiIiKyCiw1REREZBX+HySSUc/B8M55AAAAAElFTkSuQmCC",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "ks = 5\n",
-    "sigma = 2\n",
-    "half_ks = (ks - 1) // 2\n",
-    "base_kernel = [0.0] * half_ks + [1.0] + [0.0] * half_ks\n",
-    "kernel_window = gaussian_filter1d(base_kernel, sigma=sigma)\n",
-    "plt.plot(kernel_window)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8e6444b1",
-   "metadata": {},
-   "source": [
-    "# Label Distribution Smoothing - visualization\n",
-    "* visualization of  pytorch_widedeep.training._wd_dataset.WideDeepDataset._prepare_weights(...)\n",
-    "\n",
-    "**Assign weight to each sample by following procedure:**\n",
-    "1. creating histogram from label values with nuber of bins = granularity\n",
-    "2.[OPTIONAL] reweighting label frequencies by sqrt\n",
-    "3.[OPTIONAL] smoothing label frequencies by convolution of kernel function window with frequencies list\n",
-    "4. inverting values by n_samples / (n_classes * np.bincount(y)), [see](https://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_sample_weight.html)\n",
-    "5. assigning weight to each sample from closest bin value "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "45de4c6b",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAHVCAYAAAB8NLYkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABf8UlEQVR4nO3deVhUZf8G8HsAGZBlWGQRRcAlFRFRUEQFF0gyMi1zCw1NsxQ1wyWo3BdcSk1F0HpfNcvUFtRcUF5c0EJzSV+13EqTVMANRlARmef3hy/n58iwKTjD4f5c11w6z5zle+acOXPznGUUQggBIiIiIqr2jPRdABERERFVDgY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7omrA3d0dQ4YM0XcZsrdgwQI0bNgQxsbG8PHxeW7zHTJkCNzd3Z/b/Cpi7969UCgU2Lt3b5XPa9q0aVAoFLhx40aVz4tIrhjsiJ6z1atXQ6FQ4MiRIzpf79KlC7y8vJ55Ptu3b8e0adOeeTo1xa5duzBp0iR07NgRq1atwpw5c/RdkmzNmTMHmzZt0su8e/ToAVtbW2RmZhZ7LScnB3Xr1oW/vz80Go0eqiN6dib6LoCIynb27FkYGVXs77Dt27cjLi6O4a6cdu/eDSMjI/zrX/+CqampvssxGEFBQbh3716lvidz5szBG2+8gd69e1faNMtr+fLl8PLywgcffIB169ZpvfbRRx/hxo0bSEpKqvDnjchQcMslqgaUSiVq1aql7zIqJC8vT98lVEhWVhbMzc0Z6p5gZGQEMzMz2QQdDw8PTJ06Fd9++y127doltR8+fBgJCQmIiopCq1at9Fgh0bORxyeVSOaePMeuoKAA06dPR5MmTWBmZgZ7e3t06tQJycnJAB6dsxUXFwcAUCgU0qNIXl4exo8fD1dXVyiVSjRt2hSffvophBBa87137x7Gjh2LOnXqwMrKCq+++iquXLkChUKh1RNYdG7U77//jjfffBO2trbo1KkTAOC///0vhgwZgoYNG8LMzAzOzs54++23cfPmTa15FU3j3LlzGDRoEFQqFRwcHDB58mQIIZCeno5evXrB2toazs7O+Oyzz8r13j18+BAzZ85Eo0aNoFQq4e7ujo8++gj5+fnSMAqFAqtWrUJeXp70Xq1evVrn9EaPHg1LS0vcvXu32GsDBw6Es7MzCgsLAQCbN29GWFgYXFxcoFQq0ahRI8ycOVN6vSQlndd26dIlnbWdOXMGb7zxBuzs7GBmZgY/Pz9s2bJFa5iytpmK1FJ0usDvv/+Orl27onbt2qhXrx7mz59f6rSAR+91Xl4e1qxZI73XT54/mp2djSFDhsDGxgYqlQpDhw7V+X5//fXX8PX1hbm5Oezs7DBgwACkp6eXWUNUVBS8vb0xatQo3L9/H4WFhXjvvffg5uaGqVOnljk+kSHjoVgiPcnJydF5knhBQUGZ406bNg2xsbEYPnw42rVrB7VajSNHjuDYsWN48cUX8e677+Lq1atITk7G2rVrtcYVQuDVV1/Fnj17MGzYMPj4+GDnzp2YOHEirly5gkWLFknDDhkyBBs3bsTgwYPRvn177Nu3D2FhYSXW1bdvXzRp0gRz5syRQmJycjL++usvDB06FM7Ozjh9+jRWrlyJ06dP4+DBg1qBEwD69++P5s2bY+7cudi2bRtmzZoFOzs7rFixAt26dcO8efPwzTffYMKECWjbti2CgoJKfa+GDx+ONWvW4I033sD48eNx6NAhxMbG4o8//kBiYiIAYO3atVi5ciV+/fVXfPnllwCADh066Jxe//79ERcXh23btqFv375S+927d/HTTz9hyJAhMDY2BvDofEpLS0tERUXB0tISu3fvxpQpU6BWq7FgwYJS6y6v06dPo2PHjqhXrx6io6NhYWGBjRs3onfv3vjhhx/w2muvASh7m6mo27dv46WXXsLrr7+Ofv364fvvv8eHH36Ili1bokePHiWOt3btWqmGESNGAAAaNWqkNUy/fv3g4eGB2NhYHDt2DF9++SUcHR0xb948aZjZs2dj8uTJ6NevH4YPH47r169j6dKlCAoKwm+//QYbG5sSazAxMcHKlSvRoUMHzJw5E46Ojjh27BiSkpJQu3btCr8XRAZFENFztWrVKgGg1EeLFi20xnFzcxMRERHS81atWomwsLBS5xMZGSl0fcQ3bdokAIhZs2Zptb/xxhtCoVCICxcuCCGEOHr0qAAgxo0bpzXckCFDBAAxdepUqW3q1KkCgBg4cGCx+d29e7dY27fffisAiNTU1GLTGDFihNT28OFDUb9+faFQKMTcuXOl9tu3bwtzc3Ot90SX48ePCwBi+PDhWu0TJkwQAMTu3bultoiICGFhYVHq9IQQQqPRiHr16ok+ffpotW/cuLHYMula9nfffVfUrl1b3L9/X2vebm5u0vM9e/YIAGLPnj1a4168eFEAEKtWrZLagoODRcuWLbWmp9FoRIcOHUSTJk2ktvJsM7roqqVz584CgPjqq6+ktvz8fOHs7FzsfdHFwsJC57or2gbefvttrfbXXntN2NvbS88vXbokjI2NxezZs7WGO3nypDAxMSnWXpLRo0eLWrVqCUtLS53bLlF1xEOxRHoSFxeH5OTkYg9vb+8yx7WxscHp06dx/vz5Cs93+/btMDY2xtixY7Xax48fDyEEduzYAQBISkoCAIwaNUpruDFjxpQ47ffee69Ym7m5ufT/+/fv48aNG2jfvj0A4NixY8WGHz58uPR/Y2Nj+Pn5QQiBYcOGSe02NjZo2rQp/vrrrxJrAR4tK/Do0Nvjxo8fDwDYtm1bqeProlAo0LdvX2zfvh25ublS+4YNG1CvXj3pEDSgvex37tzBjRs3EBgYiLt37+LMmTMVnveTbt26hd27d6Nfv37S9G/cuIGbN28iNDQU58+fx5UrVwA82zaji6WlJQYNGiQ9NzU1Rbt27cpcJ+Xx5HYUGBiImzdvQq1WAwB+/PFHaDQa9OvXT1rmGzduwNnZGU2aNMGePXvKNZ/Zs2fD3t4eRkZGWj3VRNUZgx2RnrRr1w4hISHFHra2tmWOO2PGDGRnZ+OFF15Ay5YtMXHiRPz3v/8t13z//vtvuLi4wMrKSqu9efPm0utF/xoZGcHDw0NruMaNG5c47SeHBR6Fj/fffx9OTk4wNzeHg4ODNFxOTk6x4Rs0aKD1XKVSwczMDHXq1CnWfvv27RJreXwZnqzZ2dkZNjY20rJWVP/+/XHv3j3pPLbc3Fxs374dffv21Tq0fPr0abz22mtQqVSwtraGg4ODFIZ0LXtFXbhwAUIITJ48GQ4ODlqPonPFsrKyADzbNqNL/fr1ix1Gt7W1LXOdlMeT20DRZ6Jo2ufPn4cQAk2aNCm23H/88Ye0zGWxtrZG06ZN4erqCicnp2eum8gQ8Bw7omooKCgIf/75JzZv3oxdu3bhyy+/xKJFi5CQkKDV4/W8Pd5DVaRfv3745ZdfMHHiRPj4+MDS0hIajQYvvfSSznuFFZ2fVlYbgGIXe5TkyQDyrNq3bw93d3ds3LgRb775Jn766Sfcu3cP/fv3l4bJzs5G586dYW1tjRkzZqBRo0YwMzPDsWPH8OGHH5Z6n7SS6n3yoouiaUyYMAGhoaE6xykKtZW9zTzrOnmWaWs0GigUCuzYsUPnsJaWls9cA1F1xWBHVE3Z2dlh6NChGDp0KHJzcxEUFIRp06ZJX9IlhQM3Nzf85z//wZ07d7R67YoODbq5uUn/ajQaXLx4EU2aNJGGu3DhQrlrvH37NlJSUjB9+nRMmTJFaq+sw4FlKVqG8+fPSz2SAJCZmYns7GxpWZ9Gv3798Pnnn0OtVmPDhg1wd3eXDjEDj64mvXnzJn788UetCzwuXrxY5rSLeqiys7O12p/sYWzYsCEAoFatWggJCSlzumVtM8/LswbtRo0aQQgBDw8PvPDCC5VUFZE88FAsUTX05K1CLC0t0bhxY61beFhYWAAoHg5efvllFBYWYtmyZVrtixYtgkKhkK5oLOoBWr58udZwS5cuLXedRb0pT/biLF68uNzTeBYvv/yyzvktXLgQAEq9wrcs/fv3R35+PtasWYOkpCT069dP63Vdy/7gwYNi76cubm5uMDY2Rmpqqlb7k+M6OjqiS5cuWLFiBa5du1ZsOtevX5f+X55t5nmxsLAotl1WxOuvvw5jY2NMnz692LYlhCi2rEQ1CXvsiKohT09PdOnSBb6+vrCzs8ORI0fw/fffY/To0dIwvr6+AICxY8ciNDQUxsbGGDBgAHr27ImuXbvi448/xqVLl9CqVSvs2rULmzdvxrhx46RbT/j6+qJPnz5YvHgxbt68Kd3u5Ny5cwDK1+tibW2NoKAgzJ8/HwUFBahXrx527dpVrl6rytCqVStERERg5cqV0qHRX3/9FWvWrEHv3r3RtWvXp552mzZt0LhxY3z88cfIz8/XOgwLPLpdiq2tLSIiIjB27FgoFAqsXbu2XIcqVSoV+vbti6VLl0KhUKBRo0bYunWrznPH4uLi0KlTJ7Rs2RLvvPMOGjZsiMzMTKSlpeGff/7BiRMnAJRvm3lefH198Z///AcLFy6Ei4sLPDw84O/vX+7xGzVqhFmzZiEmJgaXLl1C7969YWVlhYsXLyIxMREjRozAhAkTqnAJiAwXgx1RNTR27Fhs2bIFu3btQn5+Ptzc3DBr1ixMnDhRGub111/HmDFjsH79enz99dcQQmDAgAEwMjLCli1bMGXKFGzYsAGrVq2Cu7s7FixYIF0tWuSrr76Cs7Mzvv32WyQmJiIkJAQbNmxA06ZNYWZmVq5a161bhzFjxiAuLg5CCHTv3h07duyAi4tLpb4nJfnyyy/RsGFDrF69GomJiXB2dkZMTEyl3Ii2f//+mD17Nho3bow2bdpovWZvb4+tW7di/Pjx+OSTT2Bra4tBgwYhODi4xPPhHrd06VIUFBQgISEBSqUS/fr1w4IFC4r9jrCnpyeOHDmC6dOnY/Xq1bh58yYcHR3RunVrrcPf5dlmnpeFCxdixIgR+OSTT3Dv3j1ERERUKNgBQHR0NF544QUsWrQI06dPBwC4urqie/fuePXVV6uibKJqQSEq40xXIqoxjh8/jtatW+Prr79GeHi4vsshIqLH8Bw7IirRvXv3irUtXrwYRkZGZf7iAxERPX88FEtEJZo/fz6OHj2Krl27wsTEBDt27MCOHTswYsQIuLq66rs8IiJ6Ag/FElGJkpOTMX36dPz+++/Izc1FgwYNMHjwYHz88ccwMeHfhUREhobBjoiIiEgmeI4dERERkUzI9liKRqPB1atXYWVlVek/J0RERET0vAghcOfOHbi4uMDIqPQ+OdkGu6tXr/LkbiIiIpKN9PR01K9fv9RhZBvsin4DMz09HdbW1nquhoiIiOjpqNVquLq6av2+d0lkG+yKDr9aW1sz2BEREVG1V55Ty3jxBBEREZFMMNgRERERyQSDHREREZFMMNgRERERyUSFg11qaip69uwJFxcXKBQKbNq0Set1IQSmTJmCunXrwtzcHCEhITh//rzWMLdu3UJ4eDisra1hY2ODYcOGITc3V2uY//73vwgMDISZmRlcXV0xf/78ii8dERERUQ1S4WCXl5eHVq1aIS4uTufr8+fPx5IlS5CQkIBDhw7BwsICoaGhuH//vjRMeHg4Tp8+jeTkZGzduhWpqakYMWKE9LparUb37t3h5uaGo0ePYsGCBZg2bRpWrlz5FItIRERE9Ozco7cVexiaZ/qtWIVCgcTERPTu3RvAo946FxcXjB8/HhMmTAAA5OTkwMnJCatXr8aAAQPwxx9/wNPTE4cPH4afnx8AICkpCS+//DL++ecfuLi4ID4+Hh9//DEyMjJgamoKAIiOjsamTZtw5swZnbXk5+cjPz9fel50z5ecnBze7oSIiIiema4gd2luWJXPV61WQ6VSlSvTVOo5dhcvXkRGRgZCQkKkNpVKBX9/f6SlpQEA0tLSYGNjI4U6AAgJCYGRkREOHTokDRMUFCSFOgAIDQ3F2bNncfv2bZ3zjo2NhUqlkh781QkiIiKqaSo12GVkZAAAnJyctNqdnJyk1zIyMuDo6Kj1uomJCezs7LSG0TWNx+fxpJiYGOTk5EiP9PT0Z18gIiIiompENr88oVQqoVQq9V0GERERkd5Uao+ds7MzACAzM1OrPTMzU3rN2dkZWVlZWq8/fPgQt27d0hpG1zQenwcRERERaavUYOfh4QFnZ2ekpKRIbWq1GocOHUJAQAAAICAgANnZ2Th69Kg0zO7du6HRaODv7y8Nk5qaioKCAmmY5ORkNG3aFLa2tpVZMhEREZFsVDjY5ebm4vjx4zh+/DiARxdMHD9+HJcvX4ZCocC4ceMwa9YsbNmyBSdPnsRbb70FFxcX6crZ5s2b46WXXsI777yDX3/9FT///DNGjx6NAQMGwMXFBQDw5ptvwtTUFMOGDcPp06exYcMGfP7554iKiqq0BSciIiKSmwqfY3fkyBF07dpVel4UtiIiIrB69WpMmjQJeXl5GDFiBLKzs9GpUyckJSXBzMxMGuebb77B6NGjERwcDCMjI/Tp0wdLliyRXlepVNi1axciIyPh6+uLOnXqYMqUKVr3uiMiIiIibc90HztDVpF7vhARERGVpcbdx46IiIiI9IfBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZILBjoiIiEgmGOyIiIiIZKLSg11hYSEmT54MDw8PmJubo1GjRpg5cyaEENIwQghMmTIFdevWhbm5OUJCQnD+/Hmt6dy6dQvh4eGwtraGjY0Nhg0bhtzc3Moul4iIiEg2Kj3YzZs3D/Hx8Vi2bBn++OMPzJs3D/Pnz8fSpUulYebPn48lS5YgISEBhw4dgoWFBUJDQ3H//n1pmPDwcJw+fRrJycnYunUrUlNTMWLEiMoul4iIiEg2FOLxrrRK8Morr8DJyQn/+te/pLY+ffrA3NwcX3/9NYQQcHFxwfjx4zFhwgQAQE5ODpycnLB69WoMGDAAf/zxBzw9PXH48GH4+fkBAJKSkvDyyy/jn3/+gYuLS5l1qNVqqFQq5OTkwNraujIXkYiIiGog9+htxdouzQ2r8vlWJNNUeo9dhw4dkJKSgnPnzgEATpw4gQMHDqBHjx4AgIsXLyIjIwMhISHSOCqVCv7+/khLSwMApKWlwcbGRgp1ABASEgIjIyMcOnRI53zz8/OhVqu1HkREREQ1iUllTzA6OhpqtRrNmjWDsbExCgsLMXv2bISHhwMAMjIyAABOTk5a4zk5OUmvZWRkwNHRUbtQExPY2dlJwzwpNjYW06dPr+zFISIiIqo2Kr3HbuPGjfjmm2+wbt06HDt2DGvWrMGnn36KNWvWVPastMTExCAnJ0d6pKenV+n8iIiIiAxNpffYTZw4EdHR0RgwYAAAoGXLlvj7778RGxuLiIgIODs7AwAyMzNRt25dabzMzEz4+PgAAJydnZGVlaU13YcPH+LWrVvS+E9SKpVQKpWVvThERERE1Ual99jdvXsXRkbakzU2NoZGowEAeHh4wNnZGSkpKdLrarUahw4dQkBAAAAgICAA2dnZOHr0qDTM7t27odFo4O/vX9klExEREclCpffY9ezZE7Nnz0aDBg3QokUL/Pbbb1i4cCHefvttAIBCocC4ceMwa9YsNGnSBB4eHpg8eTJcXFzQu3dvAEDz5s3x0ksv4Z133kFCQgIKCgowevRoDBgwoFxXxBIRERHVRJUe7JYuXYrJkydj1KhRyMrKgouLC959911MmTJFGmbSpEnIy8vDiBEjkJ2djU6dOiEpKQlmZmbSMN988w1Gjx6N4OBgGBkZoU+fPliyZElll0tEREQkG5V+HztDwfvYERERUWWqkfexIyIiIiL9YLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZYLAjIiIikgkGOyIiIiKZqJJgd+XKFQwaNAj29vYwNzdHy5YtceTIEel1IQSmTJmCunXrwtzcHCEhITh//rzWNG7duoXw8HBYW1vDxsYGw4YNQ25ublWUS0RERCQLlR7sbt++jY4dO6JWrVrYsWMHfv/9d3z22WewtbWVhpk/fz6WLFmChIQEHDp0CBYWFggNDcX9+/elYcLDw3H69GkkJydj69atSE1NxYgRIyq7XCIiIiLZUAghRGVOMDo6Gj///DP279+v83UhBFxcXDB+/HhMmDABAJCTkwMnJyesXr0aAwYMwB9//AFPT08cPnwYfn5+AICkpCS8/PLL+Oeff+Di4lJmHWq1GiqVCjk5ObC2tq68BSQiIqIayT16W7G2S3PDqny+Fck0ld5jt2XLFvj5+aFv375wdHRE69at8cUXX0ivX7x4ERkZGQgJCZHaVCoV/P39kZaWBgBIS0uDjY2NFOoAICQkBEZGRjh06JDO+ebn50OtVms9iIiIiGqSSg92f/31F+Lj49GkSRPs3LkTI0eOxNixY7FmzRoAQEZGBgDAyclJazwnJyfptYyMDDg6Omq9bmJiAjs7O2mYJ8XGxkKlUkkPV1fXyl40IiIiIoNW6cFOo9GgTZs2mDNnDlq3bo0RI0bgnXfeQUJCQmXPSktMTAxycnKkR3p6epXOj4iIiMjQVHqwq1u3Ljw9PbXamjdvjsuXLwMAnJ2dAQCZmZlaw2RmZkqvOTs7IysrS+v1hw8f4tatW9IwT1IqlbC2ttZ6EBEREdUklR7sOnbsiLNnz2q1nTt3Dm5ubgAADw8PODs7IyUlRXpdrVbj0KFDCAgIAAAEBAQgOzsbR48elYbZvXs3NBoN/P39K7tkIiIiIlkwqewJfvDBB+jQoQPmzJmDfv364ddff8XKlSuxcuVKAIBCocC4ceMwa9YsNGnSBB4eHpg8eTJcXFzQu3dvAI96+F566SXpEG5BQQFGjx6NAQMGlOuKWCK50tcVWUREVD1UerBr27YtEhMTERMTgxkzZsDDwwOLFy9GeHi4NMykSZOQl5eHESNGIDs7G506dUJSUhLMzMykYb755huMHj0awcHBMDIyQp8+fbBkyZLKLpeIiIhINir9PnaGgvexIzlijx0Rkf7UyPvYEREREZF+MNgRERERyQSDHREREZFMMNgRERERyQSDHREREZFMMNgRERERyQSDHREREZFMMNgRERERyUSl//IEUU3EGwcTEZEhYLAjMlC6wiIREVFpeCiWiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkghdPkEHh1aVERERPjz12RERERDLBYEdEREQkEwx2RERERDLBc+xINqrz+Xm8GTEREVUGBjuiKlJSWKsuYZOIiKofBjsiKqY6934SEdVkDHakFzz0SEREVPmq/OKJuXPnQqFQYNy4cVLb/fv3ERkZCXt7e1haWqJPnz7IzMzUGu/y5csICwtD7dq14ejoiIkTJ+Lhw4dVXS4RERFRtVWlPXaHDx/GihUr4O3trdX+wQcfYNu2bfjuu++gUqkwevRovP766/j5558BAIWFhQgLC4OzszN++eUXXLt2DW+99RZq1aqFOXPmVGXJVAXYO2fYuH6IiOSjynrscnNzER4eji+++AK2trZSe05ODv71r39h4cKF6NatG3x9fbFq1Sr88ssvOHjwIABg165d+P333/H111/Dx8cHPXr0wMyZMxEXF4cHDx7onF9+fj7UarXWg4iIiKgmqbJgFxkZibCwMISEhGi1Hz16FAUFBVrtzZo1Q4MGDZCWlgYASEtLQ8uWLeHk5CQNExoaCrVajdOnT+ucX2xsLFQqlfRwdXWtgqUiIiIiMlxVcih2/fr1OHbsGA4fPlzstYyMDJiamsLGxkar3cnJCRkZGdIwj4e6oteLXtMlJiYGUVFR0nO1Ws1wJxO8QvPp8H0jIqp5Kj3Ypaen4/3330dycjLMzMwqe/IlUiqVUCqVz21+RE+rss9p4zlyRERUpNKD3dGjR5GVlYU2bdpIbYWFhUhNTcWyZcuwc+dOPHjwANnZ2Vq9dpmZmXB2dgYAODs749dff9WabtFVs0XDEJUHbxJMREQ1SaWfYxccHIyTJ0/i+PHj0sPPzw/h4eHS/2vVqoWUlBRpnLNnz+Ly5csICAgAAAQEBODkyZPIysqShklOToa1tTU8PT0ru2QiIiIiWaj0HjsrKyt4eXlptVlYWMDe3l5qHzZsGKKiomBnZwdra2uMGTMGAQEBaN++PQCge/fu8PT0xODBgzF//nxkZGTgk08+QWRkJA+3UqWoqeefPcthW/Z+EhEZPr388sSiRYtgZGSEPn36ID8/H6GhoVi+fLn0urGxMbZu3YqRI0ciICAAFhYWiIiIwIwZM/RRLqFmBKHyLiPPaSMiIkP1XILd3r17tZ6bmZkhLi4OcXFxJY7j5uaG7du3V3FlRFRd1IQ/LoiInlWV/6QYERERET0fejkUS1Rd8LArERFVJ+yxIyIiIpIJ9thRtcSeNCIiouIY7IjIoDC0ExE9PQY70sIvVSIiouqL59gRERERyQR77KhSscePKoLbCxFR5WKPHREREZFMsMeOiKqt8vb48RcqiKimYLAjomfCn/oiIjIcDHZEJHsMn0RUUzDY1RD8YiMiIpI/BrsajFckEhERyQuDHRE9F4b2hwR7sYlIjni7EyIiIiKZYI8dPTVD64EhIiKq6RjsiKjSMfQTEekHgx0R0f+UFEh57h0RVRc8x46IiIhIJthjR0RUhSpy9S2v1CWiZ1XpwS42NhY//vgjzpw5A3Nzc3To0AHz5s1D06ZNpWHu37+P8ePHY/369cjPz0doaCiWL18OJycnaZjLly9j5MiR2LNnDywtLREREYHY2FiYmDCLEpFhKu+5hTwHkYiqSqUfit23bx8iIyNx8OBBJCcno6CgAN27d0deXp40zAcffICffvoJ3333Hfbt24erV6/i9ddfl14vLCxEWFgYHjx4gF9++QVr1qzB6tWrMWXKlMoul4iIiEg2FEIIUZUzuH79OhwdHbFv3z4EBQUhJycHDg4OWLduHd544w0AwJkzZ9C8eXOkpaWhffv22LFjB1555RVcvXpV6sVLSEjAhx9+iOvXr8PU1LTYfPLz85Gfny89V6vVcHV1RU5ODqytratyEasF9hAQVS5dh0ir4nPGQ7FEhkNfp0uo1WqoVKpyZZoqv3giJycHAGBnZwcAOHr0KAoKChASEiIN06xZMzRo0ABpaWkAgLS0NLRs2VLr0GxoaCjUajVOnz6tcz6xsbFQqVTSw9XVtaoWiYiIiMggVWmw02g0GDduHDp27AgvLy8AQEZGBkxNTWFjY6M1rJOTEzIyMqRhHg91Ra8XvaZLTEwMcnJypEd6enolLw0RERGRYavSKxEiIyNx6tQpHDhwoCpnAwBQKpVQKpVVPh8iIoCnNxCRYaqyHrvRo0dj69at2LNnD+rXry+1Ozs748GDB8jOztYaPjMzE87OztIwmZmZxV4veo2IiIiIiqv0HjshBMaMGYPExETs3bsXHh4eWq/7+vqiVq1aSElJQZ8+fQAAZ8+exeXLlxEQEAAACAgIwOzZs5GVlQVHR0cAQHJyMqytreHp6VnZJcsOexKIqKbgr4UQaav0YBcZGYl169Zh8+bNsLKyks6JU6lUMDc3h0qlwrBhwxAVFQU7OztYW1tjzJgxCAgIQPv27QEA3bt3h6enJwYPHoz58+cjIyMDn3zyCSIjI3m4lYiIiKgElR7s4uPjAQBdunTRal+1ahWGDBkCAFi0aBGMjIzQp08frRsUFzE2NsbWrVsxcuRIBAQEwMLCAhEREZgxY0Zll0tEZNDK2wPPHioiAqroUGxZzMzMEBcXh7i4uBKHcXNzw/bt2yuzNCIi2eLPkRER8BzuY0dEREREzweDHREREZFMMNgRERERyUSV3qCYiIjoceU9F5DnDBI9HQY7IiLSK957k6jyMNgREdUwz6M3jGGNSD8Y7IiI6JkwxBEZDgY7IiKZetbAZWiBrSqWh+ftkdww2BERkcGFOCJ6Ogx21Rx3xkRExZV338hePJIb3seOiIiISCbYY0dERFSGknoA2btHhoY9dkREREQywR47IiKix/DcZarOGOyIiIie0rNcfFGRAMlDvlReDHZERESViFfa/j++F88fgx0REVEV0+fNlRmuahYGu2qC53wQEdHjnuV7oSJhj8GwemGwIyIiogphZ4PhYrAjIiKi5xbWeNFI1WKwIyIiomqNh5b/n0EHu7i4OCxYsAAZGRlo1aoVli5dinbt2um7LCIiInoOKvs8wprAYIPdhg0bEBUVhYSEBPj7+2Px4sUIDQ3F2bNn4ejoqO/yiIiISCbkFAIVQgih7yJ08ff3R9u2bbFs2TIAgEajgaurK8aMGYPo6Ogyx1er1VCpVMjJyYG1tXVVl/vU5LQxERER1TTP4zBuRTKNQfbYPXjwAEePHkVMTIzUZmRkhJCQEKSlpekcJz8/H/n5+dLznJwcAI/ejKrmNXVnlc+DiIiIDM/zyBlF8yhPX5xBBrsbN26gsLAQTk5OWu1OTk44c+aMznFiY2Mxffr0Yu2urq5VUiMRERGRavHzm9edO3egUqlKHcYgg93TiImJQVRUlPRco9Hg1q1bsLe3h0KhKHN8tVoNV1dXpKenG/Sh25qO66l64HqqHrieqgeup+qhKteTEAJ37tyBi4tLmcMaZLCrU6cOjI2NkZmZqdWemZkJZ2dnneMolUoolUqtNhsbmwrP29ramh+caoDrqXrgeqoeuJ6qB66n6qGq1lNZPXVFjCp9zpXA1NQUvr6+SElJkdo0Gg1SUlIQEBCgx8qIiIiIDJdB9tgBQFRUFCIiIuDn54d27dph8eLFyMvLw9ChQ/VdGhEREZFBMthg179/f1y/fh1TpkxBRkYGfHx8kJSUVOyCisqiVCoxderUYodzybBwPVUPXE/VA9dT9cD1VD0Yynoy2PvYEREREVHFGOQ5dkRERERUcQx2RERERDLBYEdEREQkEwx2RERERDLBYPc/cXFxcHd3h5mZGfz9/fHrr7/quyR6TGpqKnr27AkXFxcoFAps2rRJ3yWRDrGxsWjbti2srKzg6OiI3r174+zZs/oui54QHx8Pb29v6UaqAQEB2LFjh77LolLMnTsXCoUC48aN03cp9IRp06ZBoVBoPZo1a6a3ehjsAGzYsAFRUVGYOnUqjh07hlatWiE0NBRZWVn6Lo3+Jy8vD61atUJcXJy+S6FS7Nu3D5GRkTh48CCSk5NRUFCA7t27Iy8vT9+l0WPq16+PuXPn4ujRozhy5Ai6deuGXr164fTp0/oujXQ4fPgwVqxYAW9vb32XQiVo0aIFrl27Jj0OHDigt1p4uxMA/v7+aNu2LZYtWwbg0a9cuLq6YsyYMYiOjtZzdfQkhUKBxMRE9O7dW9+lUBmuX78OR0dH7Nu3D0FBQfouh0phZ2eHBQsWYNiwYfouhR6Tm5uLNm3aYPny5Zg1axZ8fHywePFifZdFj5k2bRo2bdqE48eP67sUAOyxw4MHD3D06FGEhIRIbUZGRggJCUFaWpoeKyOq/nJycgA8Cg1kmAoLC7F+/Xrk5eXxJxsNUGRkJMLCwrS+o8jwnD9/Hi4uLmjYsCHCw8Nx+fJlvdVisL888bzcuHEDhYWFxX7RwsnJCWfOnNFTVUTVn0ajwbhx49CxY0d4eXnpuxx6wsmTJxEQEID79+/D0tISiYmJ8PT01HdZ9Jj169fj2LFjOHz4sL5LoVL4+/tj9erVaNq0Ka5du4bp06cjMDAQp06dgpWV1XOvp8YHOyKqGpGRkTh16pRezzWhkjVt2hTHjx9HTk4Ovv/+e0RERGDfvn0MdwYiPT0d77//PpKTk2FmZqbvcqgUPXr0kP7v7e0Nf39/uLm5YePGjXo5taHGB7s6derA2NgYmZmZWu2ZmZlwdnbWU1VE1dvo0aOxdetWpKamon79+vouh3QwNTVF48aNAQC+vr44fPgwPv/8c6xYsULPlREAHD16FFlZWWjTpo3UVlhYiNTUVCxbtgz5+fkwNjbWY4VUEhsbG7zwwgu4cOGCXuZf48+xMzU1ha+vL1JSUqQ2jUaDlJQUnm9CVEFCCIwePRqJiYnYvXs3PDw89F0SlZNGo0F+fr6+y6D/CQ4OxsmTJ3H8+HHp4efnh/DwcBw/fpyhzoDl5ubizz//RN26dfUy/xrfYwcAUVFRiIiIgJ+fH9q1a4fFixcjLy8PQ4cO1Xdp9D+5ublaf/1cvHgRx48fh52dHRo0aKDHyuhxkZGRWLduHTZv3gwrKytkZGQAAFQqFczNzfVcHRWJiYlBjx490KBBA9y5cwfr1q3D3r17sXPnTn2XRv9jZWVV7NxUCwsL2Nvb85xVAzNhwgT07NkTbm5uuHr1KqZOnQpjY2MMHDhQL/Uw2AHo378/rl+/jilTpiAjIwM+Pj5ISkoqdkEF6c+RI0fQtWtX6XlUVBQAICIiAqtXr9ZTVfSk+Ph4AECXLl202letWoUhQ4Y8/4JIp6ysLLz11lu4du0aVCoVvL29sXPnTrz44ov6Lo2o2vnnn38wcOBA3Lx5Ew4ODujUqRMOHjwIBwcHvdTD+9gRERERyUSNP8eOiIiISC4Y7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7AzU3r17oVAosHfvXn2XotOlS5egUCiwevXqcg/76aefPtW8DP29eJ5Wr14NhUKBI0eOVNo0hwwZAnd390qbnr5UZJusSkXr6NKlS089bmWu34o4f/48unfvDpVKBYVCgU2bNumljqrUpUsXeHl5PfX4CoUCo0ePLnO4Z9kOSpKbm4vhw4fD2dkZCoUC48aNq7RpV7aK7PenTZsGhULxHKqqGWpMsCv6kD3+cHR0RNeuXbFjxw59l1eptm/fjmnTptWY+RLVRMuXL6/0EBsREYGTJ09i9uzZWLt2Lfz8/Cp1+vRs5syZg9WrV2PkyJFYu3YtBg8erO+S9Obq1auYNm0ajh8/ru9SDE6NCXZFZsyYgbVr1+Krr77CpEmTcP36dbz88svYunWrvkurNNu3b8f06dOrdB5ubm64d++e1o6lquYbFBSEe/fuISgoqNKnTVTZBg8ejHv37sHNza1K51PZwe7evXtIS0vDsGHDMHr0aAwaNAj169evtOnXNFWxHezevRvt27fH1KlTMWjQIPj6+lbatPXpk08+wb179yo0ztWrVzF9+nQGOx1M9F3A89ajRw+tv0KHDRsGJycnfPvtt3jllVeeefoajQYPHjyAmZnZM0/LkCkUiue2jEZGRgb9fj58+BAajQampqb6LoUMgLGxMYyNjfVdRoVdv34dAGBjY1Np08zLy4OFhUWlTa86qYrtICsrC56enmUOd//+fZiamsLIqHr03ZiYmMDEpHrFEUPe71ePtV6FbGxsYG5uXmyj+vTTT9GhQwfY29vD3Nwcvr6++P7774uNX3S+xTfffIMWLVpAqVQiKSmpxPm5u7vjlVdewa5du+Dj4wMzMzN4enrixx9/LFe93333HXx9fWFubo46depg0KBBuHLlivT6kCFDEBcXJ9VW9ChJVFQU7O3tIYSQ2saMGQOFQoElS5ZIbZmZmVAoFIiPjwdQ/Hym8s535cqVaNSoEZRKJdq2bYvDhw+Xucy6zrErOk/m999/R9euXVG7dm3Uq1cP8+fP16rZxMREZy/i2bNnoVAosGzZMqktOzsb48aNg6urK5RKJRo3box58+ZBo9FIwzx+3sjixYulZfn9998BAEuXLkWLFi1Qu3Zt2Nraws/PD+vWrdOa95UrV/D222/DyckJSqUSLVq0wL///e8y34fH3b17F++++y7s7e1hbW2Nt956C7dv39YaZvPmzQgLC4OLiwuUSiUaNWqEmTNnorCwsMzpV3T737RpE7y8vKTl0fUZuHLlCoYNGybV4+HhgZEjR+LBgwfSMOVZB0XDDRkyBCqVCjY2NoiIiEB2dnaZy5WdnQ1jY2OtbfvGjRswMjIq9jkYOXIknJ2dtcY/dOgQXnrpJahUKtSuXRudO3fGzz//rDWMrnOrNBoNpk2bBhcXF9SuXRtdu3bF77//Dnd3dwwZMqRYnfn5+YiKioKDgwMsLCzw2muvScELeLQfOX36NPbt2yd91rp06QIAKCgowPTp09GkSROYmZnB3t4enTp1QnJyconvy7Rp06SepYkTJ0KhUGidd/nbb7+hR48esLa2hqWlJYKDg3Hw4EGdy71v3z6MGjUKjo6OZfb45efnY+rUqWjcuDGUSiVcXV0xadIk5Ofnaw23atUqdOvWDY6OjlAqlfD09JT2RU/asWMHOnfuDCsrK1hbW6Nt27bFPoMASt13lMc333yDpk2bwszMDL6+vkhNTdX5fjy+HRTt/w8cOIB27drBzMwMDRs2xFdffVXqvIr2gRcvXsS2bdukdX7p0iXptfXr1+OTTz5BvXr1ULt2bajVagBlf2cAj/bflpaWuHz5Ml555RVYWlqiXr160j795MmT6NatGywsLODm5qbz/SxNWft9XefYJScno1OnTrCxsYGlpSWaNm2Kjz76SHo/2rZtCwAYOnSo9H483oNdnuUuGs7T0xNmZmbw8vJCYmJisfOOS9vvP3jwAFOmTIGvry9UKhUsLCwQGBiIPXv2aM3n8WnExcWhYcOGqF27Nrp374709HQIITBz5kzUr18f5ubm6NWrF27dulWh91kiaohVq1YJAOI///mPuH79usjKyhKnTp0S7777rjAyMhK7du3SGr5+/fpi1KhRYtmyZWLhwoWiXbt2AoDYunWr1nAARPPmzYWDg4OYPn26iIuLE7/99luJdbi5uYkXXnhB2NjYiOjoaLFw4ULRsmXLYjXs2bNHABB79uwptgxt27YVixYtEtHR0cLc3Fy4u7uL27dvCyGE+OWXX8SLL74oAIi1a9dKj5L8+OOPAoA4efKk1NaqVSthZGQk3njjDantu+++EwDEqVOnhBBCXLx4UQAQq1atKnO+RcO2bt1aNG7cWMybN0/Mnz9f1KlTR9SvX188ePCgxPpKei86d+4sXFxchKurq3j//ffF8uXLRbdu3QQAsX37dmm4bt26CU9Pz2LTnD59ujA2NhYZGRlCCCHy8vKEt7e3sLe3Fx999JFISEgQb731llAoFOL999+XxitaFk9PT9GwYUMxd+5csWjRIvH333+LlStXCgDijTfeECtWrBCff/65GDZsmBg7dqw0fkZGhqhfv75wdXUVM2bMEPHx8eLVV18VAMSiRYtKfR+E+P9toGXLliIwMFAsWbJEREZGCiMjIxEUFCQ0Go00bO/evUW/fv3EggULRHx8vOjbt68AICZMmKA1zYiICOHm5qbVVpHtv1WrVqJu3bpi5syZYvHixaJhw4aidu3a4saNG9JwV65cES4uLqJ27dpi3LhxIiEhQUyePFk0b95c2nbLuw40Go0ICgoSRkZGYtSoUWLp0qWiW7duwtvbW2ubLIm3t7fo06eP9DwxMVEYGRlpbd9CCNGiRQutz0BKSoowNTUVAQEB4rPPPhOLFi0S3t7ewtTUVBw6dKjYOrp48aLUNmnSJAFA9OzZUyxbtky88847on79+qJOnToiIiKi2LitW7cW3bp1E0uXLhXjx48XxsbGol+/flo1169fXzRr1kz6rBXtPz766COhUCjEO++8I7744gvx2WefiYEDB4q5c+eW+J6cOHFCLFq0SAAQAwcOFGvXrhWJiYlCCCFOnTolLCwspHU8d+5c4eHhIZRKpTh48GCx2j09PUXnzp3F0qVLS51nYWGh6N69u7RNrFixQowePVqYmJiIXr16aQ3btm1bMWTIELFo0SKxdOlS0b17dwFALFu2TGu4VatWCYVCIby8vMTs2bNFXFycGD58uBg8eLA0THn3HSUBILy8vESdOnXEjBkzxLx584Sbm5swNzfX2o/q2g7c3NxE06ZNhZOTk/joo4/EsmXLRJs2bYRCodDa9p6UkZEh1q5dK+rUqSN8fHykdZ6bmyvtHz09PYWPj49YuHChiI2NFXl5eeX6zhDi0T7AzMxMeHp6ivfee0/ExcWJDh06SJ8nFxcXMXHiRLF06VLRokULYWxsLP76669S36eK7PenTp0qHo8jp06dEqampsLPz098/vnnIiEhQUyYMEEEBQVJ78eMGTMEADFixAjp/fjzzz+13vuylnvr1q1CoVAIb29vsXDhQjF58mRha2srvLy8tPaJpe33r1+/LurWrSuioqJEfHy8mD9/vmjatKmoVauWVhYomoaPj4/w9PQUCxcuFJ988okwNTUV7du3Fx999JHo0KGDWLJkiRg7dqxQKBRi6NChpb7HJalxwe7Jh1KpFKtXry42/N27d7WeP3jwQHh5eYlu3bpptQMQRkZG4vTp0+Wqw83NTQAQP/zwg9SWk5Mj6tatK1q3bi21PRlmHjx4IBwdHYWXl5e4d++eNNzWrVsFADFlyhSpLTIyUutDUpqsrCwBQCxfvlwIIUR2drYwMjISffv2FU5OTtJwY8eOFXZ2dlJweDLYlTbfomHt7e3FrVu3pPbNmzcLAOKnn34qtcaSgh0A8dVXX0lt+fn5wtnZWetLe8WKFcWCqxBCeHp6aq3LmTNnCgsLC3Hu3Dmt4aKjo4WxsbG4fPmy1rJYW1uLrKwsrWF79eolWrRoUeqyDBs2TNStW1cr9AghxIABA4RKpSq23T2paDv29fXV2jHOnz9fABCbN2+W2nRN69133xW1a9cW9+/fl9p0BbuKbP+mpqbiwoULUtuJEycEALF06VKp7a233hJGRkbi8OHDxWoq2qbKuw42bdokAIj58+dLwzx8+FAEBgaWK9hFRkZqbdtRUVEiKChIODo6ivj4eCGEEDdv3hQKhUJ8/vnnUo1NmjQRoaGhWuH57t27wsPDQ7z44otS25Nf6BkZGcLExET07t1bq45p06YJADqDXUhIiNZ8PvjgA2FsbCyys7OlthYtWojOnTsXW75WrVqJsLCwUt8DXYq27QULFmi19+7dW5iamkpfmkIIcfXqVWFlZSV90T5ee6dOncTDhw/LnN/atWuFkZGR2L9/v1Z7QkKCACB+/vlnqU3XthwaGioaNmwoPc/OzhZWVlbC399fax8phNB6L8u77yhJ0XfHkSNHpLa///5bmJmZiddee01qKynYARCpqalSW1ZWllAqlWL8+PFlztvNza3Yui3aPzZs2FDrfarId0ZERIQAIObMmSO13b59W5ibmwuFQiHWr18vtZ85c0YAEFOnTi211ors958MdkV/ZFy/fr3E6R8+fFjn570iy92yZUtRv359cefOHalt7969AoDOYKdrv//w4UORn5+v1Xb79m3h5OQk3n777WLTcHBw0Pocx8TESH8gFxQUSO0DBw4UpqamWvvq8qpxh2Lj4uKQnJyM5ORkfP311+jatSuGDx9e7FCoubm59P/bt28jJycHgYGBOHbsWLFpdu7cuVznPRRxcXHBa6+9Jj0vOpT222+/ISMjQ+c4R44cQVZWFkaNGqV1vllYWBiaNWuGbdu2lXv+j3NwcECzZs2kwwg///wzjI2NMXHiRGRmZuL8+fMAgP3796NTp07PdEl6//79YWtrKz0PDAwEAPz1119PNT1LS0sMGjRIem5qaop27dppTe/111+HiYkJNmzYILWdOnUKv//+O/r37y+1fffddwgMDIStrS1u3LghPUJCQlBYWFjsMEufPn3g4OCg1WZjY4N//vmnxMPLQgj88MMP6NmzJ4QQWvMJDQ1FTk6Ozu1LlxEjRqBWrVrS85EjR8LExATbt2+X2h7fhu/cuYMbN24gMDAQd+/exZkzZ0qdfkW2/5CQEDRq1Eh67u3tDWtra2k9aDQabNq0CT179tR5lWXRNlXedbB9+3aYmJhg5MiR0jSMjY0xZsyYUpepSGBgIDIzM3H27FkAj7btoKAgBAYGYv/+/QCAAwcOQAghbaPHjx/H+fPn8eabb+LmzZtSbXl5eQgODkZqamqxw8VFUlJS8PDhQ4waNUqrvbR6R4wYofVZCwwMRGFhIf7+++8yl8/GxganT5+WPrvPorCwELt27ULv3r3RsGFDqb1u3bp48803ceDAAemQX5F33nmnXOeWfffdd2jevDmaNWumtb67desGAFqHsh7fHnNycnDjxg107twZf/31F3JycgA8OnR3584dREdHFzsn98n9Vnn2HaUJCAjQunChQYMG6NWrF3bu3FnmqQ6enp7SdgU82gc3bdr0qfeDRSIiIrTep6f5zhg+fLj0fxsbGzRt2hQWFhbo16+f1N60aVPY2NiUu96n2e8Xnee5efPmEj9XJSnvcl+9ehUnT57EW2+9BUtLS2m4zp07o2XLljqnrWu/b2xsLJ1np9FocOvWLTx8+BB+fn4695d9+/aFSqWSnvv7+wMABg0apHVKmL+/Px48eKDz8HFZalywa9euHUJCQhASEoLw8HBs27YNnp6eGD16tNa5Plu3bkX79u1hZmYGOzs7ODg4ID4+XtqJPM7Dw6NCNTRu3LjYjuaFF14AgBLveVS0Q2/atGmx15o1a1auHX5JHv9C279/P/z8/ODn5wc7Ozvs378farUaJ06c0NoZPY0GDRpoPS/6sD95blh51a9fv9j7aGtrqzW9OnXqIDg4GBs3bpTaNmzYABMTE7z++utS2/nz55GUlAQHBwetR0hICIBHJy0/Ttc6//DDD2FpaYl27dqhSZMmiIyM1Dr/6vr168jOzsbKlSuLzWfo0KFa88nIyNB6PHnFWJMmTbSeW1paom7dulrbz+nTp/Haa69BpVLB2toaDg4O0peZru34cRXZ/p9cr4D2erh+/TrUanWZ9w4r7zr4+++/UbduXa2dMaD7s6FL0Xa8f/9+5OXl4bfffkNgYCCCgoK0PgfW1tZo1aqVVBvw6Mvzyfq+/PJL5Ofnl/ieFn02GzdurNVuZ2en9YX3uGf5rMyYMQPZ2dl44YUX0LJlS0ycOBH//e9/yxxPl+vXr+Pu3bs639vmzZtDo9EgPT1dq728+8Pz58/j9OnTxd7Pon3h45+5n3/+GSEhIbCwsICNjQ0cHByk862K3vc///wTAMp1j7ry7DtK8+TnD3i0D797967WuZC6lPV5eVpPvu8V/c4wMzMrFlpUKpXO90qlUpW73qfZlvv374+OHTti+PDhcHJywoABA7Bx48ZyhbzyLndJn8uS2oCSt+01a9bA29tbOqfVwcEB27ZtK9f+sijkubq66mx/mu2iel2GUgWMjIzQtWtXfP755zh//jxatGiB/fv349VXX0VQUBCWL1+OunXrolatWli1apXOk0Yf/yupOurUqRO++OIL/PXXX9i/fz8CAwOhUCjQqVMn7N+/Hy4uLtBoNM8c7Er6K148dsJ6VUxvwIABGDp0KI4fPw4fHx9s3LgRwcHBqFOnjjSMRqPBiy++iEmTJumcZtGXTRFd67x58+Y4e/Ystm7diqSkJPzwww9Yvnw5pkyZgunTp0s7pUGDBiEiIkLnfLy9vQE86hF53KpVq3SeZF+S7OxsdO7cGdbW1pgxYwYaNWoEMzMzHDt2DB9++GGpO8iKbv+VtV4rug6elouLCzw8PJCamgp3d3cIIRAQEAAHBwe8//77+Pvvv7F//3506NBBuqqw6P1asGABfHx8dE73yaD5LJ7lPQ0KCsKff/6JzZs3Y9euXfjyyy+xaNEiJCQkaPXIVJXy7g81Gg1atmyJhQsX6ny96Ivuzz//RHBwMJo1a4aFCxfC1dUVpqam2L59OxYtWlThHh2g8vdFhjDvZ/0eKqmuZ633acY3NzdHamoq9uzZg23btiEpKQkbNmxAt27dsGvXLr1dda7rPf76668xZMgQ9O7dGxMnToSjoyOMjY0RGxsr/bHxuKp6nx9X44Md8OiyZeDRXb0B4IcffoCZmRl27twJpVIpDbdq1apKmd+FCxcghND6K+jcuXMAUOIvABRdsXb27FnpUEWRs2fPat0rqaKHS4sCW3JyMg4fPozo6GgAj74g4uPj4eLiAgsLizLvmWSodw7v3bs33n33Xelw7Llz5xATE6M1TKNGjZCbmyv1Dj0tCwsL9O/fH/3798eDBw/w+uuvY/bs2YiJiYGDgwOsrKxQWFhY5nyevIKxRYsWWs/Pnz+Prl27Ss9zc3Nx7do1vPzyywAeXTV28+ZN/Pjjj1r3/7t48WKZy1DZ27+DgwOsra1x6tSpUocr7zpwc3NDSkoKcnNztcJU0aHV8ggMDERqaio8PDzg4+MDKysrtGrVCiqVCklJSTh27JjW1dRFh5qtra0rvI0UfTYvXLig9Rf/zZs3n6mXprTPm52dHYYOHYqhQ4ciNzcXQUFBmDZtWoWDnYODA2rXrq3zvT1z5gyMjIyK9TSUV6NGjXDixAkEBweXuiw//fQT8vPzsWXLFq3ejievOixaR6dOnSqxx6Wy6DrMfe7cOdSuXbtYr5e+VOQ7wxAZGRkhODgYwcHBWLhwIebMmYOPP/4Ye/bsQUhISInbTHmX+/HP5ZN0tZXk+++/R8OGDfHjjz9q1TR16tRyT6Oy1bhDsU8qKCjArl27YGpqiubNmwN4lJwVCoXWuRKXLl2qtJ/XuXr1KhITE6XnarUaX331FXx8fIrdXqGIn58fHB0dkZCQoHUrgB07duCPP/5AWFiY1FZ036jy3P4BeNS9XK9ePSxatAgFBQXo2LEjgEdffn/++Se+//57tG/fvsz7DFV0vs+LjY0NQkNDsXHjRqxfvx6mpqbo3bu31jD9+vVDWloadu7cWWz87OxsKfyX5ubNm1rPTU1N4enpCSEECgoKYGxsjD59+uCHH37QGXIeP4RTdLpA0ePJHryVK1eioKBAeh4fH4+HDx+iR48eAP7/r7/H/9p78OABli9fXuZyVPb2b2RkhN69e+Onn37S+VNZRTWWdx28/PLLePjwodbtLgoLC7F06dJy1xQYGIhLly5hw4YN0h82RkZG6NChAxYuXIiCggKtHmpfX180atQIn376qfQH4ONKO/wWHBwMExOTYrfnePxWO0/DwsJC52ftye3Q0tISjRs3LnYLkfIwNjZG9+7dsXnzZq3D/JmZmVi3bh06deoEa2vrCk8XeLS+r1y5gi+++KLYa/fu3UNeXp5UA6C9Lefk5BT7Q6N79+6wsrJCbGws7t+/r/VaZffEpaWlaZ0/lZ6ejs2bN6N79+4Gcw/DinxnGBpdt/ko6ikvWpaSvm/Ku9wuLi7w8vLCV199pfWZ3rdvH06ePFnuWnVtn4cOHUJaWlq5p1HZalyP3Y4dO6QTx7OysrBu3TqcP38e0dHR0g4qLCwMCxcuxEsvvYQ333wTWVlZiIuLQ+PGjZ/6XJXHvfDCCxg2bBgOHz4MJycn/Pvf/0ZmZmapPSK1atXCvHnzMHToUHTu3BkDBw5EZmYmPv/8c7i7u+ODDz6Qhi3qWRs7dixCQ0NhbGyMAQMGlFpTYGAg1q9fj5YtW0rnQLRp0wYWFhY4d+4c3nzzzTKX62nm+7z0798fgwYNwvLlyxEaGlrsJqwTJ07Eli1b8Morr2DIkCHw9fVFXl4eTp48ie+//x6XLl3SOnSrS/fu3eHs7IyOHTvCyckJf/zxB5YtW4awsDBYWVkBAObOnYs9e/bA398f77zzDjw9PXHr1i0cO3YM//nPf8p936IHDx4gODgY/fr1w9mzZ7F8+XJ06tQJr776KgCgQ4cOsLW1RUREBMaOHQuFQoG1a9eW6wuuKrb/OXPmYNeuXejcuTNGjBiB5s2b49q1a/juu+9w4MAB2NjYlHsd9OzZEx07dkR0dDQuXbok3QeyrPMGH1cU2s6ePYs5c+ZI7UFBQdixY4d0v60iRkZG+PLLL9GjRw+0aNECQ4cORb169XDlyhXs2bMH1tbW+Omnn3TOy8nJCe+//z4+++wzvPrqq3jppZdw4sQJ7NixA3Xq1Hnqnm5fX1/Ex8dj1qxZaNy4MRwdHdGtWzd4enqiS5cu8PX1hZ2dHY4cOYLvv/++XL9vqsusWbOke4qNGjUKJiYmWLFiBfLz8yt877fHDR48GBs3bsR7772HPXv2oGPHjigsLMSZM2ewceNG7Ny5E35+fujevTtMTU3Rs2dPvPvuu8jNzcUXX3wBR0dHXLt2TZqetbU1Fi1ahOHDh6Nt27Z48803YWtrixMnTuDu3btYs2bNU9f6JC8vL4SGhmLs2LFQKpXSH0xV/Ys/FVGR7wxDM2PGDKSmpiIsLAxubm7IysrC8uXLUb9+fXTq1AnAox5aGxsbJCQkwMrKChYWFvD394eHh0e5l3vOnDno1asXOnbsiKFDh+L27dtYtmwZvLy8dP4Bp8srr7yCH3/8Ea+99hrCwsJw8eJFJCQkwNPTs9zTqHQVvo62mtJ1uxMzMzPh4+Mj4uPjtS6HF0KIf/3rX6JJkyZCqVSKZs2aiVWrVhW7JFuIR5e+R0ZGlruOosvVd+7cKby9vaXpf/fdd1rD6brFhxBCbNiwQbRu3VoolUphZ2cnwsPDxT///KM1zMOHD8WYMWOEg4ODUCgU5br1SVxcnAAgRo4cqdUeEhIiAIiUlBStdl23OylpviXdRkEIUa7L5ku63YmuW4vounWHEEKo1Wphbm4uAIivv/5a53zu3LkjYmJiROPGjYWpqamoU6eO6NChg/j000+lW4uUtiwrVqwQQUFBwt7eXiiVStGoUSMxceJEkZOTozVcZmamiIyMFK6urqJWrVrC2dlZBAcHi5UrV5b6Pgjx/9vxvn37xIgRI4Stra2wtLQU4eHh4ubNm1rD/vzzz6J9+/bC3NxcuLi4iEmTJomdO3cWey91vWfPuv27ublp3cZDiEe3hHjrrbeEg4ODUCqVomHDhiIyMlLrVgHlWQdCPLodyeDBg4W1tbVQqVRi8ODB4rfffivX7U6KODo6CgAiMzNTajtw4IAAIAIDA3WO89tvv4nXX39dWsdubm6iX79+Wp8PXbe5ePjwoZg8ebJwdnYW5ubmolu3buKPP/4Q9vb24r333is27pO3hdH1GcjIyBBhYWHCyspKAJBufTJr1izRrl07YWNjI8zNzUWzZs3E7Nmzy7xfZGnb9rFjx0RoaKiwtLQUtWvXFl27dhW//PKL1jAl1V6aBw8eiHnz5okWLVoIpVIpbG1tha+vr5g+fbrW52bLli3C29tbmJmZCXd3dzFv3jzx73//u9j7XDRshw4dhLm5ubC2thbt2rUT3377rfR6RfcdTyra5r/++mvpM9K6deti++qSbnei61Y0nTt31nnrmieVdruTJ79DipTnOyMiIkJYWFjorEvXe1XScjyuIvv9J/ctKSkpolevXsLFxUWYmpoKFxcXMXDgwGK3Qtq8ebPw9PQUJiYmxT775VluIYRYv369aNasmVAqlcLLy0ts2bJF9OnTRzRr1qxcy6LRaMScOXOEm5ubtC1s3bq12PZU0jRKWn9P83kqohDiOZwtShJ3d3d4eXnJ6rdpiajisrOzYWtri1mzZuHjjz/WdzlE9D8+Pj5wcHAo9ddaDFmNP8eOiKiq6fqB88WLFwOA9FNgRPR8FRQUFDt/eu/evThx4kS1/lzWuHPsiIietw0bNmD16tV4+eWXYWlpiQMHDuDbb79F9+7dpYuViOj5unLlCkJCQjBo0CC4uLjgzJkzSEhIgLOzM9577z19l/fUGOyIiKqYt7c3TExMMH/+fKjVaumCilmzZum7NKIay9bWFr6+vvjyyy9x/fp1WFhYICwsDHPnzoW9vb2+y3tqPMeOiIiISCZ4jh0RERGRTDDYEREREcmEbM+x02g0uHr1KqysrAz2p66IiIiIyiKEwJ07d+Di4iL9hnVJZBvsrl69+tS/YUhERERkaNLT01G/fv1Sh5FtsCv6Caf09PSn/i1DIiIiIn1Tq9VwdXWVsk1p9BLsUlNTsWDBAhw9ehTXrl1DYmKi1o+yCyEwdepUfPHFF8jOzkbHjh0RHx+PJk2alHseRYdfra2tGeyIiIio2ivPqWV6uXgiLy8PrVq1QlxcnM7X58+fjyVLliAhIQGHDh2ChYUFQkNDcf/+/edcKREREVH1oZceux49eqBHjx46XxNCYPHixfjkk0/Qq1cvAMBXX30FJycnbNq0CQMGDHiepRIRERFVGwZ3u5OLFy8iIyMDISEhUptKpYK/vz/S0tJKHC8/Px9qtVrrQURERFSTGFywy8jIAAA4OTlptTs5OUmv6RIbGwuVSiU9eEUsERER1TQGF+yeVkxMDHJycqRHenq6vksiIiIieq4M7nYnzs7OAIDMzEzUrVtXas/MzISPj0+J4ymVSiiVyqouTyf36G3F2i7NDdNDJURERFSTGVyPnYeHB5ydnZGSkiK1qdVqHDp0CAEBAXqsjIiIiMiw6aXHLjc3FxcuXJCeX7x4EcePH4ednR0aNGiAcePGYdasWWjSpAk8PDwwefJkuLi4aN3rjoiIiIi06SXYHTlyBF27dpWeR0VFAQAiIiKwevVqTJo0CXl5eRgxYgSys7PRqVMnJCUlwczMTB/lEhEREVULCiGE0HcRVUGtVkOlUiEnJ6fKf3mC59gRERFRValIpjG4c+yIiIiI6Okw2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJBIMdERERkUww2BERERHJhEEGu8LCQkyePBkeHh4wNzdHo0aNMHPmTAgh9F0aERERkcEy0XcBusybNw/x8fFYs2YNWrRogSNHjmDo0KFQqVQYO3asvssjIiIiMkgGGex++eUX9OrVC2FhYQAAd3d3fPvtt/j111/1XBkRERGR4TLIQ7EdOnRASkoKzp07BwA4ceIEDhw4gB49epQ4Tn5+PtRqtdaDiIiIqCYxyB676OhoqNVqNGvWDMbGxigsLMTs2bMRHh5e4jixsbGYPn36c6ySiIiIyLAYZI/dxo0b8c0332DdunU4duwY1qxZg08//RRr1qwpcZyYmBjk5ORIj/T09OdYMREREZH+GWSP3cSJExEdHY0BAwYAAFq2bIm///4bsbGxiIiI0DmOUqmEUql8nmUSERERGRSD7LG7e/cujIy0SzM2NoZGo9FTRURERESGzyB77Hr27InZs2ejQYMGaNGiBX777TcsXLgQb7/9tr5LIyIiIjJYBhnsli5dismTJ2PUqFHIysqCi4sL3n33XUyZMkXfpREREREZLIMMdlZWVli8eDEWL16s71KIiIiIqg2DPMeOiIiIiCqOwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGSCwY6IiIhIJhjsiIiIiGTCRN8FyJV79LZibZfmhumhEiIiIqop2GNHREREJBMMdkREREQywWBHREREJBMMdkREREQywWBHREREJBMMdkREREQyYbDB7sqVKxg0aBDs7e1hbm6Oli1b4siRI/oui4iIiMhgGeR97G7fvo2OHTuia9eu2LFjBxwcHHD+/HnY2trquzQiIiIig2WQwW7evHlwdXXFqlWrpDYPDw89VkRERERk+AzyUOyWLVvg5+eHvn37wtHREa1bt8YXX3xR6jj5+flQq9VaDyIiIqKaxCCD3V9//YX4+Hg0adIEO3fuxMiRIzF27FisWbOmxHFiY2OhUqmkh6ur63OsmIiIiEj/FEIIoe8inmRqago/Pz/88ssvUtvYsWNx+PBhpKWl6RwnPz8f+fn50nO1Wg1XV1fk5OTA2tq6SuvV9buwuvC3YomIiKii1Go1VCpVuTKNQfbY1a1bF56enlptzZs3x+XLl0scR6lUwtraWutBREREVJMYZLDr2LEjzp49q9V27tw5uLm56akiIiIiIsNnkMHugw8+wMGDBzFnzhxcuHAB69atw8qVKxEZGanv0oiIiIgMlkEGu7Zt2yIxMRHffvstvLy8MHPmTCxevBjh4eH6Lo2IiIjIYBnkfewA4JVXXsErr7yi7zKIiIiIqg2D7LEjIiIioopjsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCRN9F1CTuEdvK9Z2aW6YHiohIiIiOWKPHREREZFMMNgRERERyQSDHREREZFMMNgRERERyQSDHREREZFMMNgRERERyQSDHREREZFMMNgRERERyQSDHREREZFMMNgRERERyUS1CHZz586FQqHAuHHj9F0KERERkcEy+GB3+PBhrFixAt7e3vouhYiIiMigGXSwy83NRXh4OL744gvY2tqWOmx+fj7UarXWg4iIiKgmMehgFxkZibCwMISEhJQ5bGxsLFQqlfRwdXV9DhUSERERGQ6DDXbr16/HsWPHEBsbW67hY2JikJOTIz3S09OruEIiIiIiw2Ki7wJ0SU9Px/vvv4/k5GSYmZmVaxylUgmlUlnFlREREREZLoMMdkePHkVWVhbatGkjtRUWFiI1NRXLli1Dfn4+jI2N9VghERERkeExyGAXHByMkydParUNHToUzZo1w4cffshQR0RERKSDQQY7KysreHl5abVZWFjA3t6+WDsRERERPWKQwa4mcY/eVqzt0twwPVRCRERE1V21CXZ79+7VdwlEREREBs1gb3dCRERERBXDYEdEREQkEwx2RERERDLBYEdEREQkEwx2RERERDLBYEdEREQkEwx2RERERDLBYEdEREQkEwx2RERERDLBYEdEREQkEwx2RERERDLBYEdEREQkEwx2RERERDLBYEdEREQkEyb6LoCIiIjoeXGP3las7dLcMD1UUjUY7IiIiEh2dAW4moDBzgDJ/a8JIiKiylRTQ5wuDHZERERUbTDElY4XTxARERHJBHvsiIiIyCCxd67i2GNHREREJBMMdkREREQyYZCHYmNjY/Hjjz/izJkzMDc3R4cOHTBv3jw0bdpU36URERFRFeBh18phkD12+/btQ2RkJA4ePIjk5GQUFBSge/fuyMvL03dpRERERAbLIHvskpKStJ6vXr0ajo6OOHr0KIKCgnSOk5+fj/z8fOm5Wq2u0hqJiIio4tgzV7UMssfuSTk5OQAAOzu7EoeJjY2FSqWSHq6urs+rPCIiIiKDYJA9do/TaDQYN24cOnbsCC8vrxKHi4mJQVRUlPRcrVYz3BEREZVTRXrSyvtrSOyde/4MPthFRkbi1KlTOHDgQKnDKZVKKJXK51QVERFR9VAV4YqBzXAZdLAbPXo0tm7ditTUVNSvX1/f5RARERk0Bi4yyGAnhMCYMWOQmJiIvXv3wsPDQ98l6V1JH9bydocTEZG8MMSRLgYZ7CIjI7Fu3Tps3rwZVlZWyMjIAACoVCqYm5vruToiIqKqwbBGz8ogg118fDwAoEuXLlrtq1atwpAhQ55/QURERM+AgY2eF4MMdkIIfZdARERUJgY2MjQGGeyIiIgMCQMcAbq3A0M7171a3KCYiIiIiMrGHjsiIqLHsHeOgOq7HTDYERFRjVVdv7ypcslpO2Cwq+aqw/F+IiIiej4Y7IiI6Jl6LAzxj0k59cAQVQSDHRFRDVPZoacqQpSusMiwRlQ2BjsiIpmqzkGoOtdOpE8MdjLE8+6Iah4GISICGOyIiKodhjgiKgmDHRGRgWBgI6JnxWBXQ/DwLFHZGKyIqLpjsKvBGPaopmBgI6KagsGOtDzrFyCDYc3D0EREZDgY7KhSlfdLvioC4PMKGDUhvDKsERFVTwx2pBfVOThU59qJiEjejPRdABERERFVDgY7IiIiIplgsCMiIiKSCQY7IiIiIplgsCMiIiKSCQY7IiIiIpkw6GAXFxcHd3d3mJmZwd/fH7/++qu+SyIiIiIyWAYb7DZs2ICoqChMnToVx44dQ6tWrRAaGoqsrCx9l0ZERERkkAz2BsULFy7EO++8g6FDhwIAEhISsG3bNvz73/9GdHR0seHz8/ORn58vPc/JyQEAqNXqKq9Vk3+3yudBREREhud55IyieQghyhzWIIPdgwcPcPToUcTExEhtRkZGCAkJQVpams5xYmNjMX369GLtrq6uVVYnERER1Wyqxc9vXnfu3IFKpSp1GIMMdjdu3EBhYSGcnJy02p2cnHDmzBmd48TExCAqKkp6rtFocOvWLdjb20OhUJQ5T7VaDVdXV6Snp8Pa2vrZFoCqDNdT9cD1VD1wPVUPXE/VQ1WuJyEE7ty5AxcXlzKHNchg9zSUSiWUSqVWm42NTYWnY21tzQ9ONcD1VD1wPVUPXE/VA9dT9VBV66msnroiBnnxRJ06dWBsbIzMzEyt9szMTDg7O+upKiIiIiLDZpDBztTUFL6+vkhJSZHaNBoNUlJSEBAQoMfKiIiIiAyXwR6KjYqKQkREBPz8/NCuXTssXrwYeXl50lWylU2pVGLq1KnFDueSYeF6qh64nqoHrqfqgeupejCU9aQQ5bl2Vk+WLVuGBQsWICMjAz4+PliyZAn8/f31XRYRERGRQTLoYEdERERE5WeQ59gRERERUcUx2BERERHJBIMdERERkUww2BERERHJBIPd/8TFxcHd3R1mZmbw9/fHr7/+qu+S6DGpqano2bMnXFxcoFAosGnTJn2XRDrExsaibdu2sLKygqOjI3r37o2zZ8/quyx6Qnx8PLy9vaU75AcEBGDHjh36LotKMXfuXCgUCowbN07fpdATpk2bBoVCofVo1qyZ3uphsAOwYcMGREVFYerUqTh27BhatWqF0NBQZGVl6bs0+p+8vDy0atUKcXFx+i6FSrFv3z5ERkbi4MGDSE5ORkFBAbp37468vDx9l0aPqV+/PubOnYujR4/iyJEj6NatG3r16oXTp0/ruzTS4fDhw1ixYgW8vb31XQqVoEWLFrh27Zr0OHDggN5q4e1OAPj7+6Nt27ZYtmwZgEe/cuHq6ooxY8YgOjpaz9XRkxQKBRITE9G7d299l0JluH79OhwdHbFv3z4EBQXpuxwqhZ2dHRYsWIBhw4bpuxR6TG5uLtq0aYPly5dj1qxZ8PHxweLFi/VdFj1m2rRp2LRpE44fP67vUgCwxw4PHjzA0aNHERISIrUZGRkhJCQEaWlpeqyMqPrLyckB8Cg0kGEqLCzE+vXrkZeXx59sNECRkZEICwvT+o4iw3P+/Hm4uLigYcOGCA8Px+XLl/VWi8H+pNjzcuPGDRQWFsLJyUmr3cnJCWfOnNFTVUTVn0ajwbhx49CxY0d4eXnpuxx6wsmTJxEQEID79+/D0tISiYmJ8PT01HdZ9Jj169fj2LFjOHz4sL5LoVL4+/tj9erVaNq0Ka5du4bp06cjMDAQp06dgpWV1XOvp8YHOyKqGpGRkTh16pRezzWhkjVt2hTHjx9HTk4Ovv/+e0RERGDfvn0MdwYiPT0d77//PpKTk2FmZqbvcqgUPXr0kP7v7e0Nf39/uLm5YePGjXo5taHGB7s6derA2NgYmZmZWu2ZmZlwdnbWU1VE1dvo0aOxdetWpKamon79+vouh3QwNTVF48aNAQC+vr44fPgwPv/8c6xYsULPlREAHD16FFlZWWjTpo3UVlhYiNTUVCxbtgz5+fkwNjbWY4VUEhsbG7zwwgu4cOGCXuZf48+xMzU1ha+vL1JSUqQ2jUaDlJQUnm9CVEFCCIwePRqJiYnYvXs3PDw89F0SlZNGo0F+fr6+y6D/CQ4OxsmTJ3H8+HHp4efnh/DwcBw/fpyhzoDl5ubizz//RN26dfUy/xrfYwcAUVFRiIiIgJ+fH9q1a4fFixcjLy8PQ4cO1Xdp9D+5ublaf/1cvHgRx48fh52dHRo0aKDHyuhxkZGRWLduHTZv3gwrKytkZGQAAFQqFczNzfVcHRWJiYlBjx490KBBA9y5cwfr1q3D3r17sXPnTn2XRv9jZWVV7NxUCwsL2Nvb85xVAzNhwgT07NkTbm5uuHr1KqZOnQpjY2MMHDhQL/Uw2AHo378/rl+/jilTpiAjIwM+Pj5ISkoqdkEF6c+RI0fQtWtX6XlUVBQAICIiAqtXr9ZTVfSk+Ph4AECXLl202letWoUhQ4Y8/4JIp6ysLLz11lu4du0aVCoVvL29sXPnTrz44ov6Lo2o2vnnn38wcOBA3Lx5Ew4ODujUqRMOHjwIBwcHvdTD+9gRERERyUSNP8eOiIiISC4Y7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCYY7IiIiIhkgsGOiIiISCb+D0xsoViBF0NVAAAAAElFTkSuQmCC",
-      "text/plain": [
-       "<Figure size 640x480 with 2 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "lds = True\n",
-    "kernel = \"gaussian\"\n",
-    "ks = 5\n",
-    "sigma = 2\n",
-    "reweight = \"sqrt\"\n",
-    "Y = df[\"MedHouseVal\"].values\n",
-    "lds_y_max = None\n",
-    "lds_y_min = None\n",
-    "granularity = 100\n",
-    "\n",
-    "fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1)\n",
-    "\n",
-    "y_max = max(Y) if lds_y_max is None else lds_y_max\n",
-    "y_min = min(Y) if lds_y_min is None else lds_y_min\n",
-    "bin_edges = np.linspace(y_min, y_max, num=granularity, endpoint=True)\n",
-    "value_dict = dict(zip(bin_edges[:-1], np.histogram(Y, bin_edges)[0]))\n",
-    "\n",
-    "ax1.set_title(\"Histogram of values in the Y\")\n",
-    "ax1.bar(\n",
-    "    value_dict.keys(),\n",
-    "    value_dict.values(),\n",
-    "    width=list(value_dict.keys())[1] - list(value_dict.keys())[0],\n",
-    ")\n",
-    "\n",
-    "if reweight:\n",
-    "    value_dict = dict(zip(value_dict.keys(), np.sqrt(list(value_dict.values()))))\n",
-    "\n",
-    "if kernel is not None:\n",
-    "    lds_kernel_window = get_kernel_window(kernel, ks, sigma)\n",
-    "    smoothed_values = convolve1d(\n",
-    "        list(value_dict.values()), weights=lds_kernel_window, mode=\"constant\"\n",
-    "    )\n",
-    "    weigths = sum(smoothed_values) / (len(smoothed_values) * smoothed_values)\n",
-    "else:\n",
-    "    values = list(value_dict.values())\n",
-    "    weigths = sum(values) / (len(values) * values)  # type: ignore[operator]\n",
-    "value_dict = dict(zip(value_dict.keys(), weigths))\n",
-    "\n",
-    "left_bin_edges = find_bin(bin_edges, Y)\n",
-    "weights = np.array([value_dict[edge] for edge in left_bin_edges], dtype=\"float32\")\n",
-    "\n",
-    "\n",
-    "ax2.set_title(\"Bar plot with inverse-balanced weights for each bin from histogram\")\n",
-    "ax2.bar(\n",
-    "    value_dict.keys(),\n",
-    "    value_dict.values(),\n",
-    "    width=list(value_dict.keys())[1] - list(value_dict.keys())[0],\n",
-    ")\n",
-    "fig.tight_layout()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b472dec0",
-   "metadata": {},
-   "source": [
-    "# Feature Distribution Smoothing\n",
-    "We use dataset feature values in this example, but during the training process features tensors are the ouput of last layer before FDS layer.\n",
-    "* labels are np.vstack-ed to reflect normal training scenario"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "36fe33d0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "features = torch.tensor(df.drop(columns=[\"MedHouseVal\"]).values)\n",
-    "labels = torch.tensor(np.vstack(df[\"MedHouseVal\"].values))\n",
-    "FDS = fds_layer.FDSLayer(feature_dim=features.size(1))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "ec0310c7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "for epoch in range(3):\n",
-    "    FDS.update_last_epoch_stats(epoch)\n",
-    "    FDS.update_running_stats(torch.clone(features).detach(), labels, epoch)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "802278aa",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkEAAAGzCAYAAAAotsMiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAACVZklEQVR4nO2dd3wUdfrHP1uym94bCQESUOkSQJoNBUVFTywoiifYEIXjRL1TzzvbHaJ4enY9z1P4iZ6iomc5C1KsCAgEpHcSAiEJIb1sm98fu9/Z2b6bbEvyeb9eeUFmZ2dmJ7s7n3mez/M8KkmSJBBCCCGEdDPUkT4AQgghhJBIQBFECCGEkG4JRRAhhBBCuiUUQYQQQgjpllAEEUIIIaRbQhFECCGEkG4JRRAhhBBCuiUUQYQQQgjpllAEEUIIIaRbQhFEIoJKpcIjjzwS6cPoEvTp0weXXnqpz/XWrFkDlUqFNWvWhP6gQsz48eMxfvz4SB+GRzZs2IBx48YhISEBKpUKJSUlkT6kqOKRRx6BSqUK2/7C8X2zd+9eXHjhhUhJSYFKpcLHH38c0v2R4EAR1IVYvHgxVCqV/KPVapGfn4+ZM2eivLw80odHSLfAaDRi6tSpqKmpwT/+8Q+89dZb6N27d8j2J8TtoUOH5GUzZ85EYmJiyPYZCh5//PGoEw6HDh3y+8ZhxowZ+PXXX7FgwQK89dZbGDlyZNCPp7m5GY888kiXuJGJFrSRPgASfB577DEUFhaitbUVP//8MxYvXowffvgB27ZtQ2xsbKQPDwDQ0tICrZZvv3ByzjnnoKWlBTqdLtKH0qXZv38/Dh8+jH/961+49dZbI304Ucmf//xn3H///Q7LHn/8cVx99dWYMmVKZA6qA7S0tGDt2rV48MEHMXfu3JDtp7m5GY8++igARHUktDPBq1AX5OKLL5bvQm699VZkZmbiySefxCeffIJrrrkmwkdnJVrEWHdCrVbzvIeByspKAEBqamrQttnU1ISEhISgbS/SaLXaLnUTVFVVBSC4f/NwYjKZYLFYuuUNEtNh3YCzzz4bgPUOVeDJUzFz5kz06dNH/l2Eg//+97/jtddeQ9++faHX63HGGWdgw4YNLs9NTExEeXk5pkyZgsTERGRlZeHee++F2Wx2WNc5Ry88Avv27cPMmTORmpqKlJQU3HTTTWhubnZ4bktLC+bNm4fMzEwkJSXhN7/5DcrLy/3K+4vUwbJly/Doo48iPz8fSUlJuPrqq1FXV4e2tjbcddddyM7ORmJiIm666Sa0tbW5bGfp0qUYMWIE4uLikJ6ejmnTpqGsrMxhne+//x5Tp05Fr169oNfrUVBQgPnz56OlpaXd580bX3/9NYYNG4bY2FgMHDgQy5cvd/valaH08ePHY/DgwdixYwfOO+88xMfHIz8/H4sWLXJ4rsFgwEMPPYQRI0YgJSUFCQkJOPvss7F69Wqfx3XppZeiqKjI7WNjx451SBu8+eabOP/885GdnQ29Xo+BAwfilVde8bkPkQpWpoQ8vWYAWLduHS666CKkpKQgPj4e5557Ln788UeHdRoaGnDXXXehT58+0Ov1yM7OxgUXXIBNmzZ5PI6ZM2fi3HPPBQBMnToVKpXK4XO2atUqnH322UhISEBqaiouv/xy7Ny502Eb4rOwY8cOXH/99UhLS8NZZ53l8xy0h/fff19+H2dmZuKGG25wSZ0H8v48ceIEfvvb3yI5ORmpqamYMWMGtmzZApVKhcWLF7u8RoFKpUJTUxOWLFkip/Nnzpwp71/5neRpGwDQ1taG+fPnIysrS/5uOHLkiNvXXl5ejptvvhk5OTnQ6/UYNGgQ3njjjQDOnv04RLrzD3/4A1QqlcPx+rMffz5fhw4dQlZWFgDg0Ucflc+T+M5rz3f6s88+K3+n79ixAwCwa9cuXH311UhPT0dsbCxGjhyJTz75JODz0lnoOlKceERcGNLS0tq9jXfeeQcNDQ24/fbboVKpsGjRIlx55ZU4cOAAYmJi5PXMZjMmTZqE0aNH4+9//zu++eYbPP300+jbty/uuOMOn/u55pprUFhYiIULF2LTpk14/fXXkZ2djSeffFJeZ+bMmVi2bBl++9vfYsyYMfj2228xefLkgF7PwoULERcXh/vvvx/79u3DCy+8gJiYGKjVapw8eRKPPPKInEosLCzEQw89JD93wYIF+Mtf/oJrrrkGt956K6qqqvDCCy/gnHPOwebNm+W7wffffx/Nzc244447kJGRgfXr1+OFF17AkSNH8P777zscT0fP2969e3Httddi9uzZmDFjBt58801MnToVX375JS644AKvzz158iQuuugiXHnllbjmmmvwwQcf4L777sOQIUNw8cUXAwDq6+vx+uuv47rrrsNtt92GhoYG/Pvf/8akSZOwfv16DBs2zOP2r732Wtx4443YsGEDzjjjDHn54cOH8fPPP+Opp56Sl73yyisYNGgQfvOb30Cr1eLTTz/FnXfeCYvFgjlz5vg8D/6watUqXHzxxRgxYgQefvhhqNVqWXx9//33GDVqFABg9uzZ+OCDDzB37lwMHDgQJ06cwA8//ICdO3di+PDhbrd9++23Iz8/H48//jjmzZuHM844Azk5OQCAb775BhdffDGKiorwyCOPoKWlBS+88ALOPPNMbNq0yeVCP3XqVJxyyil4/PHHIUlSUF67ksWLF+Omm27CGWecgYULF+L48eN47rnn8OOPPzq8jwH/3p8WiwWXXXYZ1q9fjzvuuAP9+/fHf//7X8yYMcPnsbz11lu49dZbMWrUKMyaNQsA0Ldv34Bf06233oqlS5fi+uuvx7hx47Bq1Sq33w3Hjx/HmDFjoFKpMHfuXGRlZeGLL77ALbfcgvr6etx1111+7/PKK69Eamoq5s+fj+uuuw6XXHKJ7Mfydz/+fL6ysrLwyiuv4I477sAVV1yBK6+8EgAwdOjQgM8TYL3haG1txaxZs6DX65Geno7t27fjzDPPRH5+Pu6//34kJCRg2bJlmDJlCj788ENcccUV7dpXVCORLsObb74pAZC++eYbqaqqSiorK5M++OADKSsrS9Lr9VJZWZm87rnnniude+65LtuYMWOG1Lt3b/n3gwcPSgCkjIwMqaamRl7+3//+VwIgffrppw7PBSA99thjDtssLi6WRowY4bAMgPTwww/Lvz/88MMSAOnmm292WO+KK66QMjIy5N83btwoAZDuuusuh/Vmzpzpsk13rF69WgIgDR48WDIYDPLy6667TlKpVNLFF1/ssP7YsWMdzsehQ4ckjUYjLViwwGG9X3/9VdJqtQ7Lm5ubXfa/cOFCSaVSSYcPH5aXBXLe3NG7d28JgPThhx/Ky+rq6qQePXpIxcXFLq999erV8rJzzz1XAiD93//9n7ysra1Nys3Nla666ip5mclkktra2hz2e/LkSSknJ8flb+ZMXV2dpNfrpXvuucdh+aJFi1zOhbtzNmnSJKmoqMhhmfP7V7z3Dx486LCe82u2WCzSKaecIk2aNEmyWCwO+y0sLJQuuOACeVlKSoo0Z84cr6/NHWKf77//vsPyYcOGSdnZ2dKJEyfkZVu2bJHUarV04403ysvEZ+G6664LeN+CGTNmSAkJCR4fNxgMUnZ2tjR48GCppaVFXv7ZZ59JAKSHHnrIYVv+vD8//PBDCYD07LPPysvMZrN0/vnnSwCkN9980+U1KklISJBmzJjh9rUoP4OetlFSUiIBkO68806H9a6//nqX74ZbbrlF6tGjh1RdXe2w7rRp06SUlBS370NviO/Jp556ymG5v/vx9/NVVVXl8Xsu0O/05ORkqbKy0mHdCRMmSEOGDJFaW1vlZRaLRRo3bpx0yimneD0HnRWmw7ogEydORFZWFgoKCnD11VcjISEBn3zyCXr27NnubV577bUOkSSRYjtw4IDLurNnz3b4/eyzz3a7njvcPffEiROor68HAHz55ZcAgDvvvNNhvd/97nd+bV9w4403OkSwRo8eDUmScPPNNzusN3r0aJSVlcFkMgEAli9fDovFgmuuuQbV1dXyT25uLk455RSH8HVcXJz8/6amJlRXV2PcuHGQJAmbN2/267X7e97y8vIc7tKSk5Nx4403YvPmzaioqPD63MTERNxwww3y7zqdDqNGjXLYt0ajkf0CFosFNTU1MJlMGDlypNf0kDiWiy++GMuWLXOIaLz33nsYM2YMevXqJS9TnrO6ujpUV1fj3HPPxYEDB1BXV+fjLPimpKQEe/fuxfXXX48TJ07If7+mpiZMmDAB3333HSwWCwCrv2PdunU4evRoh/d77NgxlJSUYObMmUhPT5eXDx06FBdccAH+97//uTzH+f0QTH755RdUVlbizjvvdPCJTZ48Gf3798fnn3/u83ic359ffvklYmJicNttt8nL1Gp10CJ4vhDncN68eQ7LnaM6kiThww8/xGWXXQZJkhw+x5MmTUJdXZ3P97Q/BLKfjny+2stVV10lp9cAoKamBqtWrcI111yDhoYG+VhPnDiBSZMmYe/evV2yypjpsC7ISy+9hFNPPRV1dXV444038N1330Gv13dom8oLFWBPrZ08edJheWxsrMMHS6zrvF579pOcnIzDhw9DrVajsLDQYb1+/fr5tX1P+0lJSQEAFBQUuCy3WCyoq6tDRkYG9u7dC0mScMopp7jdrlJYlZaW4qGHHsInn3zi8vqdL+gdPW/9+vVz8UeceuqpAKzp0NzcXI/P7dmzp8tz09LSsHXrVodlS5YswdNPP41du3bBaDTKy53/Fu649tpr8fHHH2Pt2rUYN24c9u/fj40bN+LZZ591WO/HH3/Eww8/jLVr17p4werq6uS/U3vZu3cvAHhN0dTV1SEtLQ2LFi3CjBkzUFBQgBEjRuCSSy7BjTfe6NHf5I3Dhw8DAE477TSXxwYMGICvvvrKxfzsz3ltL96Op3///vjhhx8clvnz/jx8+DB69OiB+Ph4h/UC/Wy2F/Hd4JxGc36NVVVVqK2txWuvvYbXXnvN7baEub0jBLqfjny+2oPzdvft2wdJkvCXv/wFf/nLXzweb35+fkiOJ1JQBHVBRo0aJZtNp0yZgrPOOgvXX389du/eLeeqVSqVW5+BJyOuRqNxu9x5G57W8xd/99NRPO3H1/4tFgtUKhW++OILt+uK82s2m3HBBRegpqYG9913H/r374+EhASUl5dj5syZcrTB137DgT/nfOnSpZg5cyamTJmCP/zhD8jOzoZGo8HChQsdDPeeuOyyyxAfH49ly5Zh3LhxWLZsGdRqNaZOnSqvs3//fkyYMAH9+/fHM888g4KCAuh0Ovzvf//DP/7xD5dzpsRT4z3n97PYxlNPPeXRxyT+htdccw3OPvtsfPTRR/j666/x1FNP4cknn8Ty5ctlr1QoUUbFIk0k35/+/m39RbwHbrjhBo9iuL0+m/bup6OfLyDw73Tn95c43nvvvReTJk1y+5xwCdpwQhHUxREfpPPOOw8vvvii3JsjLS3NbapF3CFGK71794bFYsHBgwcdojH79u0Ly/779u0LSZJQWFgoR1rc8euvv2LPnj1YsmQJbrzxRnn5ihUrQnJc4i5OecHYs2cPALitrAmUDz74AEVFRVi+fLnDPh5++GG/np+QkIBLL70U77//Pp555hm89957OPvss5GXlyev8+mnn6KtrQ2ffPKJQ6TOnwo0ETGsra11WO78fhZRguTkZEycONHndnv06IE777wTd955JyorKzF8+HAsWLAgYBEkqod2797t8tiuXbuQmZkZ1hJ45fGcf/75Do/t3r27Xc0de/fujdWrV6O5udkhGuTvZ9OT2ElLS3P5uwKuf1vx3bB//36H6I/zOReVY2az2a/3QHsJZD/+fr68ddnu6He6iHDGxMSE9LxEG/QEdQPGjx+PUaNG4dlnn0VraysA68Vg165dcn8LANiyZYtLmXC0Ie5QXn75ZYflL7zwQlj2f+WVV0Kj0eDRRx91ueuSJAknTpwAYL9zVq4jSRKee+65kBzX0aNH8dFHH8m/19fX4//+7/8wbNgwr6kwf3H3etatW4e1a9f6vY1rr70WR48exeuvv44tW7bg2muv9bmPuro6vPnmmz63LcTNd999Jy8zm80uaYgRI0agb9+++Pvf/47GxkaX7YjPg9lsdklZZmdnIy8vz23LBF/06NEDw4YNw5IlSxwu6Nu2bcPXX3+NSy65JOBtdoSRI0ciOzsbr776qsPr+eKLL7Bz586Aqy0B62fTaDTiX//6l7zMYrHgpZde8uv5CQkJbsVO3759UVdX55CePXbsmMP7HYAsTJ9//nmH5c4pV41Gg6uuugoffvghtm3b5rI/5XdiRwhkP/5+voS49HSeOvKdnp2djfHjx+Of//wnjh075vV4uxKMBHUT/vCHP2Dq1KlYvHgxZs+ejZtvvhnPPPMMJk2ahFtuuQWVlZV49dVXMWjQINmEHI2MGDECV111FZ599lmcOHFCLpEXUY9QzyPq27cv/va3v+GBBx7AoUOHMGXKFCQlJeHgwYP46KOPMGvWLNx7773o378/+vbti3vvvRfl5eVITk7Ghx9+6LfHJ1BOPfVU3HLLLdiwYQNycnLwxhtv4Pjx434JCH+49NJLsXz5clxxxRWYPHkyDh48iFdffRUDBw50KybccckllyApKQn33nuvfIFQcuGFF0Kn0+Gyyy7D7bffjsbGRvzrX/9Cdna22y9lJYMGDcKYMWPwwAMPoKamBunp6Xj33XdlQ7tArVbj9ddfx8UXX4xBgwbhpptuQn5+PsrLy7F69WokJyfj008/RUNDA3r27Imrr74ap59+OhITE/HNN99gw4YNePrppwM7eTaeeuopXHzxxRg7dixuueUWuUQ+JSUlJHOtjEYj/va3v7ksT09Px5133oknn3wSN910E84991xcd911col8nz59MH/+/ID3N2XKFIwaNQr33HMP9u3bh/79++OTTz5BTU0NAN+fzREjRuCbb77BM888g7y8PBQWFmL06NGYNm0a7rvvPlxxxRWYN28empub8corr+DUU091MA0PGzYM1113HV5++WXU1dVh3LhxWLlypdtI1BNPPIHVq1dj9OjRuO222zBw4EDU1NRg06ZN+Oabb+Rj7ij+7sffz1dcXBwGDhyI9957D6eeeirS09MxePBgDB48OCjf6S+99BLOOussDBkyBLfddhuKiopw/PhxrF27FkeOHMGWLVuCcl6iivAVopFQI8qEN2zY4PKY2WyW+vbtK/Xt21cymUySJEnS0qVLpaKiIkmn00nDhg2TvvrqK4/llM6ln5LkWubuqSzXXTms83PFOlVVVW5fk7L0uampSZozZ46Unp4uJSYmSlOmTJF2794tAZCeeOIJb6fIY/myp3Pn6bg+/PBD6ayzzpISEhKkhIQEqX///tKcOXOk3bt3y+vs2LFDmjhxopSYmChlZmZKt912m7RlyxaXcuFAzps7evfuLU2ePFn66quvpKFDh0p6vV7q37+/y2v0VCI/aNAgl206vw8sFov0+OOPS71795b0er1UXFwsffbZZx7Llz0xffp0CYA0ceJEt49/8skn0tChQ6XY2FipT58+0pNPPim98cYbLu8Bd+XA+/fvlyZOnCjp9XopJydH+tOf/iStWLHC5TVLkiRt3rxZuvLKK6WMjAxJr9dLvXv3lq655hpp5cqVkiRZ2wT84Q9/kE4//XQpKSlJSkhIkE4//XTp5Zdf9vkaPb3HJEmSvvnmG+nMM8+U4uLipOTkZOmyyy6TduzY4bCOp/dcIIiydnc/ffv2ldd77733pOLiYkmv10vp6enS9OnTpSNHjrhsy9/3Z1VVlXT99ddLSUlJUkpKijRz5kzpxx9/lABI7777rtfn7tq1SzrnnHOkuLg4CYBDufzXX38tDR48WNLpdNJpp50mLV261O02WlpapHnz5kkZGRlSQkKCdNlll0llZWVuy8qPHz8uzZkzRyooKJBiYmKk3NxcacKECdJrr73m1zlW4u170p/9BPL5+umnn6QRI0ZIOp3O5XV19DtdkqyfoxtvvFHKzc2VYmJipPz8fOnSSy+VPvjgg4DPS2dAJUkh6MJFSJgpKSlBcXExli5diunTp0f6cAghNj7++GNcccUV+OGHH3DmmWdG+nAIcYCeINLpcB47AVjz/mq1Guecc04EjogQArh+Ns1mM1544QUkJyd77LJNSCShJ4h0OhYtWoSNGzfivPPOg1arxRdffIEvvvgCs2bNcunzQwgJH7/73e/Q0tKCsWPHoq2tDcuXL8dPP/2Exx9/PKpK/gkRMB1GOh0rVqzAo48+ih07dqCxsRG9evXCb3/7Wzz44INdajI1IZ2Nd955B08//TT27duH1tZW9OvXD3fccQfmzp0b6UMjxC0UQYQQQgjpltATRAghhJBuCUUQIYQQQrolNFB4wGKx4OjRo0hKSgp5Az5CCCGEBAdJktDQ0IC8vDyo1d5jPRRBHjh69CgrjQghhJBOSllZGXr27Ol1HYogDyQlJQGwnsTk5OQIHw0hhBBC/KG+vh4FBQXyddwbFEEeECmw5ORkiiBCCCGkk+GPlYXGaEIIIYR0SyiCCCGEENItoQgihBBCSLeEIogQQggh3RKKIEIIIYR0SyiCCCGEENItoQgihBBCSLeEIogQQggh3RKKIEIIIYR0SyiCCCGEENItoQgihBBCSLeEIogQQggh3RKKoCjl7XWHsf5gTaQPgxBCCOmyUARFIfsqG/HgR9tw/4dbI30ohBBCSJeFIigKqWsxAgDqW40RPhJCCCGk60IRFIUYzRYAgMFkifCREEIIIV0XiqAoRIggo1mK8JEQQgghXReKoCjELoIYCSKEEEJCBUVQFGIwWSNAJosEi4XRIEIIISQUUARFIcoIkNHCaBAhhBASCiiCohAHEURfECGEEBISKIKiEKUIYoUYIYQQEhoogqIQgyL6Q3M0IYQQEhoogqIQo4mRIEIIISTUUARFIY6eIIogQgghJBRQBEUhNEYTQgghoYciKAqhJ4gQQggJPRRBUYhDdRhFECGEEBISKIKiEKUx2khjNCGEEBISKIKiEHqCCCGEkNBDERSF0BNECCGEhB6KoCiEniBCCCEk9FAERSHsE0QIIYSEHoqgKISzwwghhJDQQxEUhRhM9AQRQgghoYYiKApx9ASxOowQQggJBRRBUYjJwj5BhBBCSKihCIpCjEyHEUIIISGHIigKMbA6jBBCCAk5IRNBCxYswLhx4xAfH4/U1FS368ybNw8jRoyAXq/HsGHD3K7z1VdfYcyYMUhKSkJWVhauuuoqHDp0yOu+a2pqMH36dCQnJyM1NRW33HILGhsbO/aCwgg9QYQQQkjoCZkIMhgMmDp1Ku644w6v691888249tpr3T528OBBXH755Tj//PNRUlKCr776CtXV1bjyyiu9bnP69OnYvn07VqxYgc8++wzfffcdZs2a1e7XEm7YJ4gQQggJPdpQbfjRRx8FACxevNjjOs8//zwAoKqqClu3bnV5fOPGjTCbzfjb3/4Gtdqq1+69915cfvnlMBqNiImJcXnOzp078eWXX2LDhg0YOXIkAOCFF17AJZdcgr///e/Iy8vr6EsLOcp5YTRGE0IIIaEhqj1BI0aMgFqtxptvvgmz2Yy6ujq89dZbmDhxolsBBABr165FamqqLIAAYOLEiVCr1Vi3bp3HfbW1taG+vt7hJ1IoGyQyEkQIIYSEhqgWQYWFhfj666/xpz/9CXq9HqmpqThy5AiWLVvm8TkVFRXIzs52WKbVapGeno6KigqPz1u4cCFSUlLkn4KCgqC9jkChJ4gQQggJPQGJoPvvvx8qlcrrz65du4J2cBUVFbjtttswY8YMbNiwAd9++y10Oh2uvvpqSFJwxcEDDzyAuro6+aesrCyo2w8Ejs0ghBBCQk9AnqB77rkHM2fO9LpOUVFRR47HgZdeegkpKSlYtGiRvGzp0qUoKCjAunXrMGbMGJfn5ObmorKy0mGZyWRCTU0NcnNzPe5Lr9dDr9cH7dg7goMniOkwQgghJCQEJIKysrKQlZUVqmNxobm5WTZECzQaDQDAYnEvDsaOHYva2lps3LgRI0aMAACsWrUKFosFo0ePDu0BBwn2CSKEEEJCT8g8QaWlpSgpKUFpaSnMZjNKSkpQUlLi0K9n3759KCkpQUVFBVpaWuR1DAYDAGDy5MnYsGEDHnvsMezduxebNm3CTTfdhN69e6O4uBgAsH79evTv3x/l5eUAgAEDBuCiiy7CbbfdhvXr1+PHH3/E3LlzMW3atE5RGSZJEkvkCSGEkDAQshL5hx56CEuWLJF/F6Jl9erVGD9+PADg1ltvxbfffuuyzsGDB9GnTx+cf/75eOedd7Bo0SIsWrQI8fHxGDt2LL788kvExcUBsEaLdu/eDaPRKG/n7bffxty5czFhwgSo1WpcddVVcjl+tGO2SFDanWiMJoQQQkKDSgq2w7iLUF9fj5SUFNTV1SE5OTls+20xmDHgoS/l38cWZeA/s1y9T4QQQghxJZDrd1SXyHdHDE7pL6bDCCGEkNBAERRlOIseiiBCCCEkNFAERRnOooeeIEIIISQ0UARFGUaTo+hhJIgQQggJDRRBUQY9QYQQQkh4oAiKMlzSYRybQQghhIQEiqAog8ZoQgghJDxQBEUZjAQRQggh4YEiKMow2IzRsTHWP42R1WGEEEJISKAIijJEJChBp3X4nRBCCCHBhSIoyhCiJ16vAQCYLBIsFkaDCCGEkGBDERRlOEeCAMBoYTSIEEIICTYUQVGG6BAdr9PIy+gLIoQQQoIPRVCUYbRVg8UrI0GsECOEEEKCDkVQlCHSYbExamjUKodlhBBCCAkeFEFRhhA8MRo1YjRWEeQ8SoMQQgghHYciKMoQniCrCGKvIEIIISRUUARFGcpIkM4mgtg1mhBCCAk+FEFRhjBB67QqRSSIIogQQggJNhRBUYaDJ0hLTxAhhBASKiiCogylJ0ikw1giTwghhAQfiqAow7E6jMZoQgghJFRQBEUZQgTpNCrotPQEEUIIIaGCIijKcBcJoieIEEIICT4UQVGGwWTzBGntzRIZCSKEEEKCD0VQlOHeE0QRRAghhAQbiqAow8ETJFeH0RhNCCGEBBuKoCiDniBCCCEkPFAERRkOs8O0HJtBCCGEhAqKoChDNEakMZoQQggJLRRBUYZbTxBFECGEEBJ0KIKiDPeeIBqjCSGEkGBDERRlOMwOY8doQgghJGRQBEUZbvsE0RhNCCGEBB2KoChD9gRpVdDRGE0IIYSEDIqgKEOuDqMniBBCCAkpFEFRhrs+QYwEEUIIIcGHIijK4OwwQgghJDxQBEUZ9j5BanqCCCGEkBBCERRlyJEgrcruCeIAVUIIISToUARFEZIkwaj0BHGAKiGEEBIyKIKiCKOiCszBGM0+QYQQQkjQoQiKIpTeH3qCCCGEkNBCERRFKMVOjEbFsRmEEEJICKEIiiKE90elAjRqFZslEkIIISGEIiiKUJqiVSoV+wQRQgghIYQiKIoQBmidTfxQBBFCCCGhgyIoirB3i7YaonWcIk8IIYSEDIqgKMKgGJkBWBsmWpfTE0QIIYQEG4qgKELpCVL+y3QYIYQQEnwogqIIeW6YrTReRxFECCGEhAyKoChCeH+EJ8g+O4wiiBBCCAk2IRNBCxYswLhx4xAfH4/U1FS368ybNw8jRoyAXq/HsGHD3K7z1VdfYcyYMUhKSkJWVhauuuoqHDp0yOu++/TpA5VK5fDzxBNPdOwFhQEXT5BNDJksEiwW+oIIIYSQYBIyEWQwGDB16lTccccdXte7+eabce2117p97ODBg7j88stx/vnno6SkBF999RWqq6tx5ZVX+tz/Y489hmPHjsk/v/vd79r1OsKJiydIa//zGC2MBhFCCCHBRBuqDT/66KMAgMWLF3tc5/nnnwcAVFVVYevWrS6Pb9y4EWazGX/729+gVlsFwb333ovLL78cRqMRMTExHredlJSE3NzcDryC8CN7gjSOniDrYxL0IftrEUIIId2PqPYEjRgxAmq1Gm+++SbMZjPq6urw1ltvYeLEiV4FEAA88cQTyMjIQHFxMZ566imYTCav67e1taG+vt7hJ9zIfYK0jp4ggL2CCCGEkGAT1SKosLAQX3/9Nf70pz9Br9cjNTUVR44cwbJly7w+b968eXj33XexevVq3H777Xj88cfxxz/+0etzFi5ciJSUFPmnoKAgmC/FLwwmR0+QRq2CRs1J8oQQQkgoCEgE3X///S6GY+efXbt2Be3gKioqcNttt2HGjBnYsGEDvv32W+h0Olx99dWQJM9G4bvvvhvjx4/H0KFDMXv2bDz99NN44YUX0NbW5vE5DzzwAOrq6uSfsrKyoL0Of3H2BFn/LxomUgQRQgghwSQgl8k999yDmTNnel2nqKioI8fjwEsvvYSUlBQsWrRIXrZ06VIUFBRg3bp1GDNmjF/bGT16NEwmEw4dOoTTTjvN7Tp6vR56vT4ox91enD1BgFUQtRotskAihBBCSHAISARlZWUhKysrVMfiQnNzs2yIFmg0GgCAJYBqqZKSEqjVamRnZwf1+IKN8+wwgA0TCSGEkFARMk9QaWkpSkpKUFpaCrPZjJKSEpSUlKCxsVFeZ9++fSgpKUFFRQVaWlrkdQwGAwBg8uTJ2LBhAx577DHs3bsXmzZtwk033YTevXujuLgYALB+/Xr0798f5eXlAIC1a9fi2WefxZYtW3DgwAG8/fbbmD9/Pm644QakpaWF6uUGBec+Qcr/s2EiIYQQElxCVnT90EMPYcmSJfLvQrSsXr0a48ePBwDceuut+Pbbb13WOXjwIPr06YPzzz8f77zzDhYtWoRFixYhPj4eY8eOxZdffom4uDgA1mjR7t27YTQaAVjTWu+++y4eeeQRtLW1obCwEPPnz8fdd98dqpcaNIwmmydI0R9IVIoxEkQIIYQEl5CJoMWLF3vtEQQAa9as8bmdadOmYdq0aR4fHz9+vINJevjw4fj555/9PcyowpMnyPoYPUGEEEJIMInqEvnuhjdPENNhhBBCSHChCIoivHmCmA4jhBBCggtFUBRhdCuC2CeIEEIICQUUQVGEyeb70SmM0eL/jAQRQgghwYUiKIowuPEEMR1GCCGEhAaKoCjC3dgMuVmiidVhhBBCSDChCIoijCYvzRIZCSKEEEKCCkVQFOG2TxA9QYQQQkhIoAiKImRPkFbpCWLHaEIIISQUUARFEe5K5HXsGE0IIYSEBIqgKMKdMZoDVAkhhJDQQBEURXibHUZjNCGEEBJcKIKiCIO76jAxRZ6RIEIIISSoUARFEd4GqNIYTQghhAQXiqAoQvYEad2lw2iMJoQQQoIJRVAU4c4TxNlhhBBCSGigCIoi3E+RpwgihBBCQgFFUBRhN0YrPUFslkgIIYSEAoqgKMJ7nyB6ggghhJBgQhEURcieIDfGaEaCCCGEkOBCERQlWCwSTBY3kSAaowkhhJCQQBEUJRgtdpFDTxAhhBASeiiCogTlgFTODiOEEEJCD0VQlKAci+FWBLFZIiGEEBJUKIKiBJHu0qhV0Kjt6TAaowkhhJDQQBEUJRjczA0DAJ2WniBCCCEkFFAERQnuegQpf+cUeUIIISS4UARFCe5GZgD2nkH0BBFCCCHBhSIoSnA3MsP6Oz1BhBBCSCigCIoSPEaCKIIIIYSQkEARFCUIT5DOkyeIIogQQggJKhRBUYKnSFCM3DFagiTRF0QIIYQEC4qgKEEukdc6eYIUw1SNNEcTQgghQYMiKEowmrx7ggCmxAghhJBgQhEUJfjqEwRwfhghhBASTCiCogQR5XE2RmvUKogpGowEEUIIIcGDIihK8DQ2w7pM7bAOIYQQQjoORVCU4Kk6DFD2CqIxmhBCCAkWFEFRgmyM1roRQVr2CiKEEEKCDUVQlOCpWSKgSIfRGE0IIYQEDYqgKMGrJ0grGiZSBBFCCCHBgiIoSvDmCYqhJ4gQQggJOhRBUYJ/xmhGggghhJBgQREUJcieIDfGaJbIE0IIIcGHIihKMJi89QmyeYJojCaEEEKCBkVQlOCPJ4iRIEIIISR4UARFCV49QewTRAghhAQdiqAowZ8+QUYTq8MIIYSQYEERFCV4nx2mcliHEEIIIR2HIihK8DY2I4Yl8oQQQkjQoQiKEugJIr4wWyQ0tZkifRiEENJlCJkIWrBgAcaNG4f4+Hikpqa6XWfevHkYMWIE9Ho9hg0b5nadZcuWYdiwYYiPj0fv3r3x1FNP+dx3TU0Npk+fjuTkZKSmpuKWW25BY2NjB15N6PHmCeIUeQIAj366HcV/XYE9xxsifSiEENIlCJkIMhgMmDp1Ku644w6v691888249tpr3T72xRdfYPr06Zg9eza2bduGl19+Gf/4xz/w4osvet3m9OnTsX37dqxYsQKfffYZvvvuO8yaNavdryUcGPwpkWefoG7Nyp2VMJgsWLmzMtKHQgghXQJtqDb86KOPAgAWL17scZ3nn38eAFBVVYWtW7e6PP7WW29hypQpmD17NgCgqKgIDzzwAJ588knMmTMHKpWriXjnzp348ssvsWHDBowcORIA8MILL+CSSy7B3//+d+Tl5XX0pYUEo1djNNNh3Z1mgwnltS0AgG1H6yJ8NIQQ0jWIak9QW1sbYmNjHZbFxcXhyJEjOHz4sNvnrF27FqmpqbIAAoCJEydCrVZj3bp1XvdVX1/v8BNOZBHkzhjNKfLdngNVTfL/t5VTBBFCSDCIahE0adIkLF++HCtXroTFYsGePXvw9NNPAwCOHTvm9jkVFRXIzs52WKbVapGeno6KigqP+1q4cCFSUlLkn4KCguC9ED8QPYDoCSLu2Fdp97QdPtGMuhZjBI+GEEK6BgGJoPvvvx8qlcrrz65du4J2cLfddhvmzp2LSy+9FDqdDmPGjMG0adOsB64Orn574IEHUFdXJ/+UlZUFdfu+CMbYjLpmIx75ZDu2lNUG/fhIZNlf5Wjs386UGCGEdJiAPEH33HMPZs6c6XWdoqKijhyPAyqVCk8++SQef/xxVFRUICsrCytXrvS6n9zcXFRWOhpHTSYTampqkJub63Ffer0eer0+aMceKN6bJfpnjP5qRwUW/3QIpTXNeGPmGcE/SBIxlJEgANheXo9xfTMjdDSEENI1CEgEZWVlISsrK1TH4hGNRoP8/HwAwH/+8x+MHTvW43GMHTsWtbW12LhxI0aMGAEAWLVqFSwWC0aPHh22Yw4U75Eg/zxBoodMaU1zkI+ORBoRCRrVJx3rD9XgV/qCCCFRgiRJbguVOgMh8wSVlpaipKQEpaWlMJvNKCkpQUlJiUO/nn379qGkpAQVFRVoaWmR1zEYDACA6upqvPrqq9i1axdKSkrw+9//Hu+//z6effZZeRvr169H//79UV5eDgAYMGAALrroItx2221Yv349fvzxR8ydOxfTpk2L2sowQNEnyI0x2t9miSJSVH6yBZJE/1BXwWS24GC11Rh9ebH1PcwKMUJINPD19gqcseAbLPoyeFaYcBIyEfTQQw+huLgYDz/8MBobG1FcXIzi4mL88ssv8jq33noriouL8c9//hN79uyR1zl69Ki8zpIlSzBy5EiceeaZ2L59O9asWYNRo0bJjzc3N2P37t0wGu1G0bfffhv9+/fHhAkTcMkll+Css87Ca6+9FqqXGhTksRle+wR5FzZttm20GM042UzjbFehtKYZRrOEuBgNLhxoTekerG5CI7tHE0IihCRJeP37A7h96UZUNxqwendVpA+pXYSsT9DixYu99ggCgDVr1nh9PDMzE2vXrvW6zvjx412iHunp6XjnnXf8OcyowR9PkL+RIMAaDUpP0AXxCEmk2G8rjy/KSkBWkh49UmJxrK4VO47WY1RheoSPjhDS3TCZLXj4k+14e12pvKzZ0DlvyqK6RL47IQSO2xJ5f9NhisfLa+kL6ioIU3TfrEQAwOD8FACgL4gQEnYaWo24eckveHtdKVQqYNoZ1nYyTW3mCB9Z+6AIigLMFgkWWzDL7QBVP43RykjQkZMtwTtAElGECOqXbRNBeVYRtJ0iiBASZua/V4Lv9lQhLkaDV28YgTvH9wPASBDpAEpx47ZjtNwnyJcnyK7ExYgF0jnYe7wBm0tPun1MVIYJETSkZzIARoIIIeFn/cEaAMBrN47ApEG5iNdrAADNBjMsls5XkEMRFAUo01hePUE++gS1OXmCSOegqc2Eq175Cdf8cy3KnNobSJKE/c7pMFskaH9VY6e9+yKEdD4kSZILMk7LSQIAJOjs1uIWY+dLiVEERQFKcRPjphN2u4zRjAR1Gj7behT1rSYYzRK+2Xnc4bGqhjY0tJmgVgF9MuMBANnJschO0sMiATuPhXfGHSGk+9JsMMvWjcRYq/iJjVFDtAhq6oQ3ZRRBUYDoEaRVq6BWu0aCdH4OUKUI6py8u8E+omXlTsdu58IP1Cs9HnqtRl4um6OPMCVGCAkPDa1WkaNRqxAXY/0+UqlUcjSoxcBIEGkH3rpFK5f7GpuhTIfVNhvlDtIketld0YDNpbUQ2nfdwRNoaLX3eNrn5AcSCBG07SgjQYSQ8NDYZv1uSorVOnSIjtdZBVFnrBCjCIoCvPUIsi73zxjtLJIYDYp+3rNFgS4YmIOizAQYzRK+21MtPy77gZxFUJ7VHL2N5mhCSJiot0WCEvWOLQYTbL93Ro8iRVAUIPcIclMZBgTgCXJ6nObo6KbNZMbyzUcAANPO6IUJA7IBACsVviARCRKmaMGQntZI0N7KRrR2QjMiIaTzIdJhSbExDsvlSBDTYaQ9GG3jMDylw3QBGqPF+owERTdfbz+O2mYjcpNjcc6pWZgwIAcAsHp3Jcw29+H+Smu3aOd0WG5yLDISdDBbJJqjCSFhoVEWQY6RICGCmjuhBYMiKAow+PIEBWiM7p1hrSKiCIpuRCrsmpE9oVGrMLJ3GlLiYnCy2YhNpSfR0GpERX0rANdIkEqloi+IEBJWhF8xSe8sgqy/MxJE2oXRhyfIHgmSvE6HF80SCzMTADAdFs2U1TTjh33VUKmAqSOtbee1GjXOOy0LAPDNzuPyzLCsJD1S4mJctjE43+YLYoUYISQMNHiIBCXIDRMZCSLtwGd1mMIrZPRijhaRoMIsmwhiJChqWfaLNQp0Vr9MFKTHy8tFSmzlzkrZFN3PKQokGGKLBG0qPdkpO7USQjoXDbZ0V6JLOswWCWJ1GGkPvozRyqGq3lJiIq3WN9N60WQkKDoxmS2yCJp2Ri+Hx849LQtatQr7Khuxape1Z1Df7AS32xneOw06jRp7KxvxxJe7QnvQhJBuj5wOczJGJ+gYCSIdwODDGB3jpwhqc4oEHW9o9dlbiISfNburcLy+DekJOkwcmO3wWHJsDEYVpgMAvth2DIDnSFB2UiyeuGoIAOC17w5gyU+HQnfQhJBuj6d0WLyekSDSAXx5gjRqldxMz7kMXokQQbnJsdBr1ZAkoKKuNbgHSzpEXbMRj362HQBwZXG+QxdogUiJiQxXv+wkj9u7cnhP3HvhqQCARz7djq+3VwT5iAkhxIpcHebcJ4iRINIRfHmClI958gRJkiRHfWJjNMhPjQMAHKltdrs+CT8Wi4S7l5WgrKYFPdPiMPf8fm7XmzjAMTrkKR0mmHNeP1w3qgCSBMx7d7PHafSEENIRGtrcp8NYHUY6hOwJ8iKCdD4mySvFkU6rRn6aVQTRFxQ9vLxmH1buqoROq8arN4xAarzO7Xq9MxLkvkAJOg1yk2O9blelUuGvlw/GeadlodVowS1LfpFnjhFCSLDwWR3GPkGkPYhxGF4jQVoxOsO9CFIu12vVciSIFWLRwfd7q/D0ij0AgL9dPlju8eMJ0T26b3aiw4weT2g1arx4/XAMzk9GTZMBV73yE37aX+3zeYQQ4i+NHsZmiEhQMyNBpD2I6E6Mh+owwO4X8mR0Vi7XaRQiiJGgiFNe24J5/9kMSQKmnVGAa84o8PmcG0b3xuD8ZPx2TG+/95Og12LxTaNQ3CsVdS1G3Pjv9Xh3fWlHDp0QQmTqPYzNYJ8g0iFMFu/GaOtj3kdniEaJWrUKarXKng5jJCiiGEwW3Pn2JpxsNmJIfgoe+c0gv55XkB6Pz353ttxI0V8yE/X4z21j8JvT82CySLh/+a94/H875TEchBDSXuwl8l2nY7TW9yok1Ag/j1+eIA/GaBEJ0tuiSUyHRQcvr9mHLWW1SImLwcvThyM2xrUaLNjExmjw3LRh6JuViH98swevfXcAvxyqwdCeqchPjUN+WhwK0uIxKC8ZarXvVBshhBhMFrkCuSvNDqMIigKEgPGvOsx7Okw0XBSRoGO1rbBYJF7sIsDOY/V4cdU+AMBfpwx26AwdalQqFX4/8RQUZiXg3ve3YFNpLTaV1jqs89sxvfHXKYPDdkyEkM5Lo0LgePIEMRJE2oU/JfI6H8boNicRlJscC41aBYPZgqrGNuT4qDDqykRCBJrMFvzhgy0wWSRcODAHlw3tEdb9C35zeh4G5yXjx/0nUH6yBeW1Ldhf2Ygdx+rx7Z6qiBwTIaTzIVJh8ToNtE7Xqs7sCaIIigJkEaT15gmyTZL3YIx2FkFajRq5ybEor23BkZMt3VYE7TxWj2v+uRZ3jO+LO8e778sTCv753QFsK69HSlwM/jZlsF8VXqGiKCsRRYqu03XNRpz+2NcorWlGXYvR7XBWQghR0uChMgywR4KMZmu/Ok8joKKRznOkXRh/PEG+miXK6TDFNugLAtYdOIGGVhM+KTkatn3uq2zAc9/sBQA8dOlAZEeZAE2Jj0FPW7p0x9F6j+vtPFaPY3Xd971DCLHjqUcQYPcEAZ0vGkQRFAUYAkiHefQEmYUx2v5mZMNEoKbJAADYX9UYljlqZouEP3ywFQazBeedloUrh+eHfJ/tYXCetU/R9qN1bh8/crIZv3nxB1z6/A+obmwL56ERQqIQ4QlyLo8HrNcucY3qbL4giqAowBiAMdpjs0SndBigjAR139EZ1TYRZDRLYemi/MYPB7G5tBZJei0ev3JIRNNg3hicnwwA2FbuXgSt3X8CRrOEE00G/OXjbZAkltgT0p3xVB4vSOikFWIUQVGArwGqysd89QlyEEGMBKGm0SD/f+cxz6mfYLD3eAOe+no3AODByQPQIyUupPvrCINsHau3eUiH/XLIPn/si20V+GzrsbAcFyEkOvGWDgM6b4UYRVAUIHuCvHaMtkWCfHSM1ruNBHVfEXSiyZ7K8SaCVu+u9OqP8YXRbMHdy7bAYLLg3FOzcK0fXaEjyaA8ayToQFWj2xz+L4drAAAjeqcBAB767zZUNTAtRkh3RU6H6d0XUnTW+WEUQVGAX54gP/sE6T1EgrprOuOEMhJU4V7k7Kqox01vbsCtSza0+zy9tHoffi2vQ0pcDBZdPTRq02CC7KRYZCfpYZGAnccaHB6raTJgf1UTAOCV6cMxoEcyTjYb25UWM1skvL3uMO7/cCtO0FtESKel3pYOS2QkiAQbf/oE+YwEmT17gpoMZtS1GINyrJ2NE03KdFiD24v4T/tOAACO1rWi0ku047FPd+CmN9e7RJS2HqnFC7amiI9dPqjTtCMQQ1ydzdEbD1tTYX2zEpCdHIu/Tx0KrVqFL7dX4NMA0mK7Kxpw9as/4cGPtuHdDWW4670SWDi+g5BOia90WGftFUQRFAX44wnSx/iXDlOWyMfGaJCZqAMAHOmGviCDyeIg/mqaDG5Fjkj9AJ5LxuuajXjjx4NYvbsKl77wAx7/3040G0xoNZpx97ItMFskTB7SA785PS/4LyREDM5zb44W5+OMPukAgEF5KZh7vrXH0sN+pMVajWb8/avdmPz899hcWotEvRaxMWp8v7caL63eF+yXQQgJA54myAs66yR5iqAowGjy3SdIzJxq9bNZoiDPFg062g19QSebrVEgjVqFvlkJAFxFjiRJ2KAwAe/w4BvaZouWaNQqmC0SXvvuAC545jvM+89m7KtsRFaSHn+NcFPEQBkol8k7vmZhihZ+IAC4c3w/OS32ypr9HrdZ2dCKS57/Hi+u3geTRcIFA3Ow4u5z8NfLreM5/vHNHqzdfyLYL4V0Y47VtXhs9UCCh6gOS3ZTIg/YewU10RNEAsUfT1CsTdy0Gt2rbE8iSKRmvKV5uirCD5QWr5Mv+M4i5/CJZofIhicR9KstWjJpUA7emDkSPdPiUF7bgq93HAcAPHnVEKQn6IL+GkKJKJPfc7xBri5sNZrx6xHrax1piwQB1vfVHyadCgD4ZEu5R2/av384iANVTchK0uPVG4bjXzeORI+UOEwdWYCrR/SERQLmvbuZJmsSNGa+sQGXv/gjKhtaI30oXRp/q8MYCSIB02J704iUlztibSq7xcMbzG6MdpxSnpOsBwBU1ne/LwhRGZaZqMOAHkkAXCvENhyypn6EoXynh3SYEEGD81Nwfv8crJh/Lu4Y3xd6rRq3nlWI8/vnhOQ1hJL81DikxsfAaJaw97i1h9K28joYzBZkJurQJ8Nx4OvZp2QhM1GH6kYDvt/rOnfMaLbgw43lAIC/Xj4YFw12nJf22OWDcEp2Iqoa2jD/vRKY6Q8iQeDgiSaYLBKO1na/77hwIqrDPBmjRZ+gJnqCSKCItE1avOdIQqzWezrMXbNEAMhJskaCjtd3vztv0S06PUGHgT2sUQ9PIkh4eQ6eaHJr7BO+mSE2M3GcToP7LuqPHY9dhD9fOjA0LyDEqFQquXO0eH0bFKkw59RejEaNy2zn6cNN5S7bW7O7CtWNbchM1GHCgGyXx+N1Wrw8fTjiYjT4YR/9QaTjtJnM8nefSNeQ0GCPBHlIh9m8Qs1tjASRAJAkCbXN1g9varznQZayJ8hjOszWLFHjPh12vBuGiqtt6bCMRL0sgg5WNzmcQ+F/uXhILrKS9JAkYFeFY8l4XYsRh09Yu24L0SDQhHk6fbAR/YKEL2ijzRQ9sne62/WvGt4TALBix3GXisP3NpQBAK4c3tNjaveUnCT8dYrVH/Tqt/th8pBWI8QfxIXZ+f8k+NT72TGakSASEM0Gs+wJ8uYpiY3x7gnyFAnKtqXDKuq6nwgSfWkyEnTIStIjI0EHi2Qt3QaAqoY2HKhugkoFjOiVLgslZ/P0dluUpGdaHNI6me/HF/bO0XWwWCS5PH5knzT36+cl49ScRBhMFnzxq71cvrK+Fat3VwIArhnZ0+s+ryzOR1KsFs0Gs4vgJCQQlMKnkSIoZEiSpGiW6METxEgQaQ8iZaPTqhEXo/G4XpyPSJB9gCqN0QJxbjMSdFCpVBjglBITUY/TcpKQEh+DgXnuU2a/OqXCuhKDFa95b2UjTjYbodeqMSjP/WtVqVS4otgqcpYrUmLLN5fDbJEwvFcq+mUned2nWq3CsIJUAMDm0pNe1yXEG8oUWD3TYSGjyWCGaLHmKR3GSBBpFyIVlhYf47W82p4O879jNADk2kRQTZNBTpl1F5TpMAAu5uj1Bx2jHkIkOVeIKU3RXY0+GQlI0GnQarRg2S/WdNbpBaleR7hMKc6DSgWsP1SDsppmSJKEZbZUmL/jQob3sp7zzaW1HXsBpFvjEAnqZKXZnQkhNrVqlZyVcIbVYaRd+GOKBuyVYy0BpsNS42Nkn1B3K0uusVWHZdgaRtojQdYUjHNTQJEO23WswaFyydkU3ZVQq1Vy1EeIoDM8pMIEPVLicGbfTADAR5vL8cvhkzhQ3YR4nQaTh/rXLLK4VyoAYBMjQaQDKCNB9ASFDrlRYqzW48266BjNPkEkIPwVQb7SYZ76BKlUKtkX1N0qxE4o0mEAHNJhjW0m2QwsRFBhZgJiY9RoMZpx+IR1dlZ9qxGHbKboriiCAMhpQHER8WSKVnJFcT4AYPmmI7IhevKQHh67yTpTXGAVWodONHOmGGk39Q7GaKbDQkW9jx5BACNBpJ3I6bAEz5VhgP/pMJ3G1Vck+4K6Wa+gE07psL5ZiYjRqNDQZsKnW47CbJGQnxond9XWqFU4LdcxJSaiQPmpXc8ULVCm+VQqe6rKGxcNzkVcjAaHTjTjo81Wb5C/qTAASImPQb/sRABASVltYAdMiA2mw8KDEJieJsgDnB1G2okw76b6iAT5LJH3YIwG7A0Tj3cjEdRqNMtfiqLqTqdVy6bd/1t7GIBr6se5Qqwrp8IEonM0AJyabTWJ+yJBr8XFg3MBWCfFF2UlOIzZ8IdimzmaKTHSXpgOCw++GiUCQIKYIs/qMBIItXI6zPuFx2d1mId0GABki4aJ3cgTJMRljEaFZMUH19kcfUahY+rHuULs13Lrv0N6dl0R1C8rURbPI3z4gZRcMTxf/v+1IwsCnps23CaaNh2uDeh5hAjYJyg8iHOb7EUExYmpBkYzLJ2oGzxFUIQ5KVeH+YoEWf9UJovkdm6T3CzRbSRIdI3uPpEgORWWoHe4OItIj+CMPk4iyCaSnNNhXbEyTKDVqOXXN7rQtx9IMK5vJk7NSURafAyuHO69N5A7RNpty5FaNk0k7aKRnqCw0OijWzRgjwQBngt4ohH/XIwkZPhrjI5V9BBqNZpdOvJ6iwTZ54d1n0iQmBvm3IBSKYJS4mLQLyvR4fHTcpOhUllN5Ieqm3Cw2mqQ7srpMABYcMVg/LjvBC71s7oLsHqolt95Jkxmi890rjtOyU5Ekl6LhjYT9hxvlKNwhPhLQxvTYeFACExvhQ+xMWqoVIAkWXsFJfhZJBFpGAmKMLII8mGMVnp93JmjPfUJArp5JCjR8eI8QCGCzuiTBrXT2ItEvRZ9MhIAAB9sPALAaorubBPiA6V/bjJuOasw4DEgiXptuwQQYC3PP52+INIBaIwOD/5Uh6lUKjka1Jm6RlMERZiTTWJumPcLiUql8jo6w1PHaKB7GqNrnMrjBWkJOrmB5Mg+7lM/wjckRJDSOEyCy3D2CyIdQFki32wwM60aIuSRGV7SYQAQ3wm7RlMERRhhjE73427aW4WYtxL5bNtFv77VhJZO1sOhvVTLjRL1Lo9dNSIfmYl6TB7Sw+1zRcqswiYau3oqLJIU28zRJewcTdqBsw+os1UmdRbkdJiXSBAAOQXWmXoFUQRFEIPJgibbm8WXJwhQVoi5M0Z79gQl6bXycyu7yTR5T+kwAPjDpP745c8TUZAe7/a5zt6UrmyKjjSiTP5AdRNO2qJ3hPiLsw+I88NCgz/VYYAiEtSJUpMhE0ELFizAuHHjEB8fj9TUVLfrzJs3DyNGjIBer8ewYcPcrrNs2TIMGzYM8fHx6N27N5566imf++7Tpw9UKpXDzxNPPNGBVxMaRBRIrfKeaxXIkSCnGWBmiySPeXAnglQqlZwS6y7T5D2lw/xhgFMFGSNBoSM1XoeiLKsHa3MZU2IkMJwjQTRHhwa5T5APs3NCJ+waHTIRZDAYMHXqVNxxxx1e17v55ptx7bXXun3siy++wPTp0zF79mxs27YNL7/8Mv7xj3/gxRdf9Ln/xx57DMeOHZN/fve737XrdYSSmmZ7o0Rng647hN/HOR0mUmHKdZwRKbHu0itIjGLISHBNh/kiNzlW7tuUnxrnNqVGggeHqZL2YDRb5Ki4KFygOTo0NPhRIg8A8Z1wfljIatgeffRRAMDixYs9rvP8888DAKqqqrB161aXx9966y1MmTIFs2fPBgAUFRXhgQcewJNPPok5c+Z4bc6WlJSE3NzcDryC0GM3Rfvu0AsomlEZPIsgT9O/c7vZ6AwxNyzdTTrMFyqVCgPzkvHjvhM0RYeB4l6p+GDjEZqjSUAooz49UmJR02Rgr6AQIY/N8OUJYiQouLS1tSE2NtZhWVxcHI4cOYLDhw97fe4TTzyBjIwMFBcX46mnnoLJ5F2ZtrW1ob6+3uEn1ARiigaAWK1Ihzl6gkSjRJUK0HqIKHW3CjHhCcpsRyQIAM7qlwUAOOfUrKAdE3GPiASVlNbKaV1CfCEuzPE6jXwjGepIUHVjG37z4g94a+2hkO4nHByoasS/vjvgcQqBEiE4faXDWB0WZCZNmoTly5dj5cqVsFgs2LNnD55++mkAwLFjxzw+b968eXj33XexevVq3H777Xj88cfxxz/+0eu+Fi5ciJSUFPmnoMD/YZDtRXSL9rfPiqcSedkUrVF7jI7ZewV1/XRYs8Ekdyx1Z4z2h9vOLsSnc8/CdWf0CuahETecmpOEBJ0GTQYz9hxviPThRA0lZbWY+upP2MwImVsaFL1rxGDP+hB7gtYfrMHWI3Vy+4zOzNNf78GC/+3EV9srvK5nMFnka0yyj3SYXB3Wiar0AhJB999/v4vh2Pln165dQTu42267DXPnzsWll14KnU6HMWPGYNq0adYDV3s+9Lvvvhvjx4/H0KFDMXv2bDz99NN44YUX0NbmWQA88MADqKurk3/KysqC9jo8cdLPuWECkQ5z8QR56REkyO5GDRNFFEivVct3JoGi1agxpGeKX14t0jE0iqaJX/x6DJLEaBAA/N/aQ9hw6CSWbyqP9KFEJfVyiiZGLt0OdTpMeF06U7rHE1U2f6ivSQLKc+qrRD6uE0aCAvIE3XPPPZg5c6bXdYqKijpyPA6oVCo8+eSTePzxx1FRUYGsrCysXLky4P2MHj0aJpMJhw4dwmmnneZ2Hb1eD70+vAZYURKc5mcFk5wO82CM1mk9X/BzkmyjM7qBMVr4gTIT9QEP9SSR4fz+2fhp/wk8v2of9hxvxONXDunyXbp9IXondZe2FoHiEAmyXZwbQxwJEhHmriCCGmyCzpdwFCnGeJ3GZ0f5BJsI6kyRoIBEUFZWFrKywu+R0Gg0yM+3Tqz+z3/+g7FjxwZ0HCUlJVCr1cjOzg7VIbYLf4enCvQe+gS1eRmZIVCOzpAkqUuLgxoPc8NI9HLTmYVoM1nwjxV78OX2Cvxy+CSevGoIJgzIifShRYSTTQYcsM2t6w43Lu1BWbGUpNc6LAsVQvz446OJdhptc9d8pRAb/BiZIYgXxuhOdH5CVh1WWlqKmpoalJaWwmw2o6SkBADQr18/JCZah1bu27cPjY2NqKioQEtLi7zOwIEDodPpUF1djQ8++ADjx49Ha2sr3nzzTbz//vv49ttv5f2sX78eN954I1auXIn8/HysXbsW69atw3nnnYekpCSsXbsW8+fPxw033IC0tLRQvdx2URtoOswmgpwn9HobnirIthmjmw1mNLaZfJY6hhNJkrD9aD36ZiXK4dSOUO2lUSKJTjRqFeac1w/nnpqF+e+VYG9lI25Z8gtmjuuDR34zKNKHF3ZKjtTK/+9Og48DQVmxJL7PQm2MFiKoK0SCRNSsvsV7JEiZdvRFgl5EgrpoOiwQHnroISxZskT+vbi4GACwevVqjB8/HgBw6623Oggasc7BgwfRp08fAMCSJUtw7733QpIkjB07FmvWrMGoUaPk5zQ3N2P37t0wGq1/KL1ej3fffRePPPII2traUFhYiPnz5+Puu+8O1UttN8o+Qf7gyRhtUBijPRGvs4aMG1pNOF7fFlUiaM3uKty0eAOuGdkTi64+vcPbk7tFt7MyjESOwfkp+PR3Z+HvX+3Gv388iMU/HcK0UQXon9u9WhUoeyZVNbR1+ehte1B2MU4KkyeoxeZ1aTGaYbFIndYzKEmSLBh9ddlu9LMyDLBHgtx5giRJgiQh6s5ZyETQ4sWLvfYIAoA1a9Z4fTwzMxNr1671us748eMdjJTDhw/Hzz//7O9hRpRaOR3mnyCJ9ZAOM5itokgf493nnpMci4bWRlTWt6JfdmKghxsyfi2vAwB8v7c6KNurkeeGMRLUGYmN0eDPlw7EjmP1+Gn/CWwure2GIsheEWYwW1DbbPTbO9hdaHBjjA51dZgyAtRqMssX/c5Gm8kCo9l63QxmOkyOBBmcK5jNuPi575ESF4Pld4yLKkEf1SXyXR1RHeavdyXOwwBVfyJBgKJXUJQZLctPtgAAjtW1BmWshz0SxItGZ2aYrWKsuw1XtVgklJTVOiyjL8gV+eKsV6TDQm2MVlzcO/MwamXa0Fc6TIhNX+XxgCIS5JQO21/ZhANVTdhcWov9VY2BHm5IoQiKEGaLhLqWIPcJ8uIJAoCcpOjsFXSktln+f0kQ5kdVNwUmLkl0IosgJ0HQ1TlQ3YiGVhNiY9TyXDVWiLkiN/CL1cqpmoa20KbDlBGOzuwLUopFX2Zyf+eGAZ47RpfW2L/j1x+Mrr5XFEERor7FCJHF83dsht5DJMhfERStvYJEJAgIzvwokQ7L5MyvTs2wXqkAgD2VDd1qJtQm22dgaM9U5KXEAaA52h1Kw25ymErklVVPzgUqnYnAIkEBVId5mB1WWtMk/3/DoRq/jzMcUARFCGGKTtJrEeMjjSXwVR3mrUQeiM7RGRaLhKO19uMJhgg6weqwLkF2UizyU+MgScDWbhQNEp+B4l6pyO5G/b0CxbFPUIy8LJTNNlsUht/OnA5TRn8aDSZYvIyrEf2E/KoOU0SClH+HwyeUkSCKIAJFeXwAKRuPxmg/miUC0Tk6o6qxTe54DQBby2thNFu8PMM7kiTZh6cyHdbpESmxzd1KBFnTBcUFachKFiIoem5cogVlibwwRpssksv3YzDpMukwRaRGkuxCxx3KtKMvRCTIZJEcvteV6bDy2haU17a4PDdSUARFCDFB3t/KMMC/2WHeyInCdNiRk9YPR15KLJJjtWg1WrC7wr/5UT/tq8bUV3/CNlt1GWD9cAtRyBL5zk938wU1tpnk+WnWSJD1M9sVI0HrD9agrrn9Hh57iXwMEnQaiIKjUPqCHIzRxs6bonVuJeAtJebvBHkAiI+x34gru0YLESQsGxuiKBpEERQhTgbYIwiwR4LaTJ4iQf6lwyrr2wIOGTcbTHhx1V7sqwzugMsjNj9Qz/R4eX6UP3f9Da1G3PVeCTYcOonH/7dTXi5SYQk6TVAaL5LIInxBJWW13WKm2NYjtbBIQH5qHHKSY+V0WFUURW+DwboDJ3DNP9fipsXr2/13VabDVCqV3RwdQl9QNESCWo1mmDoQLQdcm0p66xXUqOjH5AutRi1fh0SvIJPZIvs+Lxxo7QC/jiKIBDo8FVB4gpw+fHKfIB8iKMv2hSr6jgTCZ1uO4e9f78ETXwRvQC6gEEFpcSjuZe3o7c/U7Ke/3iPfHf+0/wS2H7VGg+RUGP1AXYLBeSnQqFWoamjD0SC0T4h2hB9IiD+7J6hrvfYtto7Ym0pr29UfzGi2yN5I4VVJDkOZfHOEPUEtBjPO+/sanLNoNX7pgMHYWSh6E45yOkzv37VKnh9mOz9Ha1thskjQadW4dGgegOgyR1MERQh5blhAniBbOszUPmO0XquRfTKB9go6eMLq7t99PESRoNQ4FCvu+r2xrbwO/7f2EADg1Bxr08d/f38QAHCi0dYokamwLkGcToP+uUkAuke/INkUbYuKiorOrpYOO1htrxZ6cdW+gJ+vFDoiTWPvGh06EdQS4eqwX8vrcKyuFUfrWnHtaz/j1W/3ezU1e8IlEhSkdBjg2itIpMJ6pcdjdGE6AGBfZaP8XR1pKIIihH1umP8iSO9zirzvP6e4swzUHC3EypGTLUEdHigMcj3T4jGsZyoA4EBVk3x+nDFbJDz40a+wSMClQ3vgKduYjU+2HEVFXStqmtgosath9wVFV3+RYCNJkvwaRVRUfF7FzL+uglIErT9Ug3UHTgT0fCF0YmPUcnWtPR0WGk+Q0WzvsgxEJh0muuvH6zQwWyQ88cUu3Pp/v+Bkk/vvS084R8u8dY0Wpml/jNGAa9fow7by+F7p8UhL0Mk3rhsORcfnmSIoQrTHGC08Lq1Gi0Me3V9jNNB+c3SZTc1LkuMXWEcRxuj8tDikJehQmGltDucpGvTO+lJsOVKHJL0Wf7l0IE4vSMWoPukwWSQsWXtIToexPL7r0F3M0UdOtqC60YAYjQqD8qxjQhL0Wjm9UBlFBQ0d5VC19XN/es8UAMCLqwOLBrkb6ilHgkIkFp1FTyTSYdttImjWOUV4/Ioh0GnVWLWrEpOf/x77Kv3vxOxvJMhisc8YCzQSJM5X6Ql7JAgAzuhjjQZFS0qMIihCdMQYDTiaowOJBNnN0YF9oR5RNDQ8UBUcESRJkmyY65lmbQon0gDu+gVVNbRh0ZdWT9I9F54qC7pbzi4EALz982FZrGWwUWKXQaRJfy2v61D7hGhnk80LN7BHssNnvaulxJoNJlTYvn8WXDEEWrUK3++t9ssLKHDXwC9R0SsoFDiLnkikw7bZvI9D8lNw/ehe+PjOM1GYmYCjda14eY3/QlKcI61tmKmnc9ZkMMlNff0ZmwEoI0HWbYoeQb0zrCJolC0lFi39giiCIkSgc8MAIFYhcpQpqTazf54goH29glqNZlQr8rfBmv1S3WhAm8kClQroYeuMO8yLL2jB5zvQ0GrC4Pxk/HZsH3n5xAE56JMRj/pWEz7aXA6A6bCuRFFmIpICbJ/QGbE3SUxzWJ7VxRomiihQWnwMBuen4IrifADASwFEgxq8RIJCZYxudpqM7vx7qGk2mORoz5B8awRtYF4yHr5sIIDAGs022toI5KZYrweeqsNEFEirVvl1fQGUniBbJKjGUQSJSND2o3VRkeKlCIoQwhjt78gMwFp+GKOxKnflXYi/zRKB9o3OUEaBgOCJIOEHykmKlaNYxQXWC0BJWa2D4e/r7RX4uOQoVCpgwZQh0KjtU4g1ahVuPssaDRIRMqbDug5qtQqn2/xiXTklJl6biHwJ5AqxLpIOE+n0PrbU9x3j+0KtAr7ZWSlXefqiwU3Ztt0YHRpPkGs6LLxRyZ3HGmCRrKJYfI8D9nTxweomv71BQnzkpVpvPj2lw5zbEPiDvTrM2r1baYwW++yZFgeLBGw6HHlfEEVQBJAkqV3GaACI1bp2jfZ3dhgA5AhjdAB3lcK3IwhWOkxsV6TCAKB/jyTotWrUtRjlirTy2hb84YOtAIBbziyU+wkpuXpET6TE2QVlOqvDuhRd3RfUZjJjx9F6APYbAYGI3lZ1lUiQ7XMt/H9FWYmYbCudfnn1fr+24a5iKSnEfYKc01/hbpYomsKKKJAgNV4nD9r19/MhomV5PiJB7iJuvojX2yNBNU0GNLaZoFJZi18Eo/pET0qMIigCNBnMcpVBoCLI3RBVg61k3j9PkPVNX1Hnf9vyMlskqMj2pbW/qjEojeuEHyhfIYJiNGoMtZklS0qtIzTm/Wcz6lqMOL1nCv54UX+324rXaTF9dC/5d6bDuhZdXQQdqGqCwWxBSlwMCtLjHB7ravPDRCSoMCNBXjbnvL4AgP9tO+ZXpFk26+qV6bAYh8eCjXMkKNzVYUIEDbaZ5pUI4eyvr8o5EuRJONp7BPlnigYcI0GHbVGg3ORYB5+b7AuKAnM0RVAEECHL2Bh1wF2N43TWP5nbdJgf1WFCcFQ2tMnP84WI2IzrlwGtWoVmg1k2NnaEI06maIF9XtRJPLNiDzYePokkvRYvXDfcq9CbMa4PdBo1NGoVeqTEelyPdD6EV2x/VaPX7radFXFDUJAe55J2yO5i88Oc02EA0D83GaML0yFJ8KsJoFtjtO1CHar3R4uLJyjMIsgWKRzkFAkC7ClUf2fsifMnp8M8RoICqwwDFJ4gg0kuVBGpMMEZNhFUUlaLNlNkOm8LKIIiwMl2psIAZTpMIYKEMTrG958zI0GH2Bg1JAk45mc0SIiVwsxE9LKZ2/ZXdjwlZk+HOX5AhDH0863H8Moaa3j8iauGyvv2RE5yLP7vllH45w0jWB3WxchM1KNnmpgo759vpDMh/HH5qXEuj4n5YdE0+LgjHKp2TIcJZJNui+9ITr18cXZjjA5TJCiY/dJ80Wo0Y6+tUa1zOgywi6CS0lqfzRMNJotsociXPUHeI0GBpMPk6rA2s1wZ5iyCijITkJmog8FkwdYjkf08UwRFALspuh0iSMwPU3qCbP/X+xEJUqlU8hu//GRgIqhnWhyKMq2Nrg5Ud9wc7emLX3ygxXm6YUwvTB7aw69tjinKwETbfBrStejKTRM93RAAoTFGl55oxoMf/RrUnl/+UNdilHt59XESQclyibvvSI5bT1CIS+SFCBJm7HBGgnZXNMBkkZCeoHMb5T4tJwlxMRo0tJl8phOVIrFHqq/qsMC6RQNAnCIS5FweL1CpVHKVWKR9QRRBEaC2HXPDBPL8MDeRIH88QQCQb/uida768kS5wsDcN9vmCwqgMZc7JEnymA7rkRIn9zPqn5uEP08e2KF9ka6BEEGrd1dF9TDVOW9vwsRnvg1oQro/kaD6VlPQog9v/XwIb68rxZKfDgVle/4iokBZSXoXn0lynEhn+RYx7tI0oa4OE32CMm1R5nA2SxSdogfnp7it0tIqvJS+SuWFKTpep5GzEQ2tJrefKTFj0p/hqQLl7LBS0S06I8FlPSGCAintDwUUQRFAeILakw4TKa9WtyXy/v05hehwrvpyR7PBhGrbZPaeafHom2WNBO3vYIVYbbNRvpPKc/PFf9vZRRicn4yXpg93MNSR7sukQbnQa9XYePgk3v/lSKQPxy2NbSZ8/usx7KtsxH82lPr9PHdFAoLkOK382Q5WhZgoWxbiK1w4V4YpEZEgb3OsBJHpE2T9vhLtN8LZLFG0DnBnihbIA6h9REobbNGdRL1WPudmi+Q2siX6w4leVf6gnB0m9whKd41wXjq0Bz6ecyZeuWG439sOBRRBEaBGHp4aeCQoNsa1RN4+QNU/sSDuNo/48QUovpyTYrVIiYtBX1sp5oEO9goSUaCsJL1bkXPr2UX47Hdny6KLkIL0eNx9wakAgL99viMq++bsUQwYXvLTIb87XHuLBKlUqqBPkxf7q6gL7zl0VxkmSLa1uPDH2OyuT5CILDUZzDC3Y6ioL5ptJfGiwW04myUqI0GekM3RfkaCEmO1ttlr1siSu/MuRLeIRvqD8ATVNBlkH5uzJwiw9qwbVpAqz36LFBRBEaC9PYIAD+mwdkeCfIsgsU6BLYUmPEFH61rlKcHtobzWNjPMzZc+IZ645axCDMlPQX2rCY98ut3l8WaDCU9/vRv/LSmPwNHBoaP1sbpWfLGtwudzWgxmOdpa4MYTBCh9QcGJBImbG3+LI4KFu8owgT0SFEg6zLVEHgiNObpFjgRZ/xatRku7JrgHisFk75TuzhQtECJo9/EGrylBe3sBawPEJC/nvaoDkSBRHp8Uqw2oKXC4oQiKAB0zRrumwwJplgjYRZA/xmjnhoZpCTq5B09HTJWe/ECEeEOrUeOJq6wdw//3awW+2m4XGQeqGnHFSz/hhVX7cP+Hv4YkGuALcbESUYl//3DQp39JRGUS9VrZF+OMuBMPRq+gpjaT/B1kHV0TvrSOp8owwJ7O8i8S5GrY1WnV8miHUPiCRLooU9GDLBwpsT3HG2A0S0iJi/H6fZmdFGuvoPRScdXoNBU+2YuXSojuQESQiASJt33vjHi/u01HAoqgCNARY7S9Osz64ZMkyW6M9jOsKCpQKupbYfIRrreLFfsdqt0X1P6UmLvtEuIPg/JScPs5RQCAv3y8DXUtRny5rQKXv/gjdtvSUS1GMw6fCG/lE2AXQXPP7wedVo0tZbXyYFRPKFNhni4WwewV5OwDOl4XntJ7SZJwwIsIktNhPjxBZouEJpsgca5aspujQxcJSguzCJKbJOYn+xQTsi/Iy3vOuQGipzSk2SLJlXwBiSCd49+kd7rr3zqaoAiKADXCGN2OrsYiHdZqi/4YFCLG30hQVqIeOo0aZovks+lhmZvRFqJFe0fM0Ue8GEEJ8cW8CaegKDMBlQ1tuPqVnzB76UY0tJlwRp802bcW7mGrkiTJIuzMvpmYMsw6CuLfPxz0+jxvpmhBIOmwb3Ycx7z/bPYYUXGOAB8NU0qspsmAhlbrCAXnkmnAHpHwVR2mND47968JZddo4QFK0GnttoQwVIhtk03RnlNhgmLRaNaLL0iOBNm6bXtKQ55sNsBskaBSBTboO96pAXCBGz9QNEERFAFE2WH7qsMcP3zKrs/+TvlVq1XIs/WH8OULcpe2Ck4kyFVcEeIvsTEaLLxyCABgr61dw61nFeKd28ZguO1ueFeYRVBVYxtqmgxQqYB+2YnyUN8vt1V4rcT057PgbzrsQFUj5v5nEz7ZchSfbz3mfn9OkaBw+YJEZVheSpzbYggRkWhsM3lNZQpxp9eqXW78EvWhK5MX6bA4nUbu9B+OXkG/lnvuFO2MsnO0pzRso1N7AU9pSGGKTo/XBWReTnBqfeBO8EYTFEER4GSH0mGOnqA2hQjyNx0G2NNQvnxBsjFaoeaD0StIhOR70hhN2snoogzMn3gqeqbF4eXpw/HnSwciRqPGablJABwrtcLBngrr56FPRgLidBr0z03GWf0yYZHgtR+Pt8owQZacDvMsgkxmC+5etkWuHPXk2XMWZMfCVCEmBi/3yXR/UVSmtryVuXvrYhzSdJjtOzdep3FboBIKjGYLdh6ziiBvpmjBwLxk6DRq1DQZ5PJ0Z+yRIOEJct9kUoigQFJhgFWcqhVZO3fl8dEERVCYaTWa5buH9hijXdJhtn9jNCqo1f6bz+QyeS8iqKnNJKfulKF6USF2sLqpXdURdS1G+QPHdBjpCL+feAp+uO98XDLE3lFciKBwp8N2VVgvVqflJMnLbrFFg95dX+YxRRNIOqzKiyfolTX7HQbMempjIfYnLoLHasMjgrz1CAKsLT7ETZ43c7SI8rhr4BdKEaSMBMUrhoSGkn2VjTCYLEjUa/0SE3qtBoPyrb2EPKXEnBtNyk0qW9xHggIVQSqVysEX5GvcUaShCAozIhWmUasC6sIpiPWQDgskCgT41zBR3KGmxMXIdwviuTqNGm0mS7uarYkv4fQEnVxOSUiwECLo0ImmsM53EqJL7B8Azj01C0VZCWhoM+H9X8rcPs+fSJBIh51oMrgtZthWXofnVu4FAFw9oicAyCZkT/sTqZOwpcOqrd81fdz0CBKI75k6L+bohlbH6iYlwucSSmN0vE4rp8NC7QkSpuhBecl+3+T6migvRmHYq8PcG6Pl8vh2zGGMt1WIxWhU6JES3Te6FEFhRpkKa0/ZoLhTEmWt9uGpgXVVFned3kSMmADs7FXQatRySLs9viD6gUgoyUrUIy0+BhYJ2Hu84zPu/EWk35QiSK1W4aZxfQAAn2w56vIcg8mC47biBG+RoIwEHTRqFSQJck8hQavRjPnvlcBkkXDJkFzMtzWULD3R7FYwieivGFsQtnSYl8owgT8NExu8zLOyD1ENhSfIPm5CRORD7Qnabpsc761JojO+Jso7V4fJnqCW4KTDAHuvoJ5p8dAEkKGIBBRBYUaIoPakwgBFOszY0UiQ7/lh3nr5yINU21EhJvuBKIJICFCpVPaUWJh8QRaLhD02waUUQQBw9ilZAIAdR+tdOkhX1LXCIll9FN7uuNVqFTJt4xqcy+Sf/no39lY2IjNRj79NGYIeybGIjVHDZJFQ5vT5bjWa5YvbyD7WiEE4RJAkSV57BAk8XZCVyOkcfXg9QXI6LMaeDgu1J2hvpfX929/pPeUNIYJ2HK13GwmVmyXK6TAPkaAOiSDr+XHXKTraoAgKM/bKsPZ10NQ7GfJERMjf8niBuOs8VtfisRJDRGzcdbGVzdHtigT5Dv8T0hH651p9EbttPp1QU1rTjBajGXqt2iXd0zsjHkmxWrSZLC5m7SOKzum+IsNyhZiiTH7j4ZN43VaCv+jqIUhP0EGtVqFQ9u05fj6F4ImL0WBQD2t0oabJEPK04fH6NrQYzdCoVV5Lpj2lZpS4G54qCNX8MLNFkotQ4nUaOdIR6nSYGGsSyHdlfmocMhJ0MFkkt5FQeWyGc4l8kIzRgL1XULRXhgEUQWHnZAdGZgBArNZxdlig3aIFOUl6aNUqGM2SxwZs3iJBHSmTL2ejRBJiTrWZk8NVJi/2c0pOokv4X6VSyRO+nTv5+mOKFtjnh1kvTpIk4a+f7YAkWX1A5/fPkdctyhQz/hwjtcr9Jcdp5Tv2UM8QE5VqPdPivJZbi6iEt0hOvZvhqYIkDxf0jqKM+MTrtLI3M9TpMDF7KzvZ/9ldKpVKfj+56wPX4FwdJs65szG6A54g0TWakSDiQv/cZNx+bhEuHJTbruc7l8i3Nx2m1aiRm2L9YHkqk/fW1bkoq/3psCOcG0ZCTLjL5MV+Ts1xn7YY2jMVgBsR5IcpWuDcNfrTrcdQUlaLeJ0Gf7zoNId1RUNTZ3O00o+nUqnk74BQN0z0VRkmkBsm+mGMdhcJClWfIOEHUqms38HhSIc1tpnk1JX4O/lLjk00HXcjggLtE9SeSNC0Ub0wujDdoWozWmFpTpgZ0TsNI3qntfv5oirBWQTpYwLXsz3T4nDkZAuOnGzByD6uj8vdotPdeIJsX7KVDW2obzU6VI/5QhZXbrZLSDAQIuh4fRtqmw3t9uD5i6gM8+TdGGoztv5aXuuwPJAZeiIddry+Da1GM578YhcAYPa5fV2mfAuxcdA5EuQkuvJS4nCgqilskSBvlWGAn8Zof9JhQe4Y3aLwA6lUKrsICmGJvPibJOq1srjzlxwhmJ1EkMlskYWb69gM+2tpM5nlCr1AJsgLJg3KxaR23uiHG0aCOhke02EBRoIARcNENxViDa1G2b/kLhKUHBsjh+cDiQbVNdu362liNiEdJVGvlYVFOFJico8gmxfJmSG2dNiuYw0O/puA0mHJ9l5Bi386hPLaFuQmx+K2s4tc1hUi6ICTJ8h5fyLC4K85+kBVIxb/eBBNAYoMIYLEzZMn/Jkkb+8TFD5jdLNcHm/9/g1HOkxEcYSgCYQcm3BxToc1tdmPN0Fulmj912CyyO9NUYGo06g9DvXtKlAEdTJiPVWHBegJApQNE117BQlhlBYf4/EuROR7jwbQK2i/7Us5NznWpb06IcFENC0MddPEVqMZh05YP0OeIkFKs6pSlNkjM75vCMQd+d7KRry0ah8A4N5Jp8nRYSWievN4fZuDYHFOceeJdJifn+GH/rsdj3y6A1NfXRtQf6FDfkeCfE+S9x4JCs3sMGWjRABhSYcJERRoKgwAclLsUUMlor2AcuRIgk4rd3gW511EkLKS9FE9AT4YUAR1MkSJvMkiwWi22PsEtUME2Rsmun6ZHanxbV4WA2CF2dsfxKgNUV1GSKgIV5n8/qpGmC0SUuLs0VFnVCqVHA3aeqQWgLXiSAiJQIzRh080o6HNhEF5ybiyON/tuinxMciwfT6V4zOc02E9bP/6kw4zWyRssjXg23GsHr958Uds8dCLRonBZJE9Qf5Hgnx3jPY+NsPocXaW4JMtR/H93iqv6wjkRokx1u3Hh6FZooji5LQjHeXJE+RcHg9Y2y+IG10RgRN+oMx2+IE6GxRBnQyl96fVaLZ7grSBNUsEFA0T3YkgPxoaptt8Fieb/BdBwqgp7lQJCRXhGp+h7BTt7a5Z+IKEObqyoRVGswStWoUcPy422U5pkQcnD/DaRdieErN+5kxmi3xh7emUDjvqhwjaW9mAZoMZCToNTstJQlVDG67551p8ttW1CaSSPccbYDRbRaIvA3iSH5Pk/TFGG82Sw1xFZ47VtWDefzbjzrc3+RRLgN0YLSJBcbYS8JCmw2x/k5z2RIJs7xUXEeRh7pq9Ks8qMDtSGdbZoAjqZOi1aojv2Vajpd19ggC7J+dIbYvLDLAyPwybqQnWD05Nk/+VGHIkyMcdISEdRa4Qq2hwudA98/Vu3Pv+Fpfmhe3BlylaICrEfrWJIHHzkZsSC60fnr7MRL382Z84IBvj+mZ6XV9EXYQ5+lhdK8wWCTqNvTFjXoq9X5gvSmyzqIb2TMUHd4zF+f2z0WayYO47m/H69wc8Pm+HrevxwB7JPlMrzhdjd3gTQQk6rXyOvPmChI+xodXkVSwJlMNTAXtEPhyRoNwAyuMF4jknm43yNQJwLY8XOPcK6khlWGeDIqiToVKpFOZoc7tL5AHrl69aZQ1XVzc55o7tkSDP6TARCaoNIB0mR4KyGAkioaUoMxFatQoNbSYH8//PB07g+VX78MHGI/hhb3WH97PbzbgMd4heQdaIiimg8ngAiNGoMSgvGfE6De6/uL/P9UXDRGGOFvvLS42VI0g9Uq0Xy9pmo88L+hZbGu/0glQkxcbgXzeOlAfELvpyt8eGiztsU9AH5bk3jSvxlQ4zWyRFSsc1HaZWq5Co810mr5yw7m1OmcDZGB0OT1CFzc+T0w4RlBIXI98YK5trNrZ6EEFOQ1QpgkhUo+wV1BFjdIxGLd8xOPuCxO8FXsrYhSeoxk8RZDJbcNjmDeibTRFEQotOq5abeoo+PpIkYaGttBwAPtt6rMP7kdNhHnoECbKTY5GbHAuLZJ0JdSSAyjDBf24bgzX3jke/bN9jFOQyeduNh7tKtCS9Fgm2C7qvaJCYSj6sIBWAdQj0nycPQGaiHgazBduP1rl9nlg+0B8RZLsYN7SZXKLTgKPh2V0kSLncmzn68Am7CPLmPxLYjdFa27+hnyJf2QFjtEqlcpsS8zR8NsmpUzdFEIlq7BViFrSZ2y+CAPe+ILNFku+U/IkE+esJKjvZAqNZQmyMGj3acXdDSKCI6IyoyPpiWwW2lNXK1TBf76iQbySc+WlfNf788a9ey8Hrmo1yefmpfsx3EuboLWW1XpuReiIpNsbv7sF9FekwSZLsM/sUlWgqlcovc3RTm0kWkkIEiefLAzttIkmJxSLJ6bBBeb6HgIpIkCQBjW4Ehoju6DRq+XvQGXFB95YOK1NEgrxVoglEP6D4mPCkw8wWSe4M3p4SecBuqFZWiInBskke0mHinNETRKIaeYiqSWmMbt+f0t0g1f+WlKOh1YSUuBivbc/TbJ6gk83+eYKEH6goM9GroZOQYKE0RxvNFjz11W4AwJzz+iE7SY+GVhN+2OdaIWQ0WzB/WQmW/lyK/6wv9bj9PbYBl/mpcX41DLU3TaxTiJLQNA3tlREPlcoaValqbJNT3M6Rpx5+mKO3ldfBIlm9Js6RCW8iqLSmGU0Gsy0q59sHGBujkW/o3EVovPmBBImxvtNhh2vsFXOBpMPCVSJ/orENZosEtar9QsReJm//uzZ6iAQxHUY6FXrFXUh7Z4cJhPG53DbKwmi24Nlv9gIAbj+3yOPdFmCff+ZvJEh4E3yVyRISLJS9gt7dUIaD1U3ITNTh9nP74uLB1o62n2+tcHnel9sq5Dvob3Ye97j9XRX++YEEQ21RlK1H6lDuQZQEC71WI3++D1Y1efQgCRF0zEuvoBJbKbwyCiQoLrB2wN9sK59Xst0WBeqfm+SX+Rvw3jDRHxHkT8PEUod0mO+UlrMnyJ4OC40IEqbozES93+fNGXskSJEO82mMtrYWECLIU8uHrgRFUCckWJ4gQNkw0foFuOyXMpTWNCMzUY+Z4/p4fW66zRPU0GbymFJQsr/S5geiKZqECSFO9lc14jmbuJ834RQk6rWYPDQPgDUlpqygAYA3fzwo/3/DoZMezf8/HzgBABjQwz8RNMQWCTpY3YSymsCM0e2hSDZHNykGFzuLIFuFmJs5UwKlKdqZoT1ToFZZI0nOKTXhB/LHFC3w1jDRW48ggX1+mHtxU9tscCjB9y8d5myMtu6jzWSB2Y13qaOI89geP5DAnSfIUyRIbk3QYkJDm71iLpPpMBKN2NNhlg5VhwGK0RknW9BqNOOFldZOtHPO6yt/0D2RHBsjeytqW3xHgxgJIuEmPzUOCToNjGYJ1Y1t6JMRj+tG9QIAjOydJqfEftxnrxLbUlaLTaW1iNGo0DMtDmaLhNW7K1223dBqxDc7rFGiiwf7NygyPUEnixCD2QKVyl6hFQqEOXp/ZSOO1lovhp7SYV4jQU6maCUJeq08LqSkzDEaJCrDBvrhBxJ4qxBz1+zPGV9do5WVYYDV1+WLZqOTMVoRIQ9FSux4Q/srwwS5brpGy+fPpTrM3ppARIGS9Fq33ci7GhRBnRDZGG1QeIK8pK28ka/oGv32ulJU1LciLyUW14/u5fO5arVKHkx50o9eQfurGAki4UWtVjkYlu+ddBpibDcMarVKnnKtrBJb8tMhAMClQ/MwZZi1I/OKHa4psa+2H0ebyYKirISAIh2n2/oFAdZ0Q3sanfqL8OGsP1QDg9kCjVrl0ndGGKM9zQ+rrG/F0bpWqFR2Y7cznnxB2xU9gvzFWzqrPqB0mPvvJGVlmHWbARij5dlh9n5toTBHy40S22mKBpQDdxWRoDYPniBFn6Du5AcCKII6JXI6zGSWw/j6dkaC8mx3oS1GM579Zg8A4HcTTvH7izktXjRM9B4JOtlkkNcRd6eEhIP+tijF6T1TMHmIY8RGiKAVO46jzWRGZUMrPrV1QJ45rg8mDswBAHy7u8olZfbfknIAwJRh+QHNV1IKiVCmwgB7r6Bfy61pqdxk18aMPXwMURV+oFOzkzzOESy2RYiUIqiyoRVVDW1QqfxPFwLeJ8n7kw5L8pEOc44EtccTpFKpQloh1pFGiQKvJfJ6547RdmM0RRCJepRDVA0dLJHXazWy+a2h1YTeGfG4ekRPv5+f5mfDRJEK65HCwakkvNxyViEuH5aHp6853UWsKFNiP+ytxjvrSmE0SxjeKxWnF6RiaH4KspP0aDKYsXb/Cfl5lQ2tcgrt8mF5AR2PqBADgPwAyuPbQ6EtEiQaZrszYQsRVNdidNv3xpspWlDcy2qO3lpeK3fhFqXxRZkJPlPrSrwZo0XqylslnhwJ8pQOs0WCMhOt310BVYcpIu7i/83G9vUKuu+Drbj8xR/cnnP7BPmOiCDrc5sMZjkC5CmdqDRGUwSRqCdWvgOxdNgYDTgaJedPPFVOF/iDvw0TmQojkaJfdiKem1bstsGgMiX2cclRLP3ZWg4/88xC+fEJA6zRIGWV2Odbj8EiWYVBbx+T0Z0ZrIgEeRtLEwx6JMfKkWPAfTl+UmyMHD1xFw3yZooWFGUmIDlWi1ajRW4eKafCAvADAd6N0aKMv4cXw3Cijz5BIhI02CZGAzNG28VDXAeGqO6uaMB7v5Rhy5E6rD9Y4/J4RybICxL0WvnvKozWHjtGK85ZJUVQ8FiwYAHGjRuH+Ph4pKamujy+ZcsWXHfddSgoKEBcXBwGDBiA5557zmW9NWvWYPjw4dDr9ejXrx8WL17sc99bt27F2WefjdjYWBQUFGDRokVBeEXRgTw2w9SxsRkCcTd6ak4iLjs9sLtafxsm7q+iKZpEJ0IEfbrlKKob25CTrJfL5wHgQltK7JsdlfIMso9LrCmzKQFGgQDrBafIlhIOdTpMrVahj0KkeSrHz5XN0Y4iyGKRsLXMmkrzFglSq1UY1suxVN7eJNF/PxDg3Rh9TB794aWTvS1FX+mh2k2IIFGp51ckyOg4QBXo2CT5d9Ydlv8vxKISuTqsg01lxdBdcS48RoLi7ANhRedwiqAgYDAYMHXqVNxxxx1uH9+4cSOys7OxdOlSbN++HQ8++CAeeOABvPjii/I6Bw8exOTJk3HeeeehpKQEd911F2699VZ89dVXHvdbX1+PCy+8EL1798bGjRvx1FNP4ZFHHsFrr70W9NcYCeJ09hL5NtkY3f4/5ZXF+eiblYC/Xj4YmgCbGPo7RPUAI0EkShEpMcFvx/R2iIaO7ZuBeJ0GFfWt2FZej4PVTdhSVguNWiWX2QfK9DG90TsjHuecktXh4/eF8sbDU+RJmKOPOo3O2F/ViIY2E+JiNDg1x/tn19kXJFeGBWCKBrx7go7aRJC3ijrRmXrP8QaXbt8Gk0V+jWK99pTIA4p0WIAiqMVgxvLN5fLv4jwpHxcG8PZMkFciV4g1tMKimLvmHAlS/i6+q7tDt2gACKk549FHHwUAj5Gbm2++2eH3oqIirF27FsuXL8fcuXMBAK+++ioKCwvx9NNPAwAGDBiAH374Af/4xz8wadIkt9t9++23YTAY8MYbb0Cn02HQoEEoKSnBM888g1mzZgXp1UWOYA1QFZzXPxvn9c9u13P9HaLKSBCJVkRKbPFPh6DTquUSekFsjAbnnJKFL7dXYMXO49DYfEVn9sts993yLWcVysNHQ43oFQQA+anuPUh5toulc58f4Qcakp/is2mfXCFWVovGNpM8syzwSJC9Z40Sk9kil457i6DlpsSiR0osjtW14tfyOowpypAfK69tgSRZBYyonGuPMRpQNEwMsET+061H0dBqglatgkkxVkQgTNFxMRqXUvZAEQ0TK+raHMaQOFeHaTVqJOq1aGwz4YDtu5qRoAhRV1eH9PR0+fe1a9di4sSJDutMmjQJa9eu9biNtWvX4pxzzoFOp3N4zu7du3HypGtXUwBoa2tDfX29w0+04jA7LAieoI7gjyfIaLbIZkRGgkg0Mn10L6Qn6HD7OUXIcHMHfIEtJbZix3FFVVj7okDhRlmN6TMd5hQJkk3RNoHjDZEuO1jdhJ9spvHc5Fi359Mbcs+aNscITWWDdZSEVq3y2cRvmJtqNQDyAOde6fFIibdHnNwNaxVIkiT3AnJMh1mFRGuAkaB31tl8Z7ZmtAermxx6GikbJQZSdegOMWfueH2r7AfSadRuq39FiqzJ9noogiLATz/9hPfee88hWlNRUYGcnByH9XJyclBfX4+WFvfNvTw9RzzmjoULFyIlJUX+KSgo6MhLCSmxuuBVh3UUf0ZnlNY0w2SREBej6XCOm5BQcEpOEjb95QLcc+Fpbh8/r3821Cpg57F6HKhuQmyMGhcOynW7brRRqIi+5nlII+XZukYfdfIEyaZoRW8jT6TG6+RI7zu2eWv+TI53xlMkSAi03JRYn2l7EZVybt4oBqf2yoj3OaxV0Gq0yNV17ozRgUyS33G0HiVltdCqVbj93L7y9+FORUqssqHjPYIEucIT1NDqsUeQwLnijiLIA/fffz9UKpXXn127dgV8INu2bcPll1+Ohx9+GBdeeGHAz+8oDzzwAOrq6uSfsrKysB+Dv8TaBE+LUTlANTKdPdP9GKIqcsxFWQkcnEo6JekJOozsY49QTxyQ47FnTrQxsEcy+mYlYOKAHI/fE7lu0mGtRjN2HbNNjvcjEgTY54h9u8c6lDbQVBjgWK6tpNwm0IRg88YweZ5ZrWxmB+yNEnulx/sc1ipQihz3JfL+R4LEMN5Jg3KRlaSXz48yJRYsUzRgL5OvqGtV9AjyIILi7MvVKiAjoXuIoIA/xffccw9mzpzpdZ2ioqKAtrljxw5MmDABs2bNwp///GeHx3Jzc3H8uGO31uPHjyM5ORlxcR5Cux6eIx5zh16vh17fOf7oyj5BcrPEKI4E2f1ATIWRzssFA3LkcmbRSbozEBujwTd3n+t1HREhOlrXgoZWI77bU43Pth6FySIhM1Eve4Z8UdwrFR9uOiJHTtolguLs1WGSJMkpIXtlmO9jGZKfAo1ahcqGNhyra5WryURlWO8MqzcqJS4GVQ1tqGsxomea+20JP5Beq3aIQAl/kL/psGaDCR/bDNGiI//AvGSs3FUpz1gD7J6gjpqiAWU6rM2jKVqgjASlJ+gDLpLprAQsgrKyspCVFbyKhu3bt+P888/HjBkzsGDBApfHx44di//9738Oy1asWIGxY8d63ObYsWPx4IMPwmg0IiYmRn7OaaedhrQ0D+/0TkScwhMUjD5BHUGIIDFE1d1xCKNdX5qiSSdm0qBcLPpqF9LidTjn1NBXdQUTX96SXFt0paHVhOF/XQGj2R49uXBQjt/elGKniNHAHoH1CALsF2OLZPWniIu2vTLMdyQoTqdB/9wkbLeln5xFUEF6vG1fWlQ1tHk1Rws/ULzTHK1AJ8l/uuUoGtpM6JMRj7E2s7YQicoyeblRYlLHRZCI8FU2tMrdtj2lw5Rl890lFQaE2BNUWlqKkpISlJaWwmw2o6SkBCUlJWhstF4Ut23bhvPOOw8XXngh7r77blRUVKCiogJVVVXyNmbPno0DBw7gj3/8I3bt2oWXX34Zy5Ytw/z58+V1XnzxRUyYMEH+/frrr4dOp8Mtt9yC7du347333sNzzz2Hu+++O5QvN2yISFBTmwnCz9eR6rCOkBzne4jqfjkdxkgQ6bz0yojHR3eeiQ9mj4vYTUeoSNRr5TYBRrOEoswE3H5OEd6fPRZ/u3yw39s5LSdJvklLitWiID3wPkixMWrEaKxfKso0lWiU6K1HkBK7OdrqC5IkyR4JEiLISzm+oNlNo0TAezps2S9leGbFHqzZXSlvWxiirxvVS7YFiDL9vccb5RvaYEyQF4gyd6PZ/to9VZyJcwF0LxEU0qT2Qw89hCVLlsi/FxcXAwBWr16N8ePH44MPPkBVVRWWLl2KpUuXyuv17t0bhw4dAgAUFhbi888/x/z58/Hcc8+hZ8+eeP311x3K46urq7F//37595SUFHz99deYM2cORowYgczMTDz00ENdojwesM8OU35wI/WlrLENUa1pMuBkk1Ee2qeEkSDSVRicH3hko7Pw6m9H4NcjdTjrlMx2V3FqNWoM7ZmCdQdrMLBHcruqm1QqFZJjY3CiyYD6ViPyIEzbtnSY36m5NLy9rlSucKtuNKDZYIZKZa+SS7Fd+L01TBSeIOeJ6p7SYRV1rfjjB1sVr8daFbuvshE6jdphLFHPtDgkxWrR0GrCvspGDMxLlqe+d2RkhkCnVSMjQYcTTQbsr7TejHoaPqtMh3WXHkFAiCNBixcvhiRJLj/jx48HADzyyCNuHxcCSDB+/Hhs3rwZbW1t2L9/v4sn6ZFHHnF5ztChQ/H999+jtbUVR44cwX333Re6FxpmRCRI+cGNlCcIAFK9DFGtaTLIpmkOTiUkehneKw0zxvXpcBuLcX0zAQAj+7TfemD3BdnTVMfaGQnaeqTO2qbDFgnJS4mTDeLeulML3DVKBIA4nb3LspKyk83y+r0z4iFJwL5K643gRYNzHVoGqFQquZnk9qN1sFgkuTosGJEgwC6mhDfTY3VYXPdMh3WO8gbigLJPEGB18vtqZBZK0uN1OIAmtw0TRRQoPzUuoCGKhJDOyezxRSjKSsCEAe1rwArYy+SFj6XFYJZvsvypDgOs88xElGV3RQNKa6yREGWKTjk93RPuhqcCQLyHdJgQa0PyU/De7WNR2dCKTYdP4mB1M6aOdB1OPSjPGjnbcawe5zUbZD9WdpCESE6yHjuOAfttQsx5grwgKZbpMNJJiHUakRFpf4K3honsFE1I90Kv1QQ8g9CZJKcyedEjKF6ncYhYeEOtVmFYQSq+31uNzaUn5dE+vdPt30UpsifIizHaYyRIzA5z6mckDNy2SE52UiwuGtzD4/aV5mjhB8pM1AU0yNobIhLU4GFumECZDguWAOsMdC13Xzch1umOJFKmaIG3IapyjyCmwgghfmKP0Fgv3MpUWCA+I3meWVktDtsiQb0y7KND/EmHCU+QizFaiCAPkaBcPyNWoqHkToUICoYfSOC8LX/6BDESRKIa57CsPiYyjRIFqV4aJh6oZmUYISQwnMVJuVN0xV+KbZPtS8pqkWGLWPdKt4sgv4zRbkZmAIp0mBtjNOBfPyMA6JedCJ1WjYY2E345bK1kC2Znfb9FUDdNhzES1AnpTJEgMUSRpmhCiL84l64fs3WL9jY41R2n2yJBB6qasKvC2v1aKYL8KZH3nQ5zigTVB9bxOUajxmk5SQCA1bsqAQSnUaLAefwG+wQ5QhHUCdGoVXIfDSCylWGAZ0+Q2SLJAwspgggh/uI8P0xulOhnikmQnqBDH1v6S4yN6O02HebZEyQboz2UyLukw9pxrMIXtPu4VaiFMhLkqU9QTnIsEnQa5KXEdnh6fWei+7zSLkZsjAZGs20qcKRFkIdIUPnJFhjNEnRatd9lrYQQ4hyhOVrn/8gMZ4YVpOKQbWZYUqxWToEB/vYJskWCYpw9Qa4l8kazBVWN1j4/gZS4Ow+aDcbwVPu2nNJhHiJBCXot/vf7s6HXajo8vb4zwUhQJ0WZEou0CPI0RPVAtbUyrE9GfLeZQ0MI6TgiQiOiN3KjxHbcTIl+QYA1CqS8wMsGbK/pMGGMdu8JMpgsMNta91c2tEGSrBYF4UHyB+cZa8E0Rmck6KBVfP96G/zbOyMhaP2JOgsUQZ0UZZl8xNNhHiJB9AMRQtqDUpxIkhRwo0QlwhwNOPqBALvYajaYYTRb3D7fUzpM+buoIBOpsJwUvTwawx/65yZDGXwJphBRq1UOJe/KfkCEIqjTEhdFkSDnIaoCuwhiZRghxH+SFNVhdS1GWYgEWh0GAAN6JMvfkb3SHW/IlGZgT2Xyngao6rVqWbiIdYRY65EcmFhL0GtRmGE/tmB6ggD7NHnAc5+g7gpFUCfFIR0W4eowT0NUhQhijyBCSCDIhuVWE47aKsMyEnQulbH+oNOqMcQ2860w0zESpNWo5fSQp4aJzR6qw1QqlZwSExViHRl+KnxBeq3awbcUDITHSKtWRTxzEG3wbHRSYrXREwkSQ1QB4GST/W5KNEosZLdoQkgAKMdZyJVh7TBFC/5y6UDcclYhLh+W77ovW2TEkznang5zjaCIlJhYR44EteNYxUT5nOTYoBuTRWQpMVbbrUzP/sC4WCclVnFXotdGtlkiYB2iWtNkkOf7tBrNckUHPUGEkEAQkSCTRZJH7/g7M8wdwwpSHQzSDvuKi8HRulbP6TAPxmjAtWu0GO/Rox3prLF9MwC4mqSDgUiHeTNFd1d4RjopsYroT6QjQYDrENXSmmZIkjX/HEiVBCGExOs00KhVMFsk7LY1OQxVmw1fDRM9DVAF7GXzLU6RIH9HZigZVpCKFfPPQc+0eN8rB0gORZBHIn/1JO0imkrkAdeGicqZYQy/EkICQaVSyWmqXbIICk3ptog6eUqHeeoYDbimwwIdmeHMKTlJLlVowaC4Vyq0ahVO75ka9G13digLOylxUWSMBoC0eFuvIFs6jOXxhJCOkBwXg5PNRuyrtKbDAu0W7S/yJHk3XaMlSZJnhzkPUAXs38PNBhNMZgsqG9pvjA4lfbMSsfEvF3SrTtD+wjPSSYmmPkGAPRIkGiYetDVKZHk8IaQ9iAiNwda/J3TpMM8NEw1meyNEdxEaER1qNZpR2dAGi2StwMpMiL7ZW8GuOOsqRP7qSdqFMh0WDSLIeYiqiAT1yQx+fpsQ0vVx7mcTqnSYt9EZyuGovtJhwg+UkxwbUKNEElkif/Uk7SLqPEHxjp4ge48gRoIIIYGTrOhsrFGrkJ0UWk+Qu+ow4fWJ0agQ48Z2YE+HmTvsByKRIfJXT9Iuok4EJdgjQXUtRlQ3WsUQI0GEkPYg0lSAtc9NqOYPJnuJBHmrDAMc02GiPL49lWEkctAT1ElReoKiwRitHKJ6yBYFykrSc04NIaRdKCNBoYyuyMZoNx2j7ZVh7i+VyknyHRntQSIHRVAnxXF2WDQ0S7RHglgZRgjpKMkKI2+oKsMAe8foBreRIM+NEpXLmw1mOZ0W7LlfJLREPoRA2kW0GqMb2kzYc9za14Mzwwgh7SVZYYwOVWUY4CMdZnQ/QV4gbkaV6TB6gjoXkb96knbhkA6LAhGkHKK6ubQWACNBhJD2o4wEhScdZoQkSQ6PeWuUCCirw0wd6hZNIkfkr56kXUSbMVo5RLWkrBYARRAhpP04eIJCmQ6ziSCjWUKr0eLwmLfhqYBdHDW2mVDZ0AaAnqDORuSvnqRdRJsIAqxDVAH7MMEiTo8nhLQTZZ+gjkyQ90WCbU4Z4JoSk4eneqgOE+mwwyeaYbZI1kaJidHXKJF4JjquniRgos0TBNh9QQCgVgEF6SyPJ4S0D2U6LD+EniDlnDLnrtHNfqbDymutfqCcEJbyk9AQHVdPEjBxUSiC0hTT4numxUMfBVVrhJDOiRAUmYn6kI988GSOtqfDPFWHWcWTsBJF28ww4huWyHdSHPsERYfYEENUAfqBCCEdIz1Bh7dvHY2UuBioVKGNrnjqGt1i9B4Jcl5OEdT5oAjqpESjJ0gZCaIIIoR0lDFFGWHZj7JCTInoE+TJGB3r5BXKowjqdETH1ZMETDSKIKUniCKIENJZECM66poD8wS5RoJYHt/ZiI6rJwkYZTosajxBFEGEkE6Ip9EZvvoEOS9neXznIzquniRgdBo10hN00GvVITcN+gvTYYSQzognT5CvAarO6TB6gjof9AR1UlQqFd6dNQatRjMS9NHxZxRDVHVadUjb3BNCSDDxVB3ma4CqXquGWgVYbNVhoWzqSEJDdFw9Sbs4NScp0ofgQP/cZBRlJWBUn3T2yiCEdBqSPRmjjd4HqKpUKsTrtGhsM0GjViEriY0SOxsUQSRoJOi1WHn3uSEvZyWEkGAiN0tscfQE+eoTBFhTYo1tJmQn6Xnz1wmhJ4gEFQogQkhnw3c6zLMIEo/RFN05oQgihBDSrXHXJ6isphkV9dbJ8DnJngWOXQTRD9QZoQgihBDSrRHVYcpI0DvrSyFJwNmnZHoVQSJVxsqwzgk9QYQQQro1olliY5sJFosEk0XCsg1lAIDpo3t7fa4on2c6rHPCSBAhhJBujYgESRLQ0GbCl9srcKLJgJxkPSYOyPb63OJeqdCoVTijT3o4DpUEGUaCCCGEdGtiYzTQa9VoM1lQ32LE2z8fBgBMO6MXtBrvsYJ7LzwNd47vFzX92khgMBJECCGk2yMqxDaVnsS6gzXQqFW4blQvn89TqVQUQJ0YiiBCCCHdHlEh9sqa/QCACf2zaXbuBlAEEUII6faIhom7KhoAADeM8W6IJl0DiiBCCCHdHuUg6t4Z8TirX2YEj4aEC4ogQggh3Z5khQi6flQvqDkCo1tAEUQIIaTbI8rkdRo1rh7RM8JHQ8IFRRAhhJBuT16qdezF5KE9kJHIafDdBdb1EUII6fbcMKYXkmK1+M2wvEgfCgkjFEGEEEK6PUmxMawI64aENB22YMECjBs3DvHx8UhNTXV5fMuWLbjuuutQUFCAuLg4DBgwAM8995zLemvWrMHw4cOh1+vRr18/LF682Ot+Dx06BJVK5fLz888/B+mVEUIIIaSzE9JIkMFgwNSpUzF27Fj8+9//dnl848aNyM7OxtKlS1FQUICffvoJs2bNgkajwdy5cwEABw8exOTJkzF79my8/fbbWLlyJW699Vb06NEDkyZN8rr/b775BoMGDZJ/z8jICO4LJIQQQkinRSVJkhTqnSxevBh33XUXamtrfa47Z84c7Ny5E6tWrQIA3Hffffj888+xbds2eZ1p06ahtrYWX375pdttHDp0CIWFhdi8eTOGDRvWrmOur69HSkoK6urqkJyc3K5tEEIIISS8BHL9jrrqsLq6OqSn26fxrl27FhMnTnRYZ9KkSVi7dq3Pbf3mN79BdnY2zjrrLHzyySde121ra0N9fb3DDyGEEEK6LlElgn766Se89957mDVrlrysoqICOTk5Duvl5OSgvr4eLS0tbreTmJiIp59+Gu+//z4+//xznHXWWZgyZYpXIbRw4UKkpKTIPwUFBcF5UYQQQgiJSgIWQffff79b07HyZ9euXQEfyLZt23D55Zfj4YcfxoUXXhjw85VkZmbi7rvvxujRo3HGGWfgiSeewA033ICnnnrK43MeeOAB1NXVyT9lZWUdOgZCCCGERDcBG6PvuecezJw50+s6RUVFAW1zx44dmDBhAmbNmoU///nPDo/l5ubi+PHjDsuOHz+O5ORkxMXF+b2P0aNHY8WKFR4f1+v10OvZIIsQQgjpLgQsgrKyspCVlRW0A9i+fTvOP/98zJgxAwsWLHB5fOzYsfjf//7nsGzFihUYO3ZsQPspKSlBjx49OnSshBBCCOk6hLREvrS0FDU1NSgtLYXZbEZJSQkAoF+/fkhMTMS2bdtw/vnnY9KkSbj77rtRUVEBANBoNLLQmj17Nl588UX88Y9/xM0334xVq1Zh2bJl+Pzzz+X9vPjii/joo4+wcuVKAMCSJUug0+lQXFwMAFi+fDneeOMNvP7666F8uYQQQgjpRIRUBD300ENYsmSJ/LsQJatXr8b48ePxwQcfoKqqCkuXLsXSpUvl9Xr37o1Dhw4BAAoLC/H5559j/vz5eO6559CzZ0+8/vrrDj2CqqursX//fod9//Wvf8Xhw4eh1WrRv39/vPfee7j66qtD+GoJIYQQ0pkIS5+gzgj7BBFCCCGdj07dJ4gQQgghJBxQBBFCCCGkW8Ip8h4QWUJ2jiaEEEI6D+K67Y/bhyLIAw0NDQDAztGEEEJIJ6ShoQEpKSle16Ex2gMWiwVHjx5FUlISVCpVULddX1+PgoIClJWV0XQdBni+wwvPd3jh+Q4vPN/hpT3nW5IkNDQ0IC8vD2q1d9cPI0EeUKvV6NmzZ0j3kZyczA9RGOH5Di883+GF5zu88HyHl0DPt68IkIDGaEIIIYR0SyiCCCGEENItoQiKAHq9Hg8//DAHtoYJnu/wwvMdXni+wwvPd3gJ9fmmMZoQQggh3RJGggghhBDSLaEIIoQQQki3hCKIEEIIId0SiiBCCCGEdEsoggghhBDSLaEICjMvvfQS+vTpg9jYWIwePRrr16+P9CF1CRYuXIgzzjgDSUlJyM7OxpQpU7B7926HdVpbWzFnzhxkZGQgMTERV111FY4fPx6hI+5aPPHEE1CpVLjrrrvkZTzfwaW8vBw33HADMjIyEBcXhyFDhuCXX36RH5ckCQ899BB69OiBuLg4TJw4EXv37o3gEXdezGYz/vKXv6CwsBBxcXHo27cv/vrXvzoM5OT5bj/fffcdLrvsMuTl5UGlUuHjjz92eNyfc1tTU4Pp06cjOTkZqampuOWWW9DY2BjwsVAEhZH33nsPd999Nx5++GFs2rQJp59+OiZNmoTKyspIH1qn59tvv8WcOXPw888/Y8WKFTAajbjwwgvR1NQkrzN//nx8+umneP/99/Htt9/i6NGjuPLKKyN41F2DDRs24J///CeGDh3qsJznO3icPHkSZ555JmJiYvDFF19gx44dePrpp5GWliavs2jRIjz//PN49dVXsW7dOiQkJGDSpElobW2N4JF3Tp588km88sorePHFF7Fz5048+eSTWLRoEV544QV5HZ7v9tPU1ITTTz8dL730ktvH/Tm306dPx/bt27FixQp89tln+O677zBr1qzAD0YiYWPUqFHSnDlz5N/NZrOUl5cnLVy4MIJH1TWprKyUAEjffvutJEmSVFtbK8XExEjvv/++vM7OnTslANLatWsjdZidnoaGBumUU06RVqxYIZ177rnS73//e0mSeL6DzX333SedddZZHh+3WCxSbm6u9NRTT8nLamtrJb1eL/3nP/8JxyF2KSZPnizdfPPNDsuuvPJKafr06ZIk8XwHEwDSRx99JP/uz7ndsWOHBEDasGGDvM4XX3whqVQqqby8PKD9MxIUJgwGAzZu3IiJEyfKy9RqNSZOnIi1a9dG8Mi6JnV1dQCA9PR0AMDGjRthNBodzn///v3Rq1cvnv8OMGfOHEyePNnhvAI838Hmk08+wciRIzF16lRkZ2ejuLgY//rXv+THDx48iIqKCofznZKSgtGjR/N8t4Nx48Zh5cqV2LNnDwBgy5Yt+OGHH3DxxRcD4PkOJf6c27Vr1yI1NRUjR46U15k4cSLUajXWrVsX0P44RT5MVFdXw2w2Iycnx2F5Tk4Odu3aFaGj6ppYLBbcddddOPPMMzF48GAAQEVFBXQ6HVJTUx3WzcnJQUVFRQSOsvPz7rvvYtOmTdiwYYPLYzzfweXAgQN45ZVXcPfdd+NPf/oTNmzYgHnz5kGn02HGjBnyOXX3/cLzHTj3338/6uvr0b9/f2g0GpjNZixYsADTp08HAJ7vEOLPua2oqEB2drbD41qtFunp6QGff4og0uWYM2cOtm3bhh9++CHSh9JlKSsrw+9//3usWLECsbGxkT6cLo/FYsHIkSPx+OOPAwCKi4uxbds2vPrqq5gxY0aEj67rsWzZMrz99tt45513MGjQIJSUlOCuu+5CXl4ez3cXg+mwMJGZmQmNRuNSHXP8+HHk5uZG6Ki6HnPnzsVnn32G1atXo2fPnvLy3NxcGAwG1NbWOqzP898+Nm7ciMrKSgwfPhxarRZarRbffvstnn/+eWi1WuTk5PB8B5EePXpg4MCBDssGDBiA0tJSAJDPKb9fgsMf/vAH3H///Zg2bRqGDBmC3/72t5g/fz4WLlwIgOc7lPhzbnNzc10KikwmE2pqagI+/xRBYUKn02HEiBFYuXKlvMxisWDlypUYO3ZsBI+sayBJEubOnYuPPvoIq1atQmFhocPjI0aMQExMjMP53717N0pLS3n+28GECRPw66+/oqSkRP4ZOXIkpk+fLv+f5zt4nHnmmS4tH/bs2YPevXsDAAoLC5Gbm+twvuvr67Fu3Tqe73bQ3NwMtdrx8qjRaGCxWADwfIcSf87t2LFjUVtbi40bN8rrrFq1ChaLBaNHjw5shx2ydZOAePfddyW9Xi8tXrxY2rFjhzRr1iwpNTVVqqioiPShdXruuOMOKSUlRVqzZo107Ngx+ae5uVleZ/bs2VKvXr2kVatWSb/88os0duxYaezYsRE86q6FsjpMkni+g8n69eslrVYrLViwQNq7d6/09ttvS/Hx8dLSpUvldZ544gkpNTVV+u9//ytt3bpVuvzyy6XCwkKppaUlgkfeOZkxY4aUn58vffbZZ9LBgwel5cuXS5mZmdIf//hHeR2e7/bT0NAgbd68Wdq8ebMEQHrmmWekzZs3S4cPH5Ykyb9ze9FFF0nFxcXSunXrpB9++EE65ZRTpOuuuy7gY6EICjMvvPCC1KtXL0mn00mjRo2Sfv7550gfUpcAgNufN998U16npaVFuvPOO6W0tDQpPj5euuKKK6Rjx45F7qC7GM4iiOc7uHz66afS4MGDJb1eL/Xv31967bXXHB63WCzSX/7yFyknJ0fS6/XShAkTpN27d0foaDs39fX10u9//3upV69eUmxsrFRUVCQ9+OCDUltbm7wOz3f7Wb16tdvv6xkzZkiS5N+5PXHihHTddddJiYmJUnJysnTTTTdJDQ0NAR+LSpIULTAJIYQQQroJ9AQRQgghpFtCEUQIIYSQbglFECGEEEK6JRRBhBBCCOmWUAQRQgghpFtCEUQIIYSQbglFECGEEEK6JRRBhBBCCOmWUAQRQgghpFtCEUQIIYSQbglFECGEEEK6Jf8PO7BG1ztohpsAAAAASUVORK5CYII=",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "pd.DataFrame(FDS.running_mean_last_epoch.numpy()).iloc[:, 7].plot(\n",
-    "    title=\"Running mean bina values for 'Longitude' feature\"\n",
-    ");"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "24a898df",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkEAAAGzCAYAAAAotsMiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAACJcElEQVR4nO3dd3xT5f4H8M9J2iSd6R60pRMou6WsMgQEAQUFB4oXlSFw8YJ7XFAvyFVEREQFEdcFRH8KKG7Zy0FlFyjQQqF075Hupk2e3x/JOW1ouiDJSZrv+/XK69WenJzz5LRJvnme7/N9OMYYAyGEEEKInZGI3QBCCCGEEDFQEEQIIYQQu0RBECGEEELsEgVBhBBCCLFLFAQRQgghxC5REEQIIYQQu0RBECGEEELsEgVBhBBCCLFLFAQRQgghxC5REERExXEcFi1aZPbzHD58GBzH4fDhw2Y/lzW5fv06OI7DO++80+a+r732GjiOs0CrzI/jOLz22mtiN6NFW7duRXR0NBwdHeHh4SF2c6zO6NGjMXr0aIucy1LvDbt370ZMTAwUCgU4jkNZWZlZz0fah4IgG3L+/Hk88MADCA0NhUKhQFBQEO644w6sW7dO7Ka16ujRo3jttdfoRU8IgOTkZMyaNQuRkZH49NNP8cknn5j1fK+99hrCwsIMtoWFhWHy5MlmPa8p5eTk4LXXXkNiYqLYTTGwefPmdn1xKC4uxoMPPggnJyd8+OGH2Lp1K1xcXEzenosXL+K1117D9evXTX7szspB7AaQ9jl69CjGjBmDrl27Yt68eQgICEBmZib+/vtvvP/++3jyySfFbmKLjh49iuXLl2PWrFn0rdeKvfrqq1i8eLHYzej0Dh8+DK1Wi/fffx9RUVFiN8cq7d271+D3nJwcLF++HGFhYYiJiRGnUbfgxIkTqKiowOuvv45x48aZ7TwXL17E8uXLMXr06GaBLzGOgiAbsWLFCiiVSpw4caJZIFFQUCBOo0in4uDgAAcHekswN/71asovBNXV1XB2djbZ8cQmk8nEboJJmeNvbklVVVVm6bmyBjQcZiOuXr2K3r17G30R+fn5GfzO59ns2LEDvXr1gpOTE+Lj43H+/HkAwMcff4yoqCgoFAqMHj3aaNfpjh07EBcXBycnJ/j4+OCRRx5BdnZ2s/0OHjyIkSNHwsXFBR4eHpgyZQouXbok3P/aa6/hxRdfBACEh4eD4zhwHNfsnD/88AP69OkDuVyO3r17Y/fu3c3OlZ2djTlz5sDf31/Y73//+1+z/bKysjB16lS4uLjAz88Pzz77LOrq6prtZwyfF3P58mU88sgjUCqV8PX1xX/+8x8wxpCZmYkpU6bA3d0dAQEBWLNmTbNj1NXVYdmyZYiKioJcLkdISAheeumlZm3YtGkTbr/9dvj5+UEul6NXr1746KOPmh2PH7r4888/MXjwYCgUCkREROCLL75o13PirV27FqGhoXBycsKoUaOQlJRk9Lk3xf8vtfX3SU9Px7/+9S/06NEDTk5O8Pb2xrRp09rslq+vr4eXlxdmz57d7L7y8nIoFAq88MILAAC1Wo2lS5ciLi4OSqUSLi4uGDlyJA4dOtTmc581a5bRb8Yt5UF9+eWXwv+/l5cXpk+fjszMTIN9rly5gvvvvx8BAQFQKBQIDg7G9OnToVKpWmxHWFgYli1bBgDw9fVtlru0YcMG9O7dG3K5HF26dMHChQubDSOPHj0affr0walTp3DbbbfB2dkZL7/8cpvXoKMaGhrw+uuvIzIyEnK5HGFhYXj55Zeb/R935P/z3LlzGDVqFJycnBAcHIw33ngDmzZtavae0DQn6PDhwxg0aBAAYPbs2cJ7yObNm4Xzz5o1q9m5jOUVdeS94dixY5g4cSKUSiWcnZ0xatQo/PXXX+27eDe0Y+bMmQCAQYMGgeM4g/a25zzteX1t3rwZ06ZNAwCMGTNGuE58rlNLeXI3Xj9+iO/IkSP417/+BT8/PwQHBwv379q1S3jPd3Nzw6RJk3DhwoUOXxerwYhNGD9+PHNzc2Pnz59vc18ArF+/fiwkJIS99dZb7K233mJKpZJ17dqVrV+/nvXq1YutWbOGvfrqq0wmk7ExY8YYPH7Tpk0MABs0aBBbu3YtW7x4MXNycmJhYWGstLRU2G/fvn3MwcGBde/enb399tts+fLlzMfHh3l6erK0tDTGGGNnz55lDz/8MAPA1q5dy7Zu3cq2bt3KKisrhbb279+fBQYGstdff5299957LCIigjk7O7OioiLhXHl5eSw4OJiFhISw//73v+yjjz5i99xzj3BcXnV1NevevTtTKBTspZdeYu+99x6Li4tj/fr1YwDYoUOHWr12y5YtYwBYTEwMe/jhh9mGDRvYpEmTGAD27rvvsh49erAnnniCbdiwgQ0fPpwBYEeOHBEer9Fo2Pjx45mzszN75pln2Mcff8wWLVrEHBwc2JQpUwzONWjQIDZr1iy2du1atm7dOjZ+/HgGgK1fv95gv9DQUNajRw/m7+/PXn75ZbZ+/Xo2YMAAxnEcS0pKavX5pKWlMQCsb9++LCwsjK1atYotX76ceXl5MV9fX5aXl9fsuTfV3r/Pjh07WP/+/dnSpUvZJ598wl5++WXm6enJQkNDWVVVVattnDNnDvPw8GB1dXUG27ds2cIAsBMnTjDGGCssLGSBgYHsueeeYx999BF7++23WY8ePZijoyM7c+ZMs3YvW7ZM+H3mzJksNDS02bmNPec33niDcRzHHnroIbZhwwbh/7rp/39dXR0LDw9nXbp0YW+88Qb77LPP2PLly9mgQYPY9evXW3yu33//Pbv33nsZAPbRRx+xrVu3srNnzxq0Zdy4cWzdunVs0aJFTCqVskGDBjG1Wi0cY9SoUSwgIID5+vqyJ598kn388cfshx9+aPGcy5Yta/bcQ0ND2aRJk1p8DH/NALAHHniAffjhh+yxxx5jANjUqVObHas9/59ZWVnMy8uLeXt7s+XLl7N33nmHRUdHs/79+zMAwnsG/xxHjRrFGNO99v/73/8yAGz+/PnCe8jVq1eF88+cObNZ+5seg7GOvTccOHCAyWQyFh8fz9asWcPWrl3L+vXrx2QyGTt27JiwH/9e2Zq9e/ey+fPnMwDsv//9L9u6dSs7evRoh87TntfX1atX2VNPPcUAsJdfflm4Tvxr/MbXRNO/X9Prxz+nXr16sVGjRrF169axt956izHG2BdffME4jmMTJ05k69atY6tWrWJhYWHMw8PD4O9nSygIshF79+5lUqmUSaVSFh8fz1566SW2Z88egzdHHgAml8sN/ik//vhjBoAFBASw8vJyYfuSJUsM3oDUajXz8/Njffr0YTU1NcJ+v/zyCwPAli5dKmyLiYlhfn5+rLi4WNh29uxZJpFI2GOPPSZsW716dbM3uaZtlclkLDU11eAYANi6deuEbY8//jgLDAw0+OBljLHp06czpVLJqqurGWOMvffeewwA2759u7BPVVUVi4qK6lAQNH/+fGFbQ0MDCw4OZhzHCW8GjDFWWlrKnJycDN5Atm7dyiQSCfvjjz8Mjrtx40YGgP3111/CNr7NTU2YMIFFREQYbAsNDWUA2O+//y5sKygoYHK5nD3//POtPh8+CHJycmJZWVnC9mPHjjEA7Nlnn2323Jtq79/H2HNJSEhgANgXX3zRahv37NnDALCff/7ZYPtdd91lcC0aGhqaBUqlpaXM39+fzZkzp1m7byYIun79OpNKpWzFihUG+50/f545ODgI28+cOcMAsB07drT63Izhz1lYWChsKygoYDKZjI0fP55pNBph+/r16xkA9r///U/YNmrUKAaAbdy4scPn5rUVBCUmJjIAbO7cuQbbX3jhBQaAHTx40OBY7fn/fPLJJxnHcQYBa3FxMfPy8mo1CGKMsRMnTjAAbNOmTUafS3uCoPa+N2i1WtatWzc2YcIEptVqhX2rq6tZeHg4u+OOO5qdqy18YMEH9B09T3tfXzt27Gjxfa6jQdCIESNYQ0ODsL2iooJ5eHiwefPmGTw+Ly+PKZXKZtttBQ2H2Yg77rgDCQkJuOeee3D27Fm8/fbbmDBhAoKCgvDTTz8123/s2LEG3f9DhgwBANx///1wc3Nrtv3atWsAgJMnT6KgoAD/+te/oFAohP0mTZqE6Oho/PrrrwCA3NxcJCYmYtasWfDy8hL269evH+644w789ttv7X5u48aNQ2RkpMEx3N3dhTYxxvDdd9/h7rvvBmMMRUVFwm3ChAlQqVQ4ffo0AOC3335DYGAgHnjgAeF4zs7OmD9/frvbAwBz584VfpZKpRg4cCAYY3j88ceF7R4eHujRo4fQTkA3jNizZ09ER0cbtPP2228HAIOhGycnJ+FnlUqFoqIijBo1CteuXWs2pNKrVy+MHDlS+N3X17fZuVszdepUBAUFCb8PHjwYQ4YMadffqa2/z43Ppb6+HsXFxYiKioKHh4fwt2nJ7bffDh8fH2zbtk3YVlpain379uGhhx4StkmlUiFXRKvVoqSkBA0NDRg4cGCb52ivnTt3QqvV4sEHHzT4+wUEBKBbt27C30+pVAIA9uzZg+rq6ls+7/79+6FWq/HMM89AIml8W543bx7c3d2F1x1PLpcbHUI0Ff7/4rnnnjPY/vzzzwNAs/a05/9z9+7diI+PN0hs9vLywowZM0zdfKPa+96QmJiIK1eu4B//+AeKi4uF/4GqqiqMHTsWv//+O7Ra7S23pyPnuZXX182aN28epFKp8Pu+fftQVlaGhx9+2OC1IZVKMWTIkHYNS1sjyoK0IYMGDcLOnTuhVqtx9uxZfP/991i7di0eeOABJCYmolevXsK+Xbt2NXgs/6YdEhJidHtpaSkA3dgzAPTo0aPZ+aOjo/Hnn3+2uV/Pnj2xZ8+edifT3dhWAPD09BTaVFhYiLKyMnzyySctTifmEw/T09MRFRXVLM/DWDs70ialUgmFQgEfH59m24uLi4Xfr1y5gkuXLsHX17fVdgLAX3/9hWXLliEhIaHZB6lKpRL+NsbaAxheo7Z069at2bbu3btj+/btbT62PeeuqanBypUrsWnTJmRnZ4MxJtzXWo4MoEvIvv/++/F///d/qKurg1wux86dO1FfX28QBAHAli1bsGbNGiQnJ6O+vl7YHh4e3ubzaI8rV66AMWb0egGAo6OjcL7nnnsO7777Lr766iuMHDkS99xzj5BH1lEtvZ5kMhkiIiKE+3lBQUFmTR5OT0+HRCJpNnstICAAHh4ezdrTnv+R9PR0xMfHN9vPUjPk2vvecOXKFQAQ8niMUalU8PT0vKX2dOQ8t/L6ulk3vqb49vJf6G7k7u5ulnaYGwVBNkgmk2HQoEEYNGgQunfvjtmzZ2PHjh1CwiUAgwi+qZa2N31RWVpbbeK/DT3yyCMtvmH069fP7G1qz7XTarXo27cv3n33XaP78kHo1atXMXbsWERHR+Pdd99FSEgIZDIZfvvtN6xdu7bZN00x/27tOfeTTz6JTZs24ZlnnkF8fDyUSiU4jsP06dPb9a15+vTp+Pjjj7Fr1y5MnToV27dvR3R0NPr37y/s8+WXX2LWrFmYOnUqXnzxRfj5+UEqlWLlypW4evVqq8dvqZaLRqMx+F2r1YLjOOzatcvo83Z1dRV+XrNmDWbNmoUff/wRe/fuxVNPPYWVK1fi77//NkgkNYemPQPm1N7imWL+f7b2t22pXa3h/19Xr17d4nT8pv8HN6sj57nV11drbnwN8G78H+PPs3XrVgQEBDTb31Znltpmq4lg4MCBAHTDU6YQGhoKAEhJSWkW8aekpAj3N93vRsnJyfDx8RF6gW61CrGvry/c3Nyg0WjarLERGhqKpKQkMMYMzmusneYQGRmJs2fPYuzYsa0+759//hl1dXX46aefDL5Fm6tLmf8W19Tly5dNVkvk22+/xcyZMw1my9XW1ra7QOZtt92GwMBAbNu2DSNGjMDBgwfxyiuvNDtHREQEdu7caXBtmwb/LfH09DTalht7NCIjI8EYQ3h4OLp3797mcfv27Yu+ffvi1VdfxdGjRzF8+HBs3LgRb7zxRpuPbarp6ykiIkLYrlarkZaWZtbaMi21R6vV4sqVK+jZs6ewPT8/H2VlZUJ7O3rM1NTUZtuNbbtRa6+l1v62Ta9le98b+KFfd3d3s173jpynva+vjl4ntVrd7s8Ovr1+fn4W/380J8oJshGHDh0y+q2KH7vv6HBPSwYOHAg/Pz9s3LjRYOrorl27cOnSJUyaNAkAEBgYiJiYGGzZssXghZWUlIS9e/firrvuErbxwdDNVoyWSqW4//778d133zWb1g3ohst4d911F3JycvDtt98K26qrq81elZf34IMPIjs7G59++mmz+2pqalBVVQWg8Zvzjd3amzZtMku7fvjhB4MSB8ePH8exY8dw5513muT4Uqm02f/nunXrWvyWeSOJRIIHHngAP//8M7Zu3YqGhoZmQ2HGrtmxY8eQkJDQ5vEjIyOhUqlw7tw5YVtubi6+//57g/3uu+8+SKVSLF++vNnzYYwJQ5/l5eVoaGgwuL9v376QSCTtLsfQ1Lhx4yCTyfDBBx8YnPfzzz+HSqUSXneWwr9+33vvPYPtfA/nzbRnwoQJSEhIMKj6XFJSgq+++qrNx7b2HhIZGYm///4barVa2PbLL780K2nQ3veGuLg4REZG4p133kFlZWWz8zV9v7kVHTlPe19fbV2n33//3WDbJ5980u7X6IQJE+Du7o4333zTYCjaWHttCfUE2Ygnn3wS1dXVuPfeexEdHQ21Wo2jR49i27ZtCAsLM1mSpKOjI1atWoXZs2dj1KhRePjhh5Gfn4/3338fYWFhePbZZ4V9V69ejTvvvBPx8fF4/PHHUVNTg3Xr1kGpVBrUo4iLiwMAvPLKK5g+fTocHR1x9913d6j41ltvvYVDhw5hyJAhmDdvHnr16oWSkhKcPn0a+/fvR0lJCQBdMt/69evx2GOP4dSpUwgMDMTWrVstVkju0Ucfxfbt27FgwQIcOnQIw4cPh0ajQXJyMrZv3449e/Zg4MCBGD9+PGQyGe6++27885//RGVlJT799FP4+fmZrFevqaioKIwYMQJPPPEE6urq8N5778Hb2xsvvfSSSY4/efJkbN26FUqlEr169UJCQgL2798Pb2/vdh/joYcewrp167Bs2TL07dvXoAeCP8fOnTtx7733YtKkSUhLS8PGjRvRq1cvox8iTU2fPh3//ve/ce+99+Kpp55CdXU1PvroI3Tv3t0gsTQyMhJvvPEGlixZguvXr2Pq1Klwc3NDWloavv/+e8yfPx8vvPACDh48iEWLFmHatGno3r07GhoasHXrViFg7yhfX18sWbIEy5cvx8SJE3HPPfcgJSUFGzZswKBBg/DII490+JhtSU1NNdpjFRsbi0mTJmHmzJn45JNPUFZWhlGjRuH48ePYsmULpk6dijFjxnT4fC+99BK+/PJL3HHHHXjyySfh4uKCzz77DF27dkVJSUmrvRiRkZHw8PDAxo0b4ebmBhcXFwwZMgTh4eGYO3cuvv32W0ycOBEPPvggrl69ii+//NIgmR9o/3uDRCLBZ599hjvvvBO9e/fG7NmzERQUhOzsbBw6dAju7u74+eefO/z8b9SR87T39RUTEwOpVIpVq1ZBpVJBLpcLtcjmzp2LBQsW4P7778cdd9yBs2fPYs+ePc3yHFvi7u6Ojz76CI8++igGDBiA6dOnw9fXFxkZGfj1118xfPhwrF+//pavi8VZcCYauQW7du1ic+bMYdHR0czV1ZXJZDIWFRXFnnzySZafn2+wLwC2cOFCg238VOnVq1cbbD906JDRqb7btm1jsbGxTC6XMy8vLzZjxgyDKda8/fv3s+HDhzMnJyfm7u7O7r77bnbx4sVm+73++ussKCiISSQSg+mwxtrKmPFpr/n5+WzhwoUsJCSEOTo6soCAADZ27Fj2ySefGOyXnp7O7rnnHubs7Mx8fHzY008/zXbv3t2hKfJNpy8zppti7eLi0mz/UaNGsd69extsU6vVbNWqVax3795MLpczT09PFhcXx5YvX85UKpWw308//cT69evHFAqFUMPnf//7X7Ppwi1NZ75xCrAxTf/ua9asYSEhIUwul7ORI0cK9WlufO5NtffvU1paymbPns18fHyYq6srmzBhAktOTm5x+rIxWq2WhYSEMADsjTfeMHr/m2++yUJDQ5lcLmexsbHsl19+MTr9HUamA+/du5f16dOHyWQy1qNHD/bll18afc6MMfbdd9+xESNGMBcXF+bi4sKio6PZwoULWUpKCmOMsWvXrrE5c+awyMhIplAomJeXFxszZgzbv39/m8+zpf8xxnRT4qOjo5mjoyPz9/dnTzzxhEFtLsaM/891FD+t3djt8ccfZ4wxVl9fz5YvX87Cw8OZo6MjCwkJYUuWLGG1tbXNjtXe/88zZ86wkSNHMrlczoKDg9nKlSvZBx98wAAY1Kwy9tgff/yR9erVizk4ODSbLr9mzRoWFBTE5HI5Gz58ODt58qTRY3TkveHMmTPsvvvuY97e3kwul7PQ0FD24IMPsgMHDrTvIjdhbIp8R87TkdfXp59+yiIiIphUKjV4XhqNhv373/9mPj4+zNnZmU2YMIGlpqa2OEXeWFsZ031mTJgwgSmVSqZQKFhkZCSbNWsWO3nyZIevizXgGBMxI5YQQohde+aZZ/Dxxx+jsrLyphKZCbkVlBNECCHEImpqagx+Ly4uxtatWzFixAgKgIgoKCeIEEKIRcTHx2P06NHo2bMn8vPz8fnnn6O8vBz/+c9/xG4asVMUBBFCCLGIu+66C99++y0++eQTcByHAQMG4PPPP8dtt90mdtOInaKcIEIIIYTYJcoJIoQQQohdMlsQtGLFCgwbNgzOzs7w8PAwus9TTz2FuLg4yOXyFsuG79mzB0OHDoWbmxt8fX1x//334/r1662e+/Lly5gyZQp8fHzg7u6OESNG2OziboQQQggxD7PlBKnVakybNg3x8fH4/PPPW9xvzpw5OHbsmEElV15aWhqmTJmC5557Dl999RVUKhWeffZZ3Hfffa2unDt58mR069YNBw8ehJOTE9577z1MnjwZV69eNbrmiTFarRY5OTlwc3O75WUfCCGEEGIZjDFUVFSgS5cukEja6OsxdyGiTZs2MaVS2eo+y5YtY/3792+2fceOHczBwYFpNBph208//cQ4jmNqtdrosQoLCxkA9vvvvwvbysvLGQC2b9++drc7MzOzxWJidKMb3ehGN7rRzbpvmZmZbX7WW/XssLi4OEgkEmzatAmzZs1CZWUltm7dinHjxsHR0dHoY7y9vdGjRw988cUXGDBgAORyOT7++GP4+fkJyzcYU1dXZ7DmD9Pni2dmZsLd3d20T4wQQgghZlFeXo6QkBC4ubm1ua9VB0Hh4eHYu3cvHnzwQfzzn/+ERqNBfHy8sGioMRzHYf/+/cKaPxKJBH5+fti9ezc8PT1bfNzKlSuxfPnyZtvd3d0pCCKEEEJsTHtSWTqUGL148WJwHNfqLTk5+aYbfKO8vDzMmzcPM2fOxIkTJ3DkyBHIZDI88MADRldUB3Q9OAsXLoSfnx/++OMPHD9+HFOnTsXdd9/d6sKUS5YsgUqlEm43rkBMCCGEkM6lQz1Bzz//PGbNmtXqPhEREbfSHgMffvghlEol3n77bWHbl19+iZCQEBw7dgxDhw5t9piDBw/il19+QWlpqdCDs2HDBuzbtw9btmzB4sWLjZ5LLpdDLpebrO2EEEIIsW4dCoJ8fX3h6+trrrY0U11d3Syzm19fRqvVtvgYAM0eJ5FIWnwMIYQQQuyP2eoEZWRkIDExERkZGdBoNEhMTERiYiIqKyuFfVJTU5GYmIi8vDzU1NQI+6jVagDApEmTcOLECfz3v//FlStXcPr0acyePRuhoaGIjY0FABw/fhzR0dHIzs4GoFubxtPTEzNnzsTZs2dx+fJlvPjii0hLS8OkSZPM9XQJIYQQYmvaPWe8g2bOnGl0ytqhQ4eEfUaNGmV0n7S0NGGfr7/+msXGxjIXFxfm6+vL7rnnHnbp0iXh/kOHDjV7zIkTJ9j48eOZl5cXc3NzY0OHDmW//fZbh9qvUqkYAKZSqW72EhBCCCHEwjry+U1rh7WgvLwcSqUSKpWKZocRQgghNqIjn9+0dhghhBBC7BIFQYQQQgixSxQEEUIIIcQuURBECCGEELtEQRAhhBBC7BIFQYQQQgixS1a9gGpndDm/AttPZMLHTY4FoyLFbg4hhBBit6gnyMJyVbX47M80/JiYI3ZTCCGEELtGQZCFeTg5AgBU1WqRW0IIIYTYNwqCLMzDWRcEldXUi9wSQgghxL5REGRhHs4yAEC1WoO6Bo3IrSGEEELsFwVBFuYmd4CE0/2sot4gQgghRDQUBFmYRMJBKeQFURBECCGEiIWCIBHwQ2KUF0QIIYSIh4IgEfA9QWXUE0QIIYSIhoIgEQgzxGiaPCGEECIaCoJE4EE9QYQQQojoKAgSQWNOEPUEEUIIIWKhIEgElBNECCGEiI+CIBFQ1WhCCCFEfBQEiYAPgqhOECGEECIeCoJE4OFEOUGEEEKI2CgIEoHSmXKCCCGEELFRECQCD1o2gxBCCBEdBUEi4KfIV9Q1oF6jFbk1hBBCiH2iIEgE7goH4edymiFGCCGEiIKCIBE4SCVCIETT5AkhhBBxUBAkEqFqNOUFEUIIIaKgIEgkQq0gmiZPCCGEiIKCIJHQ0hmEEEKIuCgIEgkNhxFCCCHioiBIJHytIEqMJoQQQsRBQZBIGtcPo5wgQgghRAwUBImEzwkqpeEwQgghRBQUBIlEyAmi4TBCCCFEFBQEiaRx/TAaDiOEEELEQEGQSPicIOoJIoQQQsRBQZBIhCCIcoIIIYQQUVAQJBKlky4nqLy2HhotE7k1hBBCiP2hIEgk/OwwxoCKWuoNIoQQQiyNgiCRyBwkcJFJAdCQGCGEECIGswZBK1aswLBhw+Ds7AwPDw+j+zz11FOIi4uDXC5HTEyM0X22b9+OmJgYODs7IzQ0FKtXr27z3CUlJZgxYwbc3d3h4eGBxx9/HJWVlbfwbEyPpskTQggh4jFrEKRWqzFt2jQ88cQTre43Z84cPPTQQ0bv27VrF2bMmIEFCxYgKSkJGzZswNq1a7F+/fpWjzljxgxcuHAB+/btwy+//ILff/8d8+fPv+nnYg6NydE0TZ4QQgixNAdzHnz58uUAgM2bN7e4zwcffAAAKCwsxLlz55rdv3XrVkydOhULFiwAAERERGDJkiVYtWoVFi5cCI7jmj3m0qVL2L17N06cOIGBAwcCANatW4e77roL77zzDrp06XKrT80khKUzqCeIEEIIsTirzwmqq6uDQqEw2Obk5ISsrCykp6cbfUxCQgI8PDyEAAgAxo0bB4lEgmPHjrV4nvLycoObuXk40UryhBBCiFisPgiaMGECdu7ciQMHDkCr1eLy5ctYs2YNACA3N9foY/Ly8uDn52ewzcHBAV5eXsjLyzP6mJUrV0KpVAq3kJAQ0z4RI5RUK4gQQggRTYeDoMWLF4PjuFZvycnJJmvgvHnzsGjRIkyePBkymQxDhw7F9OnTdY2XmC6GW7JkCVQqlXDLzMw02bFbwi+dUVZDOUGEEEKIpXU4J+j555/HrFmzWt0nIiLiZtvTDMdxWLVqFd58803k5eXB19cXBw4caPU8AQEBKCgoMNjW0NCAkpISBAQEGH2MXC6HXC43WbvbQ8gJop4gQgghxOI6HAT5+vrC19fXHG1plVQqRVBQEADg66+/Rnx8fIvtiI+PR1lZGU6dOoW4uDgAwMGDB6HVajFkyBCLtbktQk4QJUYTQgghFmfW2WEZGRkoKSlBRkYGNBoNEhMTAQBRUVFwdXUFAKSmpqKyshJ5eXmoqakR9unVqxdkMhmKiorw7bffYvTo0aitrcWmTZuwY8cOHDlyRDjP8ePH8dhjj+HAgQMICgpCz549MXHiRMybNw8bN25EfX09Fi1ahOnTp1vNzDCgMSeolKbIE0IIIRZn1iBo6dKl2LJli/B7bGwsAODQoUMYPXo0AGDu3LkGAQ2/T1paGsLCwgAAW7ZswQsvvADGGOLj43H48GEMHjxYeEx1dTVSUlJQX9/Yo/LVV19h0aJFGDt2LCQSCe6//35hOr614HOCaDiMEEIIsTyOMUardxpRXl4OpVIJlUoFd3d3s5wjJa8CE977HV4uMpz+zx1mOQchhBBiTzry+W31U+Q7s6YVo7W0kjwhhBBiURQEiYhfSV7LgEp1g8itIYQQQuwLBUEiUjhKoXDU/QkoL4gQQgixLAqCREZLZxBCCCHioCBIZEJeEFWNJoQQQiyKgiCR8XlB1BNECCGEWBYFQSLzdKaq0YQQQogYKAgSWeP6YTQcRgghhFgSBUEiUzrTcBghhBAiBgqCREaLqBJCCCHioCBIZB7UE0QIIYSIgoIgkQmLqNIUeUIIIcSiKAgSGeUEEUIIIeKgIEhklBNECCGEiIOCIJE1XUmeMVpJnhBCCLEUCoJExgdB9RqGarVG5NYQQggh9oOCIJE5OUohk+r+DDQkRgghhFgOBUEi4ziuSXI0zRAjhBBCLIWCICsgTJOnGWKEEEKIxVAQZAWE5GgaDiOEEEIshoIgK6DUT5MvqaLhMEIIIcRSKAiyAv7ucgBAfnmtyC0hhBBC7IeD2A0gQBcPJwBAThkFQUQnv7wWa/ddRk29Bj0D3REd4IZege7wdZOD4zixm0cIIZ0CBUFWoIuHAgCQq6oRuSXEGpzJKMU/t55CQUUdAODHxBzhviAPJ3z0yAD0C/YQqXWEENJ50HCYFeii5HuCKAiyd9+dysJDn/yNgoo6dPNzxQvju2Nyv0BE+rpAwgHZZTV46dtzaNBoxW4qIYTYPOoJsgLCcJiqFowxGu6wQw0aLVbtTsanf6QBAMb19MPah2LgpnAU9imurMO4d48gOa8CW/9Ox+zh4WI1lxBCOgXqCbIC/u4KcBygbtCimGaI2aVXf0gSAqBFY6LwyaMDDQIgAPB2lePFCdEAgHf3XkahfriMEELIzaEgyArIHCTwddXNEMul5Gi7U1Fbj+9OZwEA3p8egxcm9IBEYrw38KFBIegbpERFXQNW7U62ZDMJIaTToSDISgTqh8SyKS/I7vx+uQj1GoYIHxdMiQlqdV+phMPyKb0BAN+eysKp9FJLNJEQQjolCoKsRBDNELNb+y/lAwDG9fJv1/4DunriwYHBAIBlPyVBo2VmaxshhHRmFARZiUD9DLFclfHhsMKKOrzy/Xlczq+wZLOImTVotDiUUgAAGBvt1+7HvTQxGu4KByRll+Pr4xnmah4hhHRqFARZiUClrieopeGwL/9Ox1fHMvDk/52hb/6dyKn0UpRV18PD2RFxoZ7tfpyPqxzPj+8BAFjx6yVs+iuN/i8IIaSDKAiyEkH6nKDcFoKg1IJKAEBKfgV26pNoie3jh8Ju7+EHB2nHXo4zhnTFyG4+qKnXYPnPF3H/R0eRnFdujmYSQkinREGQlQhsY+mMq4WVws/v7ruM2nqNRdpFzOvAJd1QWHvzgZpykEqwZfZgvD61D1zlDkjMLMPkD/7E27uT6f+DEELagYIgK8EvnVFQUYv6G6oBa7QM14qqAABuCgfkqmqx5eh1SzeRmNjVwkpcK6qCo5TDyG4+N3UMiYTDo0NDsf+5URjfyx8NWoYNh6/ivf1XTNxaQgjpfCgIshI+LnI4SjloWfPV5LNLa6Bu0ELuIMGrk3oCAD48lApVdb0YTSUmsv+ibihsaIR3s8KIHRWgVOCTxwYK/x+H9cnWhBBCWkZBkJWQSDgEKPlp8oZBED8UFu7jggfiQtDD3w3ltQ3YcDjV4u0kpsMPhd1xE0NhLZnULxAAcKWgEjVqGhIjhJDWUBBkRVpaSJVPio70c4VUwuHfd+pmBW06ep0WXbVRJVVqnEwvAQCM7Wm6ICjAXQFfNzk0WoYLOSqTHZcQQjojCoKsSJcWkqP5nqBIX1cAwJgefhgc7gV1gxbv7rts2UYSkziUXAAtA3oGugszA02B4zj0D1YCAM5mURBECCGtoSDIinRpoWp0YxDkAkD3QbfkTt1Cmt+dzqKlNmzQgWRdPtAdPdtfILG9+gV7AADOZZWZ/NiEENKZUBBkRQJbGA67WqibGcb3BAFAbFdPRPi4gDEgs6Taco0kt6yuQYMjKYUAbm5qfFv66nuCzlNPECGEtMqsQdCKFSswbNgwODs7w8PDw+g+Tz31FOLi4iCXyxETE2N0n+3btyMmJgbOzs4IDQ3F6tWrWz3v9evX8fjjjyM8PBxOTk6IjIzEsmXLoFarb/EZmVeQkeGwkio1Sqp07Y7Q9wTx3BQOAICqugYLtZCYwtGrxahSa+DvLkefLkqTH7+/vifoWlEVVDU0g5AQQlpi1iBIrVZj2rRpeOKJJ1rdb86cOXjooYeM3rdr1y7MmDEDCxYsQFJSEjZs2IC1a9di/fr1LR4vOTkZWq0WH3/8MS5cuIC1a9di48aNePnll2/p+ZhboH44LKfJcNg1/VBYkIcTnGUOBvu7yHW/V1IQZDMaNFqs3p0CAJjYOwASCWfyc3i5yBDsqQuok7KpN4gQQlri0PYuN2/58uUAgM2bN7e4zwcffAAAKCwsxLlz55rdv3XrVkydOhULFiwAAERERGDJkiVYtWoVFi5cCI5r/iEyceJETJw4Ufg9IiICKSkp+Oijj/DOO+8YbUddXR3q6uqE38vLLb/8AD8cVlZdjxq1Bk4yaWM+kJ9rs/1dKQiyOZv+uo6LueVQOjniybHdzHae/sEeyCqtwdmsMgyPurlCjIQQ0tlZfU5QXV0dFAqFwTYnJydkZWUhPT293cdRqVTw8vJq8f6VK1dCqVQKt5CQkJtu881yVzgIgQ3fG9SYD+TSbH9+XxoOsw2ZJdXCbL6X74qGj6vcbOfqp88LOpdJPUGEENISqw+CJkyYgJ07d+LAgQPQarW4fPky1qxZAwDIzc1t1zFSU1Oxbt06/POf/2xxnyVLlkClUgm3zMxMk7S/IziOE1aT55OjhRpBvs17goThsFoKgqwdYwxLf0xCTb0Gg8O98OBA8wbZNEOMEELa1uEgaPHixeA4rtVbcnKyyRo4b948LFq0CJMnT4ZMJsPQoUMxffp0XeMlbTc/OzsbEydOxLRp0zBv3rwW95PL5XB3dze4iaGLsJq8Ljn6xhpBTbkq+OEwqgxs7X49n4tDKYWQSSV4896+RodxTalvsBIcB+SoalFYUdf2AwghxA51OCfo+eefx6xZs1rdJyIi4mbb0wzHcVi1ahXefPNN5OXlwdfXFwcOHGjXeXJycjBmzBgMGzYMn3zyicnaZE58raDsshrU1muE6e+RfjQcZqtUNfVY/vNFAMAToyMRZSS/y9Rc5Q6I9HVFakElzmWVmbQqNSGEdBYdDoJ8fX3h6+trjra0SiqVIigoCADw9ddfIz4+vtV2ZGdnY8yYMYiLi8OmTZva1WtkDfjk6FxVDdKLq6FluqnwvkbyR1xkUgCUGG3t3t6djMKKOkT4uuBfYyItdt5+wUqkFlTibJaKgiBCCDHCrLPDMjIyUFJSgoyMDGg0GiQmJgIAoqKi4Oqq+zacmpqKyspK5OXloaamRtinV69ekMlkKCoqwrfffovRo0ejtrYWmzZtwo4dO3DkyBHhPMePH8djjz2GAwcOICgoCNnZ2Rg9ejRCQ0PxzjvvoLCwUNg3ICDAnE/5lgnDYapag6EwY8MnrvqVxykIsl5ZpdX4v+MZAIA37+0LuYPUYufuH+yBnaezcZ7yggghxCizBkFLly7Fli1bhN9jY2MBAIcOHcLo0aMBAHPnzjUIaPh90tLSEBYWBgDYsmULXnjhBTDGEB8fj8OHD2Pw4MHCY6qrq5GSkoL6el1huH379iE1NRWpqakIDg42aBNjzOTP05S6KBuHw662khQNAK5y3QcqDYdZr29PZYExYFikN4ZGeFv03Hzl6HNZKjDGzJ6HRAghtsasQdDmzZtbrREEAIcPH271fh8fHyQkJLS6z+jRow2Cm1mzZrWZt2StmiZGpwo1gprnAwFULNHaabUMO05mAQAeGmT5kgu9At3hIOFQXKVGdlkNgj2dLd4GQgixZraRKGNHAvQ9QTX1GpxKLwUARLXYE0RBkDX762oRsstq4K5wwITelh+GVThK0SPADYCuN4gQQoghCoKsjMJRCm8XGQAgq1RXK8hYtWiAZodZu20ndLWmpsYGQeFouVygpvh6QWcpL4gQQpqhIMgK8UNiAOAg4dDVy/gwBg2HWa/SKjX2XsgHALMXRmxNf6ocTQghLaIgyArxVaMBINTbGY5S438mvlhivYahroEKJlqTHxKzodZo0buLO/oEmX6l+Pbie4KSslXQaq17UgAhhFgaBUFWqGlPUEszwwDApcmq8lVUNdpqMMaEoTAxe4EAoLu/KxSOElTUNeBiruUXBSaEEGtGQZAV4qtGAy3nAwGAVMLBSZ9rQuuHWY+k7HIk51VA5iDB1JggUdviIJVgZDddUdHFO89RjyEhhDRBQZAVam9PENB0/TAKgqzFtpO64ogTewdA6ewocmuA/07pDQ9nRyRll2P17hSxm0MIIVaDgiArxC+dAQCRvsZrBPGEGWJqCoKsQW29Bj8m5gAQpzaQMYFKJ6x+oD8A4LM/03AopUDkFhFCiHWgIMgKBXs2BkERbfQEuchpOMya7ErKRUVtA0K8nBBv4QrRrbmjlz9mxocCAF7YfhYF5bUit4gQQsRHQZAV8ndX4InRkXhxQg8onVofTqGCidbl57O5AIAHBoRAIrGuZSqW3NUTPQPdUVylxrPbE2m2GCHE7lEQZKX+PTEaC8dEtbkfFUy0HrX1Ghy9WgQAmNjH+hbqVThKse7hWDg5SvFXajE+OnJV7CYRQoioKAiycVQw0Xr8fa0YtfVadFEq0N2/9WFMsUT5ueK1e3oBANbuu4xzVEmaEGLHKAiycTQcZj0OpxQCAEZH+1n1iu0PDgzBnX0C0KBleGZbImrUNG2eEGKfKAiycTQcZh0YYziYrJt1NaaHn8itaR3HcXjz3r7wc5PjWmEVVu66JHaTCCFEFBQE2TgaDrMO14qqkFFSDZlUgmGR1jMrrCWeLjKsnqabNv9FQjoO07R5QogdoiDIxjUOh9GQhpgO6XuBhkR4CYGptRvV3VeYNv/it+dQUqUWuUWEEGJZFATZOBoOsw5CPpCVD4XdaPGdPRHp64LCijq8vPM8GKNp84QQ+0FBkI0ThsOoWKJoquoacCytGAAwpoevyK3pGCeZFO9Pj4WDhMPuC3n45Vyu2E0ihBCLoSDIxgkVo6knSDR/pRahXsMQ6u2McJ/WlzmxRn2ClEJNqrX7LqNBoxW5RYQQYhkUBNk4NwWtHSa2Q/qhsDE9rHtqfGvmjgyHh7MjrhVV4aezOWI3hxBCLIKCIBtHw2HiYowJM6tG29hQWFNuCkfMvy0CAPDBgSvUG0QIsQsUBNk4FxlNkRdTSn4FclW1UDhKMNSKFky9GTPjw+DlIsP14mr8kEi9QYSQzo+CIBvHD4fVNWhRT9/eLe5Qsm4obFikDxSOUpFbc2tc5A4GvUH0/0QI6ewoCLJxTWvS0DR5yzuUwleJtt2hsKYeiw+Ft4sMGSXV+P50ttjNIYQQs6IgyMY5SiWQOej+jDQkZlmlVWqcSi8FYHv1gVriLHPAglGRAIAPDlJvECGkc6MgqBNwEwomUtVoSymvrcecLSeg0TJEB7ghxMtZ7CaZzCNDQ+HjKkdWaQ2+PZXV6r4/nMnGy9+fR0FFrYVaRwghpkNBUCfQuH5YvcgtsQ+qmno8+vlxnMkog9LJEe/o1+DqLJxkUiwY1ZgbVFBuPMD56WwOntmWiP87loEp6//C+SyVJZtJCCG3jIKgTsCF1g+zmLJqNR757BjOZpbB09kR/zdvCPoEKcVulsk9MjQUIV5OyFXVYvonfyNPZRgIHbtWjBe2nwWg64nMVdXigY1H8WMi5RERQmwHBUGdgButH2YRpVVqzPjsGM5nq+DlIsP/zRuK3l06XwAEAApHKb56fCiCPJxwragKD32SgJyyGgBAakEl5m89BbVGi4m9A/DHv8dgTA9f1DVo8fQ3iVi1OxkaLa1BRgixfhQEdQLC0hlUMNFsGGP455encCGnHD6uMnw9byh6BrqL3Syz6urtjG/mD0WIlxPSi6vx0CcJSMwsw6xNx6GqqUdsVw+8Nz0GHs4yfDZzEJ4YrUuo/ujwVbz07TmRW08IIW2jIKgTaBwOoyDIXA5cKsDxtBIoHCX4et5Q9AhwE7tJFhHi5Yxt8+MR6u2MzJIaTP3wL2SV1iDM2xmfPTZQqI0klXD498RovD89BhIO+O50Fs5llYnbeEIIaQMFQZ2AsH4YBUFmodUyvLM3BQAwe3g4uvnbRwDE6+LhhG3z44XFYb1cZNg8ezC8XeXN9p0SE4SpsUEAdIuxEkKINaMgqBOgpTPM66ezOUjOq4C7wgELbosUuzmiCFAqsO2fQ/HU2G74Zv5QhOkDImOeur0bpBIOh1IKcTqj1IKtJISQjqEgqBOg4bBbV61uQLW6+fVTN2jxrr5H45+jIqF0drR006yGn5sCz93RHd3b6AkL83HB/QOoN4gQYv0oCOoEaDjs1tQ1aDB2zREMe+sg/kotMrhv28lMZJRUw8dVjtnDw8RpoA168vZucJBw+ONKEU5cLxG7OYQQYhQFQZ0A9QTdmlPppchV1aKsuh6P/e84Nv+VBsYYatQarDtwBQDw5O1RcJY5tHEkwgvxcsa0gSEAgHf3Um8QIcQ60bt6J0BB0K05mloMQNejVlHbgNd+vojkvAp08XBCQUUdgj2d8PDgriK30vYsuj0K357KRMK1Yhy9WoRhkT5iN4kQQgxQT1AnQGuH3Zo/9UNg/5ncC6/c1RMSDvjmRKaQC/TsuO7CIrWk/YI8nDB9kC54fG/fFTBGBRQJIdaF3tk7AeoJunnltfVCPZsRUT6Yd1sE/jdrkJBn1d3fVZjyTTruX2MiIXOQ4Pj1Ehy5XCh2cwghxAAFQZ2AUDGagqAO+/tqMbQMiPBxQRcPJwDA6B5++HHhcMwZHo4P/zEAUgkncittV6DSCY8ODQUAvPpDEiXvE0KsilmDoBUrVmDYsGFwdnaGh4eH0X2eeuopxMXFQS6XIyYmxug+27dvR0xMDJydnREaGorVq1e3uw11dXWIiYkBx3FITEzs+JOwAW5y3bRt+oDpuKNXdflAw6K8DbZH+Lpi6d297K4wojk8e0d3BHk4Iau0Bm/vTha7OYQQIjBrEKRWqzFt2jQ88cQTre43Z84cPPTQQ0bv27VrF2bMmIEFCxYgKSkJGzZswNq1a7F+/fp2teGll15Cly5dOtx2W8L3BFWrNbRwZQfxU+KHU9Ku2bjKHbDq/n4AgC0J6fj7WrHILSKEEB2zBkHLly/Hs88+i759+7a4zwcffICFCxciIiLC6P1bt27F1KlTsWDBAkRERGDSpElYsmQJVq1a1Wai5a5du7B371688847t/Q8rB2fEwQAVUYK/hHj8strcaWgEhwHxEd6t/0ActNGdPPBw4N1U+Zf+vac0cKUhBBiaVafE1RXVweFQmGwzcnJCVlZWUhPT2/xcfn5+Zg3bx62bt0KZ2fndp2nvLzc4GYr5A4SOEp1eSs0JNZ+R6/qeoH6dFHCw1kmcms6v5fv6okuSgUySqqxek+K2M0hhBDrD4ImTJiAnTt34sCBA9Bqtbh8+TLWrFkDAMjNzTX6GMYYZs2ahQULFmDgwIHtOs/KlSuhVCqFW0hIiMmeg7lxHNc4Q6yWgqD2+vOK8XwgYh5uCkes1A+LbT56HcfTqJI0IURcHQ6CFi9eDI7jWr0lJ5su+XHevHlYtGgRJk+eDJlMhqFDh2L69Om6xkuMN3/dunWoqKjAkiVL2n2eJUuWQKVSCbfMzEyTtN9SaBHVjmGMCT1BI6IoH8hSRnX3xYMDg8EY8O/vzlEOGyFEVB2uGP38889j1qxZre7TUn7PzeA4DqtWrcKbb76JvLw8+Pr64sCBA62e5+DBg0hISIBcLjfYPnDgQMyYMQNbtmxp9hi5XN5sf1vSuH4YFUxsj7SiKuSqaiGTSjAw1Evs5tiVVyb1wq/ncpFWVIXL+RXoGegudpMIIXaqw0GQr68vfH19zdGWVkmlUgQF6YrWff3114iPj2+xHR988AHeeOMN4fecnBxMmDAB27Ztw5AhQyzSXkujgokdw88KGxDqASeZVOTW2BelkyNiunrgr9RinEovpSCIECIas64dlpGRgZKSEmRkZECj0Qh1eqKiouDq6goASE1NRWVlJfLy8lBTUyPs06tXL8hkMhQVFeHbb7/F6NGjUVtbi02bNmHHjh04cuSIcJ7jx4/jsccew4EDBxAUFISuXQ3XeeLPFRkZieDgYHM+ZdFQENQxf+nXC6OhMHHEdfXEX6nFOJ1eikf0xRQJIcTSzBoELV261GDoKTY2FgBw6NAhjB49GgAwd+5cg4CG3yctLQ1hYWEAgC1btuCFF14AYwzx8fE4fPgwBg8eLDymuroaKSkpqK+vN+fTsWqN64dRENQWjZYh4RqfFE1BkBgGhHoCAE5llIrcEkKIPTNrELR582Zs3ry51X0OHz7c6v0+Pj5ISEhodZ/Ro0e3WjMoLCys0y/eSEtntN+FHBVUNfVwkzugX5BS7ObYpdiuuiAovbgahRV18HWz3Xw8QuwZYwwfHkpFkKcT7o21vZEWq58iT9qHhsPajx8KGxLhDQcpvQTEoHRyRHd/3TD1aeoNIsRmJWWX4529l/HstrP47I9rYjenw+gToJNwpeGwdimpUuPzP9MAAKO601CYmOL0Q2IUBBFiuy7mqoSf3/j1Er5IuC5eY24CBUGdhCv1BLWJMYZXvj+Poso6dPNzxbSBtlMQszMaoB8SO51OQRAhtupSbgUAwN9dN6S99McL+Pp4hphN6hAKgjoJqhjdth8Ss7ErKQ8OEg5rH4qBwpGmxouJ7wk6m6WCukErcmsIITcjJU8XBD0/vgfmjggHALz8/Xl8eypLzGa1GwVBnYQwHEYLUxqVU1aDpT9eAAA8PbYb+lBCtOjCfVzg6ewIdYMWF3JUbT+AEGJVGGNIztOts9kr0B2vTOqJmfGhYAx46duzOKqvx2bNKAjqJBqHw6hi9I20WoYXdpxFRW0DYkI88MToSLGbRKCrBs8PiZ2iITFCbE5BRR1Kq+sh4YAoP1dwHIdld/fGpH6B0DLg53M5YjexTRQEdRKNw2H2WyupJVsSruPo1WIoHCV498H+NCPMigyg5GhCbFayfigs3MdFSC+QSDhM7B1gcL81o0+DTqJxdhj1BDWVU1aDt3bpFvR95a6eiPB1FblFpCk+L+hUemmnr+VFSGeTnKsbCou+Yemb6AA3ALp8Ia2VL5JMQVAn4aqgKfLGHLlciLoGLfoFK2l5BivUP9gDUgmH/PI65KhqxW4OIaQD+KToaH83g+3hPi6QSSWoVmuQVVojRtPajYKgTkKoGK1uoG/UTZy4XgIAuK2bLziOE7k15EZOMil6d9F9i6S8IEJsyyU+CLqhJ8hBKkGUn67XnU+ctlYUBHUS/HAYY0C1mobEePwHa1yYp8gtIS2hekGE2J56jRapBfogKMCt2f3Rgbpt1p4XREFQJ+HkKIVE39FBQ2I6BRW1SC+uBsc1ftAS6zMglGaIEWJr0oqqUK9hcJU7INjTqdn9TfOCrBkFQZ0Ex3HCDLEKCoIAAKeu6z5Ue/i7QenkKHJrSEv45OiLueWopjpXhNiES/qk6B4BbkZTDXoE6IbILtFwGLEUWj/M0Al9EDSQhsKsWhelAgHuCmi0DGczqWgiIbaAH+YyNhQGAD31268XVaG23npTNCgI6kRo/TBDp9J1SdEDQ71EbglpDcdxtJgqITYmpY0gyNdNDk9nR2gZkFpQacmmdQgFQZ0IrR/WqFrdgAs5um5Y/gOWWK/Yrh4AgDMZZaK2gxDSPi3VCOJxHIdofkgs13qHxCgI6kRo/bBGiZllaNAyBLgrjCbtEevCr+VmzW+WhNij0ip1s4KHqup6oa5XjxZ6gpreZ83J0RQEdSK0flgjPik6LsyT6gPZgF76WkHZZTUoq1aL3BpCCACczSxD7Ov7sHjnOYPtKfm6oCbIwwnuipYnnfS0gWnyFAR1IjQc1uiEfrr1IBoKswnuCkeEeOl67C7mUG8QIdbgjD5Hb/vJLIN8Pb4AYkv5QDx+hhgFQcQiXPVVo+19dphGy3AmnZ8ZRknRtqKXPrfgIg2JEWIVSqoae2Xf/PWSsBrBpVy+UnTrQVB3f1dwHFBUWYeiyjrzNfQWUBDUifDrh9n77LCUvApU1DXARSZt85sKsR69u+jygqgniBDrUNJkaPpkein2XMgHAKTk8TWCjCdF85xlDgj1ctY/xjp7gygI6kRcaIo8gMap8bFdPeEgpX9xW0E9QYRYl9KqegBAoFIBAFi1Oxl1DRohoOnZji+Z0VY+JEafEJ0IFUvUOZlORRJtEZ8cfaWg0qqLqxFiL4qrdENYT43tBh9XGdKKqrB6dwqq1BrIpBKE+7i0eQx+hliylX65oSCoE6FiiTon+UrRVCTRpgQqFfBwdoRGy3Al33qLqxFiL/ieoBBPZzwzrjsA4LM/0wAAUX6u7eppF9YQy6eeIGJmNBwG5JTVILusBlIJhxh9AT5iGziOQ+8u/JAYLZ9BiNj4nCBPF0dMHxSCSN/Gnp+2kqIb99O9plPyKqC5od6QNaAgqBPx0C8SWlxpv3VW+KGwnoFuQs8YsR18XtAFSo4mRFSMMZTqZ4d5u8jhIJVgyZ09hft7tpEUzevq5QyFowR1DVqkF1eZpa23goKgTiRMPz6bVVqNugb7zKk4dZ3WC7NlfF4QzRAjRFzltQ1o0PfceDjrvmCP7emH0T18wXFAfKR3u44jlXDo4W+9RRMpCOpE/NzkcJFJoWVAenG12M0RxbE0fRBESdE2qVdg4/IZN5bqJ4RYDt8L5CKTQuGoq0HHcRw+eXQg/vr37cJSN+0hJEdTEETMieM4RPq5AgCuFdpfYmlmSTWS8yog4YChEe37lkKsS6SvC2QOElSpNcgosc9AnhBrwOcDebnKDLbLHCTo4tGx9Rj5afJ8fSFrQkFQJxOhHxK7Wmh9Y6/mtudCHgBgUJgXfFzlIreG3AwHqUSYTUJ5QYSIp0SfW+rlLGtjz7ZFU08QsZQIX74nyP6CoL36aqYT+wSI3BJyKxqLJtIMMULE0jgz7NaDIH44LKOk2urq2FEQ1MlE6KcwXiuyr+Gwwoo6nNBXih7fm4IgW9abkqMJER2fE+RlgiDI21UOLxcZGIPVDXNTENTJRPg09gTxi93Zg30X88EY0C9YiaAOjlcT6yLMELPSCrOE2AN+8VRTDIcBQLCn7n05q7TGJMczFQqCOplwHxdwHKCqqTdYAbiz4/OBJlAvkM3rEeAOjgPyy6135WlCOjv+88MUw2EAhC+nWaXUE0TMyEkmRRel7p/NXpKjVTX1OHq1CAAFQZ2Bq9wBYd66YV0aEiNEHKXVfKFE0/YEZVNPEDE3IS/ITqbJH0ouQL2GIcrPFVH6EgHEttGQGCHiKjZxT1CwpzMAGg4jFhDJzxArso+eIH4obCL1AnUatHwGIeIyZWI00CQnqIyGw4iZ2VNPUI1ag8MphQBoKKwzaVw+g6bJEyIGISfIRInRQZQYTSwl0o5qBf1+pRA19RoEeTihT1D7FvQj1q+3vifoWlEVqtXWVVeEkM6uXqNFea3udWeqnCA+Mbqsuh6VVlQriIKgTojvCcooqUa9Ritya8xrT1LjrDCO40RuDTEVP3cFfFzlYAy4lGt9VWYJ6cz4pGgJB7g7OZrkmG4KR2EhVmtKjqYgqBMKcFfAWSZFg5YZXUj157M52KvPo7Fl9Rot9l/SVYme0Ntf5NYQU+sXrFugMTGzTNyGEGJnSqvqAQAezjJIJab7cmmN0+TNFgStWLECw4YNg7OzMzw8PIzu89RTTyEuLg5yuRwxMTFG99m+fTtiYmLg7OyM0NBQrF69ul3n//XXXzFkyBA4OTnB09MTU6dOvbknYoM4jkO4j/G8oOtFVXjy6zNY+H+nbX6Y4e9rxSivbYC3iwwDw7zEbg4xsQFdPQAApzNKxW0IIXamxMRJ0TxhmnyZHfQEqdVqTJs2DU888USr+82ZMwcPPfSQ0ft27dqFGTNmYMGCBUhKSsKGDRuwdu1arF+/vtVjfvfdd3j00Ucxe/ZsnD17Fn/99Rf+8Y9/3PRzsUURLcwQ+/V8LgCgXsOsrnx5R+3WD4Xd0cvfpN9WiHUY0NUTAHAmnYIgQizJ1NWiedY4Td7BXAdevnw5AGDz5s0t7vPBBx8AAAoLC3Hu3Llm92/duhVTp07FggULAAARERFYsmQJVq1ahYULFxrNAWloaMDTTz+N1atX4/HHHxe29+rV61aejs2JbGGG2G/6IAgArhdVIzrANpOJtVqGvRf1Q2G0YGqn1D/EAxIOyFHVIldVg0AlLYdCiCU0Lp5qmnwgnl0Nh5lCXV0dFAqFwTYnJydkZWUhPT3d6GNOnz6N7OxsSCQSxMbGIjAwEHfeeSeSkpLaPFd5ebnBzZYZW00+vbjKoO5KerHtzh47k1mKwoo6uMkdMCzSW+zmEDNwkTsIQfrp9DJxG0OIHWmsESQ36XGtcf0wqw6CJkyYgJ07d+LAgQPQarW4fPky1qxZAwDIzc01+phr164BAF577TW8+uqr+OWXX+Dp6YnRo0ejpKSkxXOtXLkSSqVSuIWEhJj+CVlQBJ8T1GQ47NfzhtfsupGkaVux54KuF2hMtB/kDlKRW0PMZUCoBwDKCyLEkhpzgkzbE8QPh9ns7LDFixeD47hWb8nJySZr3Lx587Bo0SJMnjwZMpkMQ4cOxfTp03UNlxhvularmxL+yiuv4P7770dcXBw2bdoEjuOwY8eOFs+1ZMkSqFQq4ZaZmWmy5yEGfpp8SZVaiOp3ndfl0AwM1eVa2GpPEGOssUo0DYV1anxeEAVBhFiOqQsl8viCicVVaquZmNOhnKDnn38es2bNanWfiIiIW2mPAY7jsGrVKrz55pvIy8uDr68vDhw40Op5AgMDARjmAMnlckRERCAjI6PFc8nlcsjlpu36E5OzzAGBSgVyVbW4VlQJ31oFzmerIOGA+bdF4OTWU0anz9uC5LwKpBdXQ+YgwajuvmI3h5hRnD5gv5Bdjtp6DRSO1OtHiLkJi6e6mjYIUjo5wk3hgIraBmSX1qCbv5tJj38zOhQE+fr6wtfX8h86UqkUQUFBAICvv/4a8fHxLbaDn3KfkpKCESNGAADq6+tx/fp1hIaGWqzN1iDC1wW5qlpcLazCieu6b9JDI7yFD5YcVY1NfrDwvUC3dfOFi9xsuf3ECnT1coa3iwzFVWpcyFEhLpRKIRBibsWV5ukJAnRDYpdyy5FlJUGQ2XKCMjIykJiYiIyMDGg0GiQmJiIxMRGVlY2zlVJTU5GYmIi8vDzU1NQI+6jVuj9AUVERNm7ciOTkZCQmJuLpp5/Gjh078N577wnHOH78OKKjo5GdnQ0AcHd3x4IFC7Bs2TLs3bsXKSkpwjT9adOmmevpWqWmy2fws8Lu6hsILxcZ3OQOYMy6svTba7dQJZoKJHZ2HMchlh8So+RoQiyC7wkydZ0goOlCqtaRF2S2r9FLly7Fli1bhN9jY2MBAIcOHcLo0aMBAHPnzsWRI0ea7ZOWloawsDAAwJYtW/DCCy+AMYb4+HgcPnwYgwcPFh5TXV2NlJQU1NfXC9tWr14NBwcHPProo6ipqcGQIUNw8OBBeHp6muvpWiU+Ofr3y4W4mFsOCde4vESojzOSsstxvagaUX7iR+PtlV5cheS8CkglHMb1pCDIHgwI9cD+S/mUF0SIBTDGzFYsEbC+afJmC4I2b97cao0gADh8+HCr9/v4+CAhIaHVfUaPHg3GmME2R0dHvPPOO3jnnXfa09ROi58mfzFXNy1+cLgXfN10eU+h3i66IMjGkqP5obAh4V7wNMMLlFifuCbJ0YwxWiOOEDOqVmtQ16CbYGTWniArmSFm1VPkya3hZ4jxJvUNFH4O89ZNVbS15Gh+ajzNCrMf/YI94CDhkF9ehxxVrdjNIaRT43uB5A4SOJkhX9TaqkZTENSJdVE6QeGo+xNznGFl5VBvXYBkSz1BBeW1OKVfQmF8LwqC7IWTTIqegbqiiadoCQ1CzEqYGeYiM0uvq7B+GAVBxNwkEg7hProhsUFhXvBza6y+HWaDQRC/TEZMiAcClIo29iadibCYKgVBhJhVMV8jyEzpBnwQVFRZh9p6jVnO0REUBHVyMSEeAID7YoMMtvPDYdmlNVDrx3+tHRVItF8D9GUdzlByNCFmVWrGpGhAVyvIVV/axBqGxCgI6uQW3xmNrY8PxkODDJcB8XWTw8lRCq2NTJO/VliJhKvFAHQz3Ih94StHX8gpt4pvj4R0VuaqFs3jOK5xSMwKpslTENTJKZ0cMbKbb7OxXY7jEGojydGMMbz6QxIatAyje/gi3Mel7QeRTiXY0wm+bnI0aBnOZ6vEbg4hnZY5awTxrGmaPAVBdsxW8oK+P5ONo1eLoXCU4PUpfcRuDhEBx3FCXhAlRxNiPuasEcSzpmnyFATZsVAf6+8JKq1S441fLwEAnhrbDSFeziK3iIiFX+6FkqMJMZ8SMydGA9Y1TZ6CIDtmCz1Bb+1KRkmVGj383TBvpOkW5yW2h88LOpNZ1qxAKiHENEqrdKsveJtzOEyYJi/+F3AKguyYtecEHU8rwbaTmQCAN+/rA0cp/bvas95dlJBKOBRW1CGvnIomEmIOxVV1AMyXGA3QcBixEnxPUGZJNRo01jVNXt2gxcvfnwcAPDy4K60eTuAkk6K7ftXps5mUHE2IOZRW63qCzJsTpPsCXlAhfq0gCoLsWIC7AjIHCRq0DDll1vXNevPRNKQWVMLHVYbFE6PFbg6xEv2DlQCAs1ll4jaEkE5Io2Uos8DsME9nR2FJjhyRp8lTEGTHJBIOofpEY2vLC9qnrw799NhuUDo7itwaYi3664t/nqMgiBCTU9XUQ6tPt/Mw4/uuNdUKoiDIzvFriKVbURCk1TJcyCkHAAyJ8Ba5NcSa9NP3BJ3LUkGrpeRoQkyJnxnmrnAwew6mteQFURBk5/jlM65bUXL0taIqVKs1UDhKEOnrKnZziBXp7u8GuYMEFbUNVtd7SYitExZPdZWb/VyN0+TF/eyhIMjOhfpYX0/QhRxd0muvQHdIJaZfxZjYLkepBL276FaUp7wgQkyruJJfMsP8KQhB1BNErIE19gSdz9IFQX2DlCK3hFgjPi+IZogRYlqWWDKDJ+QEiRwEOYh6diI6fpp8RnE1NFpmFT0vSfqeoN4UBBEj+gd7AKDkaEJMzRJLZvDiI7yxbf5QdPUWdxUACoLsXKBSAUcpB7VGi7zyWmFhO7FotQwXsnVJ0dQTRIzhk6Mv5JSjXqOlIpqEmEipBZbM4Hm7yi2Se9QWevewcw5SCUL0CWrpReLnBWWUVKOirgEyBwmi/CgpmjQX5u0Cd4UD6hq0SMmrELs5hHQaQk+QGatFWxsKgoiwfIY15AWdz9YNhfUMdKdv+MQoiYRDP2FIjPKCCDGVEgvmBFkL+pQhQq0ga5hyzOcD9dHPACLEmMZ6QWXiNoSQTqTUgjlB1oKCIIJIX10QdCVf/KGFpGyaGUbaxvcEJWaWidoOQjqTYgvmBFkLCoIIogN1vS6XcsUNghhjSNInRfehIIi0IkY/Tf5KQSVq1OIuwEhIZ0E5QcQuRQfoVubOK68VukPFkFVaA1VNPRylnLBaOCHGBCgV8HOTQ6NlQnFNQsjNU1XXo1r/hSJAqRC5NZZDQRCBm8IRIV66qfGXcstFawc/FNYjwA0yB/rXJK3jh8TOUnI0IbcsU798hY+rHAr9Cu/2gD5pCACgZ4B+SEzEKcfnKR+IdEB/fXL0WTvKC8osqcbeC3liN4N0QvzyFXwlZ3tBQRABoJuSDojcE6RfOb53FwqCSNv45TPsZYZYbb0G0z/5G/O3nsJfqUViN4d0MvxCpiFe4lZwtjQKgggA8YMgXVI09QSR9uOnyV8vroaqul7k1pjfFwnXkV2m+7Z+MLlA5NaQzoZ6gohd66UPgq7kV6Jeo7X4+XNVtSipUsNBwqFHACVFk7Z5OMuEQp/nssvEbUwH/X65EA98dBSn0kvbtX9ZtRrrD6YKv/95hXqCiGllluh6gigIInYp2NMJrnIHqDVaXCu0fNFEPh+om7+bXSXlkVvDL6ba3mDCWnz2ZxpOppfin1tPIk9V2+b+Hx5KRXltAyJ8XMBxQEp+BQrK234cIe3F9wTxyyjZCwqCCADdUgT8VHkxhsQuZFOlaNJxQyK8AABHrxaL3JL202gZTuuDtqJKNZ746hTqGlqudZRZUo0tR9MBAEvv7oU++py5PykviJgIY0zICaKeIGK3xMwLEmaGBVM+EGm/4ZE+AIAzGaWoVjeI3Jr2Sc4rR2VdA5xlUrgrHHAmowz//flii/uv2ZsCtUaL4VHeGNXdFyO66Z4zBUHEVEqr61GlrxHUxYOCIGKnogP1PUEiTJOnmWHkZoR6OyPIwwn1GoYT121jSIwfuosL9cT7D8eC44CvjmVg+4nMZvsmZavwQ2IOAGDJnT3BcRxGROmDoCtFYIxZruHEZnx3Kgtzt5xAeW37JgzwvUD+7vZVIwigIIg0IVZPUH55LQor6iDhGhO0CWkPjuMwLNIbAHDURnpG+GBtUJgXxvTww3PjugMAXv0hCafSS1CtbhBub/52CQAwNaaLsJRMXKgn5A4SFFTU4UpBpThPglitanUDXvvpAvZfKsCu87ntekzjzDD7ygcCAAexG0CsR3SAGzgOKKyoQ1FlHXxc5RY5b7K+5ynC1xVOMvv6FkJu3fAoH+w4lYW/rlp/EMQYw4m0EgDAwDBPAMDCMVE4l63Cvov5uP+jhGaPkUkleH58D+F3haMUg8O98MeVIvxxpYiWmCEGfj6bg4o63dDwhZz2faG115lhAPUEkSacZQ4I89atKG/J3qAM/Qsw3MfFYucknQffE3Qhp1zUte/aI7usBnnltZBKOGERWImEw5oH+2NgqKfRxyy6PapZAbuRfF7QlUKztpfYnq+OZQg/87XX2mKvM8MA6gkiN+gZ6Ia0oipcyi3HyG6+Fjkn/y2kq51VKiWm4eeuQHd/V1zOr0TCtWLc1TdQ7Ca1iM8H6tPFHc6yxrdfd4UjdiyIR229YY0ujoPRHI0RUb4AknEsrQTqBi2ttUcA6Kqnn2uylt7F3HJotAxSCdfq4zLtdGYYQD1B5AbCGmK5lkuOziimIIjcmmH6WWLWvpzEiev8UJhXs/s4joOTTGpwaylJNTrADd4uMlSrNTiTYRsJ4cT8vvpb1ws0uV8gnGVS1NZrca2w7bwxe84JoiCIGBAjOTqDeoLILRqunzFl7fWCTgpJ0caHvtpLIuGE50xT5QkAqGrq8dNZ3UzCx+LDhEkmSTmtD4k1rREU4kU9QcTO9dQXK0wtqIS6wfzLZzDGhOEwe1u4j5jOkAgvSDggragKOfr1tayNqroeKfm6Hta40OY9QR3F1wv6g5bQIAB+OJONmnoNuvu7YlCYpzCbMCm79S+0RZVq1NZrwXFAoJKCIJNZsWIFhg0bBmdnZ3h4eBjd56mnnkJcXBzkcjliYmKM7rN9+3bExMTA2dkZoaGhWL16dZvnvnz5MqZMmQIfHx+4u7tjxIgROHTo0C08G/vRRamAu8IBDVqGVAtMvy2rrhdmMtjjeDQxDXeFI/rpl9Cw1iGx0xmlYAwI83aGr9utz7zkk6PPZZV1ygVkGWNY+H+nMeOzvy3yhcyWMcbw1TFdVfEZQ0LBcRx667/QtpUczfcCBbgr7DK3zGzPWK1WY9q0aXjiiSda3W/OnDl46KGHjN63a9cuzJgxAwsWLEBSUhI2bNiAtWvXYv369a0ec/LkyWhoaMDBgwdx6tQp9O/fH5MnT0ZeXt5NPx97wXGcRYfE+KEweyzSRUxreJS+XpCVDomdTG85H+hmBCqdEOnrAi0DEq5Z53O+FbmqWvx6Lhd/pRbj7074/EzpZHopLudXwslRinsHBAGA0BN0MaccWm3LRTUz7XhmGGDGIGj58uV49tln0bdv3xb3+eCDD7Bw4UJEREQYvX/r1q2YOnUqFixYgIiICEyaNAlLlizBqlWrWqyUWlRUhCtXrmDx4sXo168funXrhrfeegvV1dVISkoyyXPr7MQIgigfiNyq4U2So62xkvIJE+UDNSVUj07tfFPlz2WVCT/vvUhfYHmpBZXYdzEfGcXVwv/5V3/reoHu6d8F7gpHAECUnytkDhJU1DUIs7+Msdc1w3hWPUW+rq4Ozs6GH45OTk7IyspCeno6wsLCmj3G29sbPXr0wBdffIEBAwZALpfj448/hp+fH+Li4lo9V11dnfB7ebnl18+yFnxC3aU8ywVBlA9EbtWAJpWUrxZWIsrPeooI1jVocDazDIDpeoIAYEQ3X2xJSMfBSwVouFsLB2nnGc5IzGwcxtl/sQCvT2HguNanend2qpp63LfhL5TX6lIIXOUO6BnohrP6azVjaFdhX0epBD0D3HA2S4Wk7HKEehuvwybMDLPT92CrfsVMmDABO3fuxIEDB6DVanH58mWsWbMGAJCba7wcOMdx2L9/P86cOQM3NzcoFAq8++672L17Nzw9W/4GtnLlSiiVSuEWEhJiludkCxp7girM/o2aagQRU1E4SoUqzH9aWbJwUnY56hq08HKRIcKERUFHdvOBl4sMOapa7L7QuXpLmvYE5ZXXCoss27NvjmegvFa3+K5MKkFlXQNOXC+FWqNFv2ClkBfH680nR7cyQ8yeq0UDHQyCFi9eDI7jWr0lJyebrHHz5s3DokWLMHnyZMhkMgwdOhTTp0/XNVxivOmMMSxcuBB+fn74448/cPz4cUydOhV33313i4ETACxZsgQqlUq4ZWY2X8zQXnTzd4WEA0qq1CioqGv7AbeAhsOIKQn1gqwsL+iUPh8oLtTTpL0ZCkcpHosPBQB8+vs1qxwGvBlaLcN5fdG/CF9d0LjvYr6YTRJdvUaLzUevAwBeu7s3Lvx3AnY/MxJrH+qPZ8Z1w7sPxjR7TJ8u/AyxloOgbKFGEAVBbXr++edx6dKlVm8t5ffcDI7jsGrVKlRWViI9PR15eXkYPHgwALR4noMHD+KXX37BN998g+HDh2PAgAHYsGEDnJycsGXLlhbPJZfL4e7ubnCzVwpHqbCERYqZV5SnIIiYEl875+9rxWjQWM+MInPkA/EeHRoKuYMEZ7NUOKZfl8zWXSuqQkVdAxSOEjwxKhIABUG/nstFrqoWPq5yTIntAkepBNEB7rg3NhjPjOuOKD/XZo/pE6T7HLuQU240QNZqGbLK7DsxukM5Qb6+vvD1tcxSCk1JpVIEBeky3r/++mvEx8e32I7qat2H6o09RRKJBFqt9bwpWrsoP1dcLazCtcJK3NbdPH/zeo1WqOlCQRAxhb5BSng4O6Ksuh4n00sxNMJb7CaBMYaTrVSKvlXernI8EBeMr45l4NPfr1nFc75V/FBY7y5K3NHLH1IJh+S8CmQUV6Ort/29VzDG8Okf1wAAM+NDIXdo30za7v5ucJBwKKlSI1dViy4ehr09hZV1UDdoIZVwCFQqTN5uW2C2nKCMjAwkJiYiIyMDGo0GiYmJSExMRGVlY+2Z1NRUJCYmIi8vDzU1NcI+arVuEcSioiJs3LgRycnJSExMxNNPP40dO3bgvffeE45x/PhxREdHIzs7GwAQHx8PT09PzJw5E2fPnsXly5fx4osvIi0tDZMmTTLX0+10Inx13yquFVWZ7Rw5ZTXQMkDuIDFJ3RRCpBIOY6P9AQB7L1hHz8HVwkqUVtdD7iARhidMbe7ICHAccCC5AKkFllvypr2+SLiORz8/1u56RnwSef9gD3g4yzBYHzza6yyxhGvFuJBTDoWjBI8MDW334xSOUqGHyNiQWNMaQZ0pqb4jzPasly5ditjYWCxbtgyVlZWIjY1FbGwsTp48Kewzd+5cxMbG4uOPP8bly5eFfXJycoR9tmzZgoEDB2L48OG4cOECDh8+LAyJAbqen5SUFNTX615cPj4+2L17NyorK3H77bdj4MCB+PPPP/Hjjz+if//+5nq6nU6kPgi62o51Z25WZkljL5C9z/ogpnNHL10QtO9SnlXkyPAVneNCPc1WjC7cxwXj9c/7sz/SzHKOm1VUWYcVv17CH1eK8OPZ7HY95qw+H6h/iC5oFP6mdjokxv9NH4gLhqeLrEOPFSpH5zSf7cu/B9vjchk8s02R37x5MzZv3tzqPocPH271fh8fHyQkJLS6z+jRo5u90Q0cOBB79uxpTzNJC/hkxGuF5usJonwgYg63dfeB3EGCzJIaJOdVCLMdxXLksq6GzygzDSvz5t8WgT0X8rHzdDaeG98dfm7WMbyxNSEddfqKz0dSCvFYfFir+6sbtLior1HWXz/b6Y5e/vjvLxdx4noJSqvUHQ4EbFlqQQUOJheA44A5w8M7/Pg+Xdzx7SngQis9Qfa4cCrPPvu/SJsifXQ9QbmqWlTpl7UwNaoRRMzBWeaAkd10AYfYQ2K19Rqh2vGoHuYNguJCvTCgqwfUGi2+OJpu1nO1V41ag61/N7bl6NVi1NZrWn1MSl4F1A1aKJ0cEarP/wnxckbPQHdoGXAwucCsbbY2n/95HQAwNtpfSFPoiD6tTJPPsvOZYQAFQaQFSmdH+Ljqvm2lmSkviGoEEXMZ31ufFyRyDsmJ6yWordfC312OHv7mL944/zbdTKqtf6ejWm2eLy8d8d3pLJRUqRHi5QR/dzlq6jU4qZ8p15Kz+qTofsFKg2FyfkhM7L+pJRVX1mHn6SwAwLyRHe8FAnR13zgOyC+vQ+ENJU/4StL2OjMMoCCItCLCzHlB1BNEzGVstB8knG5qcFYrSwaY25EU3VDYbd18LZL3dkcvf4R5O0NVU4+tCeL2Bmm0DJ//qctleXx4uDAceDil9Z6cpknRTfE5T79fLmqzN6mz+OZEJuoadIUQB4ff3MxCF7mDUKDzwg29QdQTREEQaUWkPi/oqpnygigniJiLt6tcmI4uZjLt71f0+UBmHgrjSSUc/jUmCgDwwYEryFPVWuS8xuy/lI+0oioonRwxbWAIRnX3A9CYI9WSc0JStIfB9t5d3NFFqUBNvQZ/pVpXRXBz4YdSp8UF31IQzQ+JXWiSHK3RMqFEib0umQFQEERaYc4ZYqrqeqhqdDP67HlmAjEfvudArLygnLIaXM6vhIRrXOjUEh4YEIzYrh6oUmuw4rdLFjvvjT79XVfX5pGhXeEid8CIKB9IOOBKQSWy9R++N6qqa8AV/RT//sGG5QQ4jsM4O5olxhgTlgqJCbm1IpvGKkfnl9eiXsPgIOEQ4G4dSfRioCCItIifIXa1wPRBED8W7eMqh7PMqtfxJTZqfK8AAMBx/YwiS/td3+PRP0RX68ZSJBIOr0/pAwkH/Hw2R5Rek1PppTiZXgqZVIKZ+tlgSmdHDOiq+zDnhwlvlJStgpYBgUoF/Ix8MI+J1vUmHW1jWRSNlllFeYRbkVFSjbLqesikEvQIuLV8st76ytEHkgvw9u5kqKrrhaGwLh5OkErst0QJBUGkRXxPUFpRFbRa076hNA6FUS8QMY+u3s6IDnCDRstEmVEkDIWZeWq8MX2ClEJRvaU/JkHdYNlq+Z/pqxtPje1iEMzw1+LIZeN/D34orF+w8aKSA0M9IeF07x+5KuO9SUWVdRi68gD+ufXUTbffGvDXomeg2y3XlxoU5oWR3XygbtBiw+GrGPn2QWw4nArAvvOBAAqCSCuCPZ0hk0pQ16Btsfv6ZlE+ELGE8SLNKGrQaIUiiWIEQQDw/Pge8HaR4WphlZCgbAnpxVXCivZzRxqu8Ti6h64n56/UYqOBGT8z7MZ8IJ6bwlHIbzl2zfg6afsv5qOwog4HkgtQ12C7CdT80iF9WwgIO8JRKsEXcwbjk0fj0N3fFeW1DTis742z55lhAAVBpBVSCYcwH90LxNTLZ1AQRCxhfG/dkJilZxSdzSpDRW0DPJwd0e+GWU6WonRyxJK7egLQJUnnmPiLTEt+PZ8LxoCR3XzQ/YayAL27uMPbRYbKugaczmg+VV4Iglq5ZkP0s6SOpRkfEuODT42WmbXYq7k19op5mOR4HMdhfO8A7Hr6Nqx9qL+Qi9nHBEGWLaMgiLQqQl800dR5QZk0PZ5YQNMZRX9esVxuDJ/zMiLKR9R8i/sHBGFQmCdq6jV4/ZeLFjknXwdojL7XpymJhBMWZD58Q15QSZVaWMaB7+0xZki4boFYYz1BGi3Dn01yoC7nW986au2h0TIhibmlocGbJZVwuDc2GAeeG43fnhqJR4Z0NenxbQ0FQaRVkX765TOKTBsEUU8QsQT+2y+gK9xnKZZaKqMtHMfhv1P6QCrhsCspz+gimqak1TKcvK4LTgaFGa9rM7oHnxdkGATxvUARvi5QOjm2eI5B4V7gOF3vdEG5YQmAc1llwqxTQFd92halFVWiSq2Bk6MUUTdRJbo9ZA4S9OribvfrNlIQRFrV2BNkum5ljZYhWz8zoas3BUHEvB4cGAKOA3Yl5SGhjVlFNyO1oAIVtY0fvCVVapzTBxu3iRwEAbqKwZP6BgIA/mfm3KArBZUor22As0yKnoHGZzSNiPIBxwGXcsuRrw9iclU1+FJf3LG1oTBAN8zXM0A32+lYmmFv0O+Xdb1ADvreN1vtCTqbqfv/6d3F3W5Xd7cUurqkVZF+pq8VlKuqQYOWQSaVwN9KFnkknVevLu6Yoe/yf+2nC6jXmG6m1I+J2Rj37u+Ie2M/5m45ie/PZGFXki4nJjrADf5WUn9lzgjdkgs/n8tp1ntiSif0vUADunq2+OHt7SpHP/1w17ensvDaTxcw6u3DOKCfwccHbK0ZEmE8L+gP/Yy8yf10x0i20Z6g89mmzQciLaMgiLSKrxVUUFFn8G33VvBDYcFeTpDYcX0KYjkvjO8BT2dHpORX4AsTLSfBGMNHh68C0K18vv9SPp7ddhavfJ8EwHJVotsjJsQDcaGeqNcwgwVNTY0fChsY1npxv1H6fKHVe1Kw+eh1qDVaDA73wjfzhwoFEVtjLC+ovLYeZ/RLbvCz0rJKa1BppgWgzalxlpx9Jy1bAgVBpFXuCkf4uskBwGQzLYSkaDufmkksx8NZhpcmRgMA3tt3GQUVt94bciytBMl5FVA4SrBjQTyeuj1KWGoGaCzWaC3mDNf1Bn11LMNsM+VO6JOiW8oH4o3r2Zg0PTDUE/83dwi2zR+KoRHe7ToPv47WlYJKFFfqFgU9mloMjZYhwtcFfYKUwvvWFRsbEqvXaHFRv7xF31YSxIlpUBBE2sS/sZsqOZqSookYHhoYgv7BSlTUNeCtXcm3fLzNf10HANw3IBiDwrzw3Pge2P/cKOx55jZ8/69hiAu9taUOTG1Cb38EeTihpEqNHxOzTX78nLIaZJfVQCrhENNCnR9ev2APbJo1CF/PG4odC+IxLMqnQwm6Xi4y9NBPvz+uzwvii1Pe1k3XAxetr7Jsa3lBl/MrUNeghZvCAWHeLm0/gNwSCoJIm4TV5E2UHJ2hnwZLQRCxJImEw/IpfcBxwM7T2cLQzc3IKq0WCjDOGhYmbOc4Dj0C3BDb1boCIABwkEowc5iuivTnf6aZfFmJk+m6XqDeXdzhIm97KZwx0X6Ij/S+6dlJjXlBJWCMCcuU3NZdt04bX6MoJc/0y/6Y03l9faC+QUpKF7AACoJIm/jlM0zWE1SsC6aoRhCxtJgQDzw0MAQA8J8fL6DhJpOkt/6dDi0Dhkd5NysIaM0eGtQVzjIpLudX4q9U086UE/KBQlsfCjMVfkjs72vFuF5cjazSGjhKOWFIje8pSskvb/EY1uisiYskktZREETa1LiQ6q33BOWqaoSZD9G3uCggITfjxQk9oHRyxKXc8ptaU6xGrcG2E5kAICwOaiuUTo6YFhcMAPj8z2smPXZjPpBlesH4ICglvwI/JeYA0AVg/ILM3QNssyeIXy7D1EUSiXEUBJE28cW60oqroLnFhVS/PpYBLdOVvg/zofFuYnnernLcP0AXCOxK6viaYj8mZqOsuh7Bnk4Y27PtmUzWZtbwcHAccCilsM3SF+0dMiuvrUdynq7HJc5CQZCfmwIRvi5gDPhMH9A1rcvUTV/eo6iyTkietna19RqhwCMFQZZBQRBpUxcPJ8gcJFA3aIUihzejXqPFN/pv0PwK14SIYVI/3cyt/RfzO7TIJmMMm49eB6DrBRJzSYybFe7jgrHRutlZ7+693GKgcyW/AiNWHcJz2xPbPObp9FIwBoR5O8PPgrW/+KnyFbW6afAju/kI97nIHYS8w8v5ttEbdCm3HA1aBi8XGYI87Ht1d0uhIIi0SSrhEKHvtbmVoon7LuajoKIOPq5yTOhtXdOHiX2JDfFEgLsCFXUNHVpT7O9rumnxTo5SPKjPLbJF/xoTBamEw6/nc7HuYGqz+wsr6jB78wlkl9Xg+zPZKGqjJ4VfL2xgG1PjTW1oROP5fFxl6BXobnA/n69lKzPEzjdZL8zel7OwFAqCSLsIeUG3EAR9qS/S9tCgYMgc6F+PiEci4TCxjy4Q/+18+4fENh/VLTtx34AgKJ1bXt/K2g3o6on/TukNAHh332WDKfM1ag3mbjmBLH2vL2PAoTZyp04I64VZdlYc3xME6JbjuHE2VY8A3ZBYio0EQfxyGZQUbTn0SUTahZ8hdvUmCyZeLazE0avF4Djg4cH2vWoxsQ536Zdn2HcxD+qGtmeJXcwpx96L+QAMp8XbqhlDQjFXv5zGi9+ew6n0Emi0DM9sO4OzWSp4ODvi3tggAMCBSy0HQeoGLRL1lZot3RMUoFQgXN9LbWydtsZp8rYRBJ3PLgMAYVkRYn5tF3MgBI09QXzyY0d99XcGAOD2Hn4IpkrRxAoMDPWEn5scBRV1+OtqEcb08GtxX8YY3vj1IhgDJvULRDcbmhbfmiV39cT14mrsv5SP+V+cwphoP+y5kA+ZVIJPHxsIhYMU35/Jxh9XClHXoIHcQdrsGEk5KtQ1aOHlIhOGzS1p5X198VdqEe7u36XZfT34gol5FWCMWfUQU1VdA1ILdD3tlBRtOdQTRNplYKgXpBIOZzLKhKJk7VWj1uDbU5QQTayLwZDYudxW991/qQBHrxZD5iDBYv3yG52BVMLhg4dj0LuLO4qr1Pj2VBYA4J0H+2NQmBf6BLnD312OKrUGf18zXlyysT6QpyhBxtAIbzw/vgccjSzYGuHjCgcJh4q6BuSqzLdwrCkkZaugZUCAuwJ+VrLwrj2gIIi0S4iXs1AT5bWfL7Rr+ID389kclNc2IMTLyWiXNSFi4YfE9l7Mb3F1eXWDFm/+dgmAbv2tzlbk01nmgM9nDkKA/oP3xQk9cI++V4XjONwerSsDcOBSvtHHH09r33phYpA5SIRebGvPC+IXf43t6iFqO+wNBUGk3Z65oxt8XGW4VlglJIi2x5fHdAnR/xgcapNTiknnNSjMCz6uMqhq6nH0qvEKylv/TkdaURV8XGVYOCbSwi20jAClAr89PRI7FsTjX6MNnyO/2OmBSwXNptPX1mtwMr19K8eLRZghZuV5QYkZZQDQ5rprxLQoCCLt5q5wFFbifn//FRSUt929fPJ6Cc5lqSCTSvDgwGBzN5GQDpFKOKFcw67zzYfEyqrV+ODAFQDAc3f0gJvCdmeEtcXLRYZBYV7NhrSGR/lA4ShBdlkNkm8IJLYcvY6y6noEKhXoY6XJvI3LZ1hvEMQYw+kMXY+aNa4715lREEQ65IEBwYgJ8UCVWtPmStyn0kvw+JaTAHTJpN6ucks0kZAOmaQfEttzIa/ZWmLv7b8CVU09ogPc8NAg260LdCsUjlKMiNIVIWw6JFZapcb6Q7oaQy3l5FiDxuUzrDcIylXVoqCiDlIJh75WGkx2Vtb5X0uslkTCYfk9vXUrcZ9peSXufRfz8Y9Pj0FVU4/Yrh5YOrmXhVtKSPsMDveCl4sMpdX1QvJvaZUa+y7mC7WtXp3Uy66HcvnlQfY3mSq/7mAqKmob0DPQXZhKb434nqArBZW3vOyPufAlBqID3OAkaz4Dj5gPTZEnHdZfvxL3NycysfTHC9j6+GB4uciEbvRvjmfg5e/PQ8uAsdF+WP+PAfTCJlbLQSrBhN4B+Pp4BhbvPIcGDUNek6HesdF+GNFkOQZ7xC+zcTarDAUVtahRa7D17+sAgJfvirbqALGrlzMUjhLU1muRXlyFCH3NM2tyRj8URvlAlkdBELkpL07ogd/O5+Jibjni3tgPd4UDwn1c4OUiw6EU3RT6BwcG4817+8LBSrvJCeFN7heIr49nCFWSAd2H54CuHnj5rp4itsw6+Lkr0C9YiXNZKhxKLsDvl4tQr2G4rbsvRnaz7hmfEgmH7v5uOJelwuX8CqsMghKFmWGUD2RpFASRm+LtKse7D8Zg+S8XkFlSg/LaBpzNUgn3LxoThefHd7fq4mSE8IZFeuOdaf1RrdYN70QHuHXqJOibMa6nP85lqfDx79dwrbAKHAcsudM2aiZ189MFQVfyKzGxj9itMVSv0eKc/r2TeoIsj4IgctPG9fLHuF7+qK3XIL24GmlFlbheXI3oADeMbqX6LiHWhuM4PBBHsxdbM7anH97ddxnX9EvnPDAgGD1vWLDUWgUqdTWQCipaXwhWDCl5Fahr0MJd4SBKxW17R0EQuWUKRyl6BLgJJeoJIZ1Pr0B3dFEqkKOqhcJRgufH9xC7Se3m4yoDABRVWl8QxOcD9Q/xaLYALDE/StYghBDSJo7jMFlfSXr+bZEIUNrO0g6+brq2WmUQRPlAoqKeIEIIIe3y3B3dMa6nPwZZaXXoljT2BKlFbklzfKXoWMoHEgUFQYQQQtpF4SjF4HDrWyOsLT5uukKtRVaWE1RWrca1Il2OFSVFi4OGwwghhHRqPvpq9RV1Dait14jcmkb81Pgwb2d4usjEbYydMmsQtGLFCgwbNgzOzs7w8PBodv/Zs2fx8MMPIyQkBE5OTujZsyfef//9ZvsdPnwYAwYMgFwuR1RUFDZv3tzmuc+dO4eRI0dCoVAgJCQEb7/9tgmeESGEEFvjrnCATF+vzJrygqg+kPjMGgSp1WpMmzYNTzzxhNH7T506BT8/P3z55Ze4cOECXnnlFSxZsgTr168X9klLS8OkSZMwZswYJCYm4plnnsHcuXOxZ8+eFs9bXl6O8ePHIzQ0FKdOncLq1avx2muv4ZNPPjH5cySEEGLdOI4T8oIKrWhI7AytHC86s+YELV++HABa7LmZM2eOwe8RERFISEjAzp07sWjRIgDAxo0bER4ejjVr1gAAevbsiT///BNr167FhAkTjB73q6++glqtxv/+9z/IZDL07t0biYmJePfddzF//nwTPTtCCCG2wsdNjhxVrdUkRzPGmvQEeYjaFntmdTlBKpUKXl6NiXcJCQkYN26cwT4TJkxAQkJCi8dISEjAbbfdBplMZvCYlJQUlJaWGn1MXV0dysvLDW6EEEI6B199XpAlhsO+OZ6Bj49cbXWftKIqqGrqIXOQIDrANopOdkZWFQQdPXoU27ZtM+itycvLg7+/v8F+/v7+KC8vR01NzY2HaPUx/H3GrFy5EkqlUriFhITcylMhhBBiRfjkaHPPECupUmPJ9+exclcyMoqrW9yP7wXqG6SEzMGqPortSoev/OLFi8FxXKu35OTkDjckKSkJU6ZMwbJlyzB+/PgOP/5WLVmyBCqVSrhlZmZavA2EEELMw8fNMlWjj14tAmO6n1MLK1rc7wzVB7IKHc4Jev755zFr1qxW94mIiOjQMS9evIixY8di/vz5ePXVVw3uCwgIQH5+vsG2/Px8uLu7w8nJyejxWnoMf58xcrkccrm8Q+0mhBBiG4SeIDPnBP15pUj4+WpBFW5vYY3Zc1llAHTLZRDxdDgI8vX1ha+vr8kacOHCBdx+++2YOXMmVqxY0ez++Ph4/Pbbbwbb9u3bh/j4+BaPGR8fj1deeQX19fVwdHQUHtOjRw94etJUREIIsTd8EGTO2WGMMfzRNAgqrDS6n1bLcDlfd5+tLELbWZl1IDIjIwOJiYnIyMiARqNBYmIiEhMTUVmp++MnJSVhzJgxGD9+PJ577jnk5eUhLy8PhYWFwjEWLFiAa9eu4aWXXkJycjI2bNiA7du349lnnxX2Wb9+PcaOHSv8/o9//AMymQyPP/44Lly4gG3btuH999/Hc889Z86nSwghxEr5WCAxOr24GtlljbmqLQVB2WU1qKnXQCaVIMzb2WztIW0z6xT5pUuXYsuWLcLvsbGxAIBDhw5h9OjR+Pbbb1FYWIgvv/wSX375pbBfaGgorl+/DgAIDw/Hr7/+imeffRbvv/8+goOD8dlnnxlMjy8qKsLVq42Z+EqlEnv37sXChQsRFxcHHx8fLF26lKbHE0KInfLV5wQVmjEI+jNV1wvk5SJDSZUaVwurjO53OV+XKxTh6wIHKSVFi4ljjE/hIk2Vl5dDqVRCpVLB3Z26KwkhxJapquvR/797AQDJr0+EwlFq8nMs2HoKuy/k4YnRkfjosO6L+Zn/3NFsSYyPDl/Fqt3JuLt/F6x7ONbk7bB3Hfn8phCUEEJIp+fu1Lh0RnGV6ZOjNVqGo1d1PUF39PJHF6UCAHCtqPmQ2BV9T1B3P1eTt4N0DAVBhBBCOj2O4+CtXzrDHLWCzmerUF7bADeFA/oFKRGpD3CuFjQfErtcoAuCuvm7mbwdpGMoCCKEEGIXzJkc/Zc+Hyg+whsOUgkiffVB0A3J0VotQ2qBblt3f+oJEhsFQYQQQuyCORdR/eOKblbzyG4+AIBIXxcAzYOgzNJq1NZrIXOQINTbxeTtIB1DQRAhhBC7YK6eoGp1A06nlwEARnTT1dFr7AkyHA7j6wNF+rpCKuFM2g7ScRQEEUIIsQs+buapGn08rQRqjRZBHk5C3Z8IfRCUUVINdYNW2JefHt+DhsKsAgVBhBBC7AK/krypawXxS2WMiPIBx+l6d/zd5XCRSaHRMmSUNPYG8UEQJUVbBwqCCCGE2AWhJ8jEOUF8kcTh+nwgQDcbjZ8hllrQNAjik6IpCLIGFAQRQgixC3xitClzggor6pCcp+vdGR7pbXDfjTPENFom/Ewzw6wDBUGEEELsgq8ZFlHlCyT2CnSHt/74vBtniKUXV0HdoIXCUYIQT1ozzBpQEEQIIcQu8LPDymsbUNegMckxhXygJkNhvBtniPFDYVF+rpDQzDCrQEEQIYQQu6B0coSDPvgoNsEMMcYYjl4tBgAMj2oeBPEzxK4VVoIx1mS5DMoHshYUBBFCCLELEkmTpTNuyAtSN2ix8chVXC8yvvK7MRkl1cguq4GjlMOgMM9m94d6O0PCARW1DSisrMNlfaVomhlmPSgIIoQQYjd83YwXTNx+MhNv7UrGP7eeglbL2nWsv1J1vUCxIZ5wljk0u1/hKEWIly7352pBVWNPECVFWw0KggghhNgNoWp0heFw2LG0EgBASn4Fdl/Ia9ex+KToYVHeLe7D5wVdzq/ANX1uEE2Ptx4UBBFCCLEbPi0UTDx1vUT4+YMDV9rsDdJqGRL0+UDDIpvnA/H4GWIHkgug1mjh5ChFkIfTTbWdmB4FQYQQQuyGj5Fp8jllNchR1UIq4eAmd0ByXgX2tNEbdLmgAsVVajg5ShET4tHifnxP0FF9QcVu/jQzzJpQEEQIIcRuGCuYeDK9FICu1s/sEeEAgPfb6A3i84EGhXtB5tDyRyk/Q6xBf6xuNDPMqlAQRAghxG4YS4zmh8LiQj3x+PBwoTdo78WWe4MS9PlAN1aJvhE/HMbrEUBJ0daEgiBCCCF2Q0iMblIn6FSGricoLtQTSmdHzB4eBgB4b7/x3qAGjRbHrukCJ2P1gZrycpHBw9lR+J2mx1sXCoIIIYTYjRt7gqrqGnApVzd1faC+1s+cEa33Bp3LVqGirgFKJ0f0DHRv9Xwcxwl5QQDNDLM2FAQRQgixG3xPUFl1PdQNWiRmlkGjZQjycEKgUjdry8NZhln63qD3D6Q26w3iZ4XFR3hD2o4kZ35IzFXugC5KhameCjEBCoIIIYTYDQ8nRyFwKa6qw8nrjUNhTT0+Ihyucgdcyi3H//5KM7jvr9S26wM1xSdHR/m5guNoZpg1oSCIEEKI3ZBIOHi76GeIVahxMl2X2zPwhmUvPJxleHFCDwDAm79dwsHkfABAbb1GmE3WWn2gpib0DkA3P1f8Y3BXkzwHYjoUBBFCCLEr/JBYQUUtzmSUAWjeEwQAj8WH4uHBIdAy4KmvE5GSV4HT6aVQN2jh7y5vNvOrJeE+Ltj33Cg8OCjEZM+BmAYFQYQQQuyKjz45+q/UYlTWNcBV7oDogOYJzhzHYfk9fTA0wguVdQ14fMsJ/HwuF4CuF4iGtmwfBUGEEELsCl8wka8KHdvVo8UEZ5mDBB/NiEOotzOySmvw9fEMAMCwNuoDEdtAQRAhhBC7wk+Tzy6rAQAM6Np8KKwpTxcZPp85CG6KxpXih7VRH4jYBgqCCCGE2BVffU4Q78akaGOi/Fzx4T8GwEHCoW+QkhZB7SQc2t6FEEII6Tx8mgRBEg6IbaMniHdbd18ceWmMQY8QsW30lySEEGJXmgZB0QHucJW3/6OQeoA6FxoOI4QQYld83GTCz+0ZCiOdFwVBhBBC7ErTniBj9YGI/aAgiBBCiF3xdJbBRSYFxwGDwrzEbg4REeUEEUIIsStSCYePHx2IKnUDulCOj12jIIgQQojdGdGN6vwQGg4jhBBCiJ2iIIgQQgghdomCIEIIIYTYJQqCCCGEEGKXKAgihBBCiF0yaxC0YsUKDBs2DM7OzvDw8Gh2/9mzZ/Hwww8jJCQETk5O6NmzJ95///1m+x0+fBgDBgyAXC5HVFQUNm/e3Op5Dx8+jClTpiAwMBAuLi6IiYnBV199ZaJnRQghhJDOwKxBkFqtxrRp0/DEE08Yvf/UqVPw8/PDl19+iQsXLuCVV17BkiVLsH79emGftLQ0TJo0CWPGjEFiYiKeeeYZzJ07F3v27GnxvEePHkW/fv3w3Xff4dy5c5g9ezYee+wx/PLLLyZ/joQQQgixTRxjjJn7JJs3b8YzzzyDsrKyNvdduHAhLl26hIMHDwIA/v3vf+PXX39FUlKSsM/06dNRVlaG3bt3t7sNkyZNgr+/P/73v/+1a//y8nIolUqoVCq4u7u3+zyEEEIIEU9HPr+tLidIpVLBy6uxjHlCQgLGjRtnsM+ECROQkJBwS8e9UV1dHcrLyw1uhBBCCOm8rCoIOnr0KLZt24b58+cL2/Ly8uDv72+wn7+/P8rLy1FTU9Ou427fvh0nTpzA7NmzW9xn5cqVUCqVwi0kJOTmngQhhBBCbEKHg6DFixeD47hWb8nJyR1uSFJSEqZMmYJly5Zh/PjxHX58Sw4dOoTZs2fj008/Re/evVvcb8mSJVCpVMItMzPTZG0ghBBCiPXp8Nphzz//PGbNmtXqPhERER065sWLFzF27FjMnz8fr776qsF9AQEByM/PN9iWn58Pd3d3ODm1vvDdkSNHcPfdd2Pt2rV47LHHWt1XLpdDLpd3qN2EEEIIsV0dDoJ8fX3h6+trsgZcuHABt99+O2bOnIkVK1Y0uz8+Ph6//fabwbZ9+/YhPj6+1eMePnwYkydPxqpVqwyG1wghhBBCADOvIp+RkYGSkhJkZGRAo9EgMTERABAVFQVXV1ckJSXh9ttvx4QJE/Dcc88hLy8PACCVSoVAa8GCBVi/fj1eeuklzJkzBwcPHsT27dvx66+/CudZv349vv/+exw4cACAbghs8uTJePrpp3H//fcLx5XJZK0mRzfFT5qjBGlCCCHEdvCf2+2a/M7MaObMmQxAs9uhQ4cYY4wtW7bM6P2hoaEGxzl06BCLiYlhMpmMRUREsE2bNhncv2zZMoPHtHTeUaNGtbvtmZmZRo9BN7rRjW50oxvdrP+WmZnZ5me9ReoE2SKtVoucnBy4ubmB4ziTHru8vBwhISHIzMykGkQWQNfbsuh6WxZdb8ui621ZN3O9GWOoqKhAly5dIJG0Pv/LrMNhtkwikSA4ONis53B3d6cXkQXR9bYsut6WRdfbsuh6W1ZHr7dSqWzXflZVJ4gQQgghxFIoCCKEEEKIXaIgSARyuRzLli2jukQWQtfbsuh6WxZdb8ui621Z5r7elBhNCCGEELtEPUGEEEIIsUsUBBFCCCHELlEQRAghhBC7REEQIYQQQuwSBUGEEEIIsUsUBFnYhx9+iLCwMCgUCgwZMgTHjx8Xu0mdwsqVKzFo0CC4ubnBz88PU6dORUpKisE+tbW1WLhwIby9veHq6or7778f+fn5IrW4c3nrrbfAcRyeeeYZYRtdb9PKzs7GI488Am9vbzg5OaFv3744efKkcD9jDEuXLkVgYCCcnJwwbtw4XLlyRcQW2y6NRoP//Oc/CA8Ph5OTEyIjI/H6668bLMhJ1/vm/f7777j77rvRpUsXcByHH374weD+9lzbkpISzJgxA+7u7vDw8MDjjz+OysrKDreFgiAL2rZtG5577jksW7YMp0+fRv/+/TFhwgQUFBSI3TSbd+TIESxcuBB///039u3bh/r6eowfPx5VVVXCPs8++yx+/vln7NixA0eOHEFOTg7uu+8+EVvdOZw4cQIff/wx+vXrZ7CdrrfplJaWYvjw4XB0dMSuXbtw8eJFrFmzBp6ensI+b7/9Nj744ANs3LgRx44dg4uLCyZMmIDa2loRW26bVq1ahY8++gjr16/HpUuXsGrVKrz99ttYt26dsA9d75tXVVWF/v3748MPPzR6f3uu7YwZM3DhwgXs27cPv/zyC37//XfMnz+/441p97Lq5JYNHjyYLVy4UPhdo9GwLl26sJUrV4rYqs6poKCAAWBHjhxhjDFWVlbGHB0d2Y4dO4R9Ll26xACwhIQEsZpp8yoqKli3bt3Yvn372KhRo9jTTz/NGKPrbWr//ve/2YgRI1q8X6vVsoCAALZ69WphW1lZGZPL5ezrr7+2RBM7lUmTJrE5c+YYbLvvvvvYjBkzGGN0vU0JAPv++++F39tzbS9evMgAsBMnTgj77Nq1i3Ecx7Kzszt0fuoJshC1Wo1Tp05h3LhxwjaJRIJx48YhISFBxJZ1TiqVCgDg5eUFADh16hTq6+sNrn90dDS6du1K1/8WLFy4EJMmTTK4rgBdb1P76aefMHDgQEybNg1+fn6IjY3Fp59+KtyflpaGvLw8g+utVCoxZMgQut43YdiwYThw4AAuX74MADh79iz+/PNP3HnnnQDoeptTe65tQkICPDw8MHDgQGGfcePGQSKR4NixYx06H60ibyFFRUXQaDTw9/c32O7v74/k5GSRWtU5abVaPPPMMxg+fDj69OkDAMjLy4NMJoOHh4fBvv7+/sjLyxOhlbbvm2++wenTp3HixIlm99H1Nq1r167ho48+wnPPPYeXX34ZJ06cwFNPPQWZTIaZM2cK19TY+wtd745bvHgxysvLER0dDalUCo1GgxUrVmDGjBkAQNfbjNpzbfPy8uDn52dwv4ODA7y8vDp8/SkIIp3OwoULkZSUhD///FPspnRamZmZePrpp7Fv3z4oFAqxm9PpabVaDBw4EG+++SYAIDY2FklJSdi4cSNmzpwpcus6n+3bt+Orr77C//3f/6F3795ITEzEM888gy5dutD17mRoOMxCfHx8IJVKm82Oyc/PR0BAgEit6nwWLVqEX375BYcOHUJwcLCwPSAgAGq1GmVlZQb70/W/OadOnUJBQQEGDBgABwcHODg44MiRI/jggw/g4OAAf39/ut4mFBgYiF69ehls69mzJzIyMgBAuKb0/mIaL774IhYvXozp06ejb9++ePTRR/Hss89i5cqVAOh6m1N7rm1AQECzCUUNDQ0oKSnp8PWnIMhCZDIZ4uLicODAAWGbVqvFgQMHEB8fL2LLOgfGGBYtWoTvv/8eBw8eRHh4uMH9cXFxcHR0NLj+KSkpyMjIoOt/E8aOHYvz588jMTFRuA0cOBAzZswQfqbrbTrDhw9vVvLh8uXLCA0NBQCEh4cjICDA4HqXl5fj2LFjdL1vQnV1NSQSw49HqVQKrVYLgK63ObXn2sbHx6OsrAynTp0S9jl48CC0Wi2GDBnSsRPeUlo36ZBvvvmGyeVytnnzZnbx4kU2f/585uHhwfLy8sRums174oknmFKpZIcPH2a5ubnCrbq6WthnwYIFrGvXruzgwYPs5MmTLD4+nsXHx4vY6s6l6ewwxuh6m9Lx48eZg4MDW7FiBbty5Qr76quvmLOzM/vyyy+Ffd566y3m4eHBfvzxR3bu3Dk2ZcoUFh4ezmpqakRsuW2aOXMmCwoKYr/88gtLS0tjO3fuZD4+Puyll14S9qHrffMqKirYmTNn2JkzZxgA9u6777IzZ86w9PR0xlj7ru3EiRNZbGwsO3bsGPvzzz9Zt27d2MMPP9zhtlAQZGHr1q1jXbt2ZTKZjA0ePJj9/fffYjepUwBg9LZp0yZhn5qaGvavf/2LeXp6MmdnZ3bvvfey3Nxc8RrdydwYBNH1Nq2ff/6Z9enTh8nlchYdHc0++eQTg/u1Wi37z3/+w/z9/ZlcLmdjx45lKSkpIrXWtpWXl7Onn36ade3alSkUChYREcFeeeUVVldXJ+xD1/vmHTp0yOj79cyZMxlj7bu2xcXF7OGHH2aurq7M3d2dzZ49m1VUVHS4LRxjTUpgEkIIIYTYCcoJIoQQQohdoiCIEEIIIXaJgiBCCCGE2CUKggghhBBilygIIoQQQohdoiCIEEIIIXaJgiBCCCGE2CUKggghhBBilygIIoQQQohdoiCIEEIIIXaJgiBCCCGE2KX/B2Vy3yv7CIOEAAAAAElFTkSuQmCC",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "pd.DataFrame(FDS.smoothed_mean_last_epoch.numpy()).iloc[:, 7].plot(\n",
-    "    title=\"Smoothed mean bina values for 'Longitude' feature\"\n",
-    ");"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "id": "7afc8719",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "smoothed_features = FDS._smooth(torch.clone(features).detach(), labels, epoch).numpy()\n",
-    "left_bin_edges_indices = find_bin(\n",
-    "    FDS.bin_edges, labels.squeeze(), ret_value=False\n",
-    ").numpy()\n",
-    "continuous_cols = df.drop(columns=[\"MedHouseVal\"]).columns.values.tolist()\n",
-    "\n",
-    "df_w_bins = df.copy()\n",
-    "df_w_bins[\"MedHouseVal_bins\"] = left_bin_edges_indices\n",
-    "df_smoothed_w_bins = df_w_bins.copy()\n",
-    "df_smoothed_w_bins[continuous_cols] = smoothed_features"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "id": "d3e732b1",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAGzCAYAAADaCpaHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABvIElEQVR4nO3dd3xUVfo/8M/MJDPpk95DINQAoSuCgEAQRFRsgOz+VljF7lrXgq4CrlhQV113bbsKlq/SxLKKItKbCEgChIQSEpIQ0ntPZs7vj8m9mUkmlUzmzuTzfr3yIty5c+fc3CnPnPOc56iEEAJEREREDkpt7wYQERERXQoGM0REROTQGMwQERGRQ2MwQ0RERA6NwQwRERE5NAYzRERE5NAYzBAREZFDYzBDREREDo3BDBERETk0BjOkaDt37oRKpcLOnTt75PGmTp2KqVOndusxP/vsMwwZMgSurq7w9fXt1mP3RmvWrIFKpUJ6erq9m9JCeno6VCoVXn/99W497pkzZzBz5kzo9XqoVCp888033Xp8JVCpVFi+fLn8f2vXuW/fvrjuuut6vnHN2OJ9gi4Ng5leTnrDOHz4sL2b0mFffPEF3nrrLXs3o0NSUlKwePFi9O/fH//5z3/w4Ycf2uRx9u/fj+XLl6OkpMQmxyf7WrRoEY4fP46VK1fis88+w7hx4+zdJKd38uRJLF++XJFBM7XkYu8GELVlypQpqK6uhlarlbd98cUXOHHiBB555BH7NayDdu7cCaPRiLfffhsDBgyw2ePs378fK1aswOLFi9n742Sqq6tx4MABPPvss3jwwQft3Zwe86c//Qm33XYbdDqdXR7/5MmTWLFiBaZOnYq+ffta3Pbzzz/bpU3UOvbMkKKp1Wq4ublBrXbMp2peXh4AOGyAUVlZae8m9Hr5+fkAuvc55AjXVaPRwM3NDSqVqluO153nrNVqLb5gkf055icE9bijR49i9uzZ8PHxgZeXF+Lj4/Hrr79a7CMNWe3btw+PPfYYgoKC4OnpiZtuukl+Q5YYjUYsX74c4eHh8PDwwLRp03Dy5En07dsXixcvlvdrnjMzdepU/PDDDzh//jxUKhVUKpX8ram1XIrW8m4+/PBD9O/fH+7u7rj88suxZ88eq+deW1uLZcuWYcCAAdDpdIiKisKTTz6J2traNv9mffv2xbJlywAAQUFBLXICfvzxR0yePBmenp7w9vbGnDlzkJSUZHGMY8eOYfHixYiJiYGbmxtCQ0Nxxx13oLCwUN5n+fLleOKJJwAA/fr1k/8u6enpcg7HmjVrWrSveXuWL18OlUqFkydP4g9/+AP8/PwwadIk+fbPP/8cY8eOhbu7O/z9/XHbbbchMzOzzb/Bxo0boVKpsGvXrha3ffDBB1CpVDhx4kSHz7U1zc9F0vz5BAAlJSV45JFHEBUVBZ1OhwEDBuDVV1+F0Wi02G/t2rUYO3YsvL294ePjg7i4OLz99tvttkXy5ptvIjo6Gu7u7rjqqqvk8zSXkpKCW2+9Ff7+/nBzc8O4cePw3XffybcvX74c0dHRAIAnnnjC4vkOdO51uWvXLtx///0IDg5GZGSkfHtHnoetKSkpwaOPPoq+fftCp9MhMjISt99+OwoKCgAAdXV1eP755zF27Fjo9Xp4enpi8uTJ2LFjR7vHbis36ueff8aoUaPg5uaGoUOHYtOmTR0+5/Pnz+P+++/H4MGD4e7ujoCAAMybN8/icdasWYN58+YBAKZNmya/pszfh5rnzOTl5eHOO+9ESEgI3NzcMHLkSHzyyScW+5jnVEnvPzqdDpdddhkOHTrU7t+EWsdhJmpXUlISJk+eDB8fHzz55JNwdXXFBx98gKlTp2LXrl0YP368xf5/+ctf4Ofnh2XLliE9PR1vvfUWHnzwQaxbt07eZ+nSpVi1ahWuv/56zJo1C4mJiZg1axZqamrabMuzzz6L0tJSZGVl4c033wQAeHl5dfqcPvroI9xzzz2YOHEiHnnkEZw7dw433HAD/P39ERUVJe9nNBpxww03YO/evbj77rsRGxuL48eP480338Tp06fbTMR866238Omnn+Lrr7/Ge++9By8vL4wYMQKAKSl40aJFmDVrFl599VVUVVXhvffew6RJk3D06FH5A2vr1q04d+4c/vznPyM0NBRJSUn48MMPkZSUhF9//RUqlQo333wzTp8+jS+//BJvvvkmAgMDAZgCqOZBZEfMmzcPAwcOxEsvvQQhBABg5cqVeO655zB//nwsWbIE+fn5eOeddzBlyhQcPXq01V6DOXPmwMvLC+vXr8dVV11lcdu6deswbNgwDB8+vMPneqmqqqpw1VVX4cKFC7jnnnvQp08f7N+/H0uXLsXFixflXKytW7di4cKFiI+Px6uvvgoASE5Oxr59+/Dwww+3+ziffvopysvL8cADD6CmpgZvv/02pk+fjuPHjyMkJASA6XV15ZVXIiIiAk8//TQ8PT2xfv163Hjjjfjqq69w00034eabb4avry8effRRLFy4ENdee638fO/s6/L+++9HUFAQnn/+ebmXoqPPQ2sqKiowefJkJCcn44477sCYMWNQUFCA7777DllZWQgMDERZWRn++9//YuHChbjrrrtQXl6Ojz76CLNmzcJvv/2GUaNGdfIKmpKhFyxYgHvvvReLFi3C6tWrMW/ePPz000+4+uqr2z3nQ4cOYf/+/bjtttsQGRmJ9PR0vPfee5g6dSpOnjwJDw8PTJkyBQ899BD++c9/4plnnkFsbCwAyP82V11djalTp+Ls2bN48MEH0a9fP2zYsAGLFy9GSUlJi+fMF198gfLyctxzzz1QqVRYtWoVbr75Zpw7dw6urq6d/psQAEG92urVqwUAcejQoVb3ufHGG4VWqxWpqanytuzsbOHt7S2mTJnS4lgzZswQRqNR3v7oo48KjUYjSkpKhBBC5OTkCBcXF3HjjTdaPM7y5csFALFo0SJ5244dOwQAsWPHDnnbnDlzRHR0dKvnkpaWZrG9+THq6upEcHCwGDVqlKitrZX3+/DDDwUAcdVVV8nbPvvsM6FWq8WePXssjvn+++8LAGLfvn1W/2aSZcuWCQAiPz9f3lZeXi58fX3FXXfdZbFvTk6O0Ov1FturqqpaHPPLL78UAMTu3bvlba+99prVc09LSxMAxOrVq1scB4BYtmxZi7YuXLjQYr/09HSh0WjEypUrLbYfP35cuLi4tNje3MKFC0VwcLBoaGiQt128eFGo1WrxwgsvdPpcrV3n5uciiY6Otng+/f3vfxeenp7i9OnTFvs9/fTTQqPRiIyMDCGEEA8//LDw8fGxaHNHSH9vd3d3kZWVJW8/ePCgACAeffRReVt8fLyIi4sTNTU18jaj0SgmTpwoBg4c2OKYr732msVjdfZ1OWnSJIvz6czz0Jrnn39eABCbNm1qcZv0+m9oaLB4jQkhRHFxsQgJCRF33HGHxfbm19DadY6OjhYAxFdffSVvKy0tFWFhYWL06NHtnrMQ1p9nBw4cEADEp59+Km/bsGFDi/ceyVVXXWXxPvHWW28JAOLzzz+Xt9XV1YkJEyYILy8vUVZWJoRoupYBAQGiqKhI3vfbb78VAMT//ve/Fo9FHcNhJmqTwWDAzz//jBtvvBExMTHy9rCwMPzhD3/A3r17UVZWZnGfu+++2+Jb9OTJk2EwGHD+/HkAwLZt29DQ0ID777/f4n5/+ctfbHgmTQ4fPoy8vDzce++9FuPeixcvhl6vt9h3w4YNiI2NxZAhQ1BQUCD/TJ8+HQA61F3e3NatW1FSUoKFCxdaHFOj0WD8+PEWx3R3d5d/r6mpQUFBAa644goAwO+//97px+6Ie++91+L/mzZtgtFoxPz58y3aGxoaioEDB7b7N1iwYAHy8vIshvk2btwIo9GIBQsWyNt64lw3bNiAyZMnw8/Pz+JcZsyYAYPBgN27dwMw5adUVlZi69atXXqcG2+8EREREfL/L7/8cowfPx6bN28GABQVFWH79u2YP38+ysvL5XYUFhZi1qxZOHPmDC5cuNDq8bvyurzrrrug0Wjk/3fmeWjNV199hZEjR+Kmm25qcZv0+tdoNPJrzGg0oqioCA0NDRg3blyXr2l4eLjFY/r4+OD222/H0aNHkZOT0+Y5A5bPs/r6ehQWFmLAgAHw9fXtcps2b96M0NBQLFy4UN7m6uqKhx56CBUVFS2GWRcsWAA/Pz/5/5MnTwYAnDt3rkuPTxxmonbk5+ejqqoKgwcPbnFbbGwsjEYjMjMzMWzYMHl7nz59LPaTXrTFxcUAIAc1zWf3+Pv7W7zAbUV6/IEDB1psd3V1tfhgAExd2snJyQgKCrJ6LCnBtzPOnDkDAHJA1JyPj4/8e1FREVasWIG1a9e2eKzS0tJOP3ZH9OvXz+L/Z86cgRCixd9L0l63+DXXXAO9Xo9169YhPj4egGmIadSoURg0aJC8X0+c65kzZ3Ds2LF2r+f999+P9evXY/bs2YiIiMDMmTMxf/58XHPNNR16HGt/q0GDBmH9+vUAgLNnz0IIgeeeew7PPfdcq20xD4jMdeV1ae26Ah17HlqTmpqKW265pc19AOCTTz7BG2+8gZSUFNTX17fano4aMGBAiyFH6XmUnp6O0NDQNh+juroaL7/8MlavXo0LFy7IQ6lA159n58+fx8CBA1tMVJCGpaT3HEl775HUeQxmqNs1/yYkMX/TsIXWcioMBkOXj2k0GhEXF4d//OMfVm83z6/pzDEBU76C+RuvxMWl6WU5f/587N+/H0888QRGjRoFLy8vGI1GXHPNNS0SVq3pyt/E/Jur1F6VSoUff/zR6rVtL2dJp9PhxhtvxNdff413330Xubm52LdvH1566SWL/S71XK1pfp5GoxFXX301nnzySav7Sx+KwcHBSEhIwJYtW/Djjz/ixx9/xOrVq3H77be3SOrsCul8/vrXv2LWrFlW9+nuqfzWrivQsedhV33++edYvHgxbrzxRjzxxBMIDg6GRqPByy+/jNTU1Es+fnuanzNg6gFevXo1HnnkEUyYMEEuRHjbbbd1+XnWWfZ6j3RmDGaoTUFBQfDw8MCpU6da3JaSkgK1Wt3pD3RpdsbZs2ctvjkVFhZ26JtJax/Q0reb5oXjmn8rkh7/zJkzFt9K6+vrkZaWhpEjR8rb+vfvj8TERMTHx3fbFNH+/fsDMH1gzpgxo9X9iouLsW3bNqxYsQLPP/+8vF36Rm3uUv8m7bVXCIF+/fpZ9KR0xoIFC/DJJ59g27ZtSE5OhhDCYoipM+dqjZ+fX4tzrKurw8WLF1ucS0VFRZt/d4lWq8X111+P66+/HkajEffffz8++OADPPfcc+0GGtbaffr0aTmhVuoBdHV17VBbmuuO12VHn4dt3d/aDC1zGzduRExMDDZt2mTxHJVm+XWF1KtlfrzTp08DQJsJy+ZtWrRoEd544w15W01NTYvnT2de79HR0Th27BiMRqNF70xKSop8O9kWc2aoTRqNBjNnzsS3335rMXUxNzcXX3zxBSZNmtRud3Rz8fHxcHFxwXvvvWex/V//+leH7u/p6Wm1O1h6c5byHgDTN/PmVXfHjRuHoKAgvP/++6irq5O3r1mzpsUb2vz583HhwgX85z//afF41dXVXapdMWvWLPj4+OCll16y6HaXSDOQpG9vzb+tWat+7OnpCaBl0OLj44PAwECLvwkAvPvuux1u78033wyNRoMVK1a0aIsQokNTp2fMmAF/f3+sW7cO69atw+WXX24RyHbmXK3p379/i3P88MMPW/TMzJ8/HwcOHMCWLVtaHKOkpAQNDQ0A0OKc1Gq1PBOtvSn5APDNN99Y5Lz89ttvOHjwIGbPng3AFEBMnToVH3zwQYuAC0C7s9C643XZ0edha2655RYkJibi66+/bnGbdB2tXdeDBw/iwIEDbR67LdnZ2RaPWVZWhk8//RSjRo2y2sPUnEajafE8e+edd1o8V1p7TVlz7bXXIicnx2LGZkNDA9555x14eXm1mMlH3Y89MwQA+Pjjj/HTTz+12P7www/jxRdfxNatWzFp0iTcf//9cHFxwQcffIDa2lqsWrWq048VEhKChx9+GG+88QZuuOEGXHPNNUhMTMSPP/6IwMDAdr8RjR07FuvWrcNjjz2Gyy67DF5eXrj++usxbNgwXHHFFVi6dCmKiorg7++PtWvXyh9QEldXV7z44ou45557MH36dCxYsABpaWlYvXp1i5yZP/3pT1i/fj3uvfde7NixA1deeSUMBgNSUlKwfv16bNmypdOl5X18fPDee+/hT3/6E8aMGYPbbrsNQUFByMjIwA8//IArr7wS//rXv+Dj44MpU6Zg1apVqK+vR0REBH7++WekpaVZ/ZsApqnrt912G1xdXXH99dfD09MTS5YswSuvvIIlS5Zg3Lhx2L17t/xNtiP69++PF198EUuXLkV6ejpuvPFGeHt7Iy0tDV9//TXuvvtu/PWvf23zGK6urrj55puxdu1aVFZWtli7qDPnas2SJUtw77334pZbbsHVV1+NxMREbNmyRZ6mLnniiSfw3Xff4brrrsPixYsxduxYVFZW4vjx49i4cSPS09MRGBiIJUuWoKioCNOnT0dkZCTOnz+Pd955B6NGjWp1eq65AQMGYNKkSbjvvvtQW1uLt956CwEBARbDW//+978xadIkxMXF4a677kJMTAxyc3Nx4MABZGVlITExsc3HuNTXZUefh6154oknsHHjRsybNw933HEHxo4di6KiInz33Xd4//33MXLkSFx33XXYtGkTbrrpJsyZMwdpaWl4//33MXToUFRUVLTbRmsGDRqEO++8E4cOHUJISAg+/vhj5ObmYvXq1R26/3XXXYfPPvsMer0eQ4cOxYEDB/DLL78gICDAYr9Ro0ZBo9Hg1VdfRWlpKXQ6HaZPn47g4OAWx7z77rvxwQcfYPHixThy5Aj69u2LjRs3Yt++fXjrrbfg7e3dpXOlTrDDDCpSEGkKY2s/mZmZQgghfv/9dzFr1izh5eUlPDw8xLRp08T+/futHqv5NG9r06sbGhrEc889J0JDQ4W7u7uYPn26SE5OFgEBAeLee+9t874VFRXiD3/4g/D19RUALKZpp6amihkzZgidTidCQkLEM888I7Zu3Wp1iuW7774r+vXrJ3Q6nRg3bpzYvXt3iymXQpimWL766qti2LBhQqfTCT8/PzF27FixYsUKUVpa2ubf19rUbPNzmzVrltDr9cLNzU30799fLF68WBw+fFjeJysrS9x0003C19dX6PV6MW/ePJGdnW11KvLf//53ERERIdRqtcWU1qqqKnHnnXcKvV4vvL29xfz580VeXl6rU7OttVUIIb766isxadIk4enpKTw9PcWQIUPEAw88IE6dOtXm30AiXQeVSiU/r8x19FytTdk1GAziqaeeEoGBgcLDw0PMmjVLnD17tsXUbCFMU5KXLl0qBgwYILRarQgMDBQTJ04Ur7/+uqirqxNCCLFx40Yxc+ZMERwcLLRarejTp4+45557xMWLF9s8R/Np1G+88YaIiooSOp1OTJ48WSQmJrbYPzU1Vdx+++0iNDRUuLq6ioiICHHdddeJjRs3Wj1mc5fyupR05HnYmsLCQvHggw+KiIgIodVqRWRkpFi0aJEoKCgQQpimaL/00ksiOjpa6HQ6MXr0aPH999+LRYsWtSiv0JHrHB0dLebMmSO2bNkiRowYIXQ6nRgyZIjYsGFDh8+5uLhY/PnPfxaBgYHCy8tLzJo1S6SkpFh9rvznP/8RMTExQqPRWLyHWHufyM3NlY+r1WpFXFxci5IIbV1La69p6jiVEMw4ImUoKSmBn58fXnzxRTz77LP2bg4RETkI5syQXVRXV7fYJuVHNC8TTkRE1BbmzJBdrFu3DmvWrJHLs+/duxdffvklZs6ciSuvvNLezSMiIgfCYIbsYsSIEXBxccGqVatQVlYmJwW/+OKL9m4aERE5GObMEBERkUNjzgwRERE5NAYzRERE5NB6Rc6M0WhEdnY2vL29u60kPREREdmWEALl5eUIDw9vsZCnuV4RzGRnZ3dpQUAiIiKyv8zMTERGRrZ6e68IZqRS0pmZmZ1eR4iIiIjso6ysDFFRUe0uCdErghlpaMnHx4fBDBERkYNpL0WECcBERETk0BjMEBERkUNjMENEREQOjcEMEREROTQGM0REROTQGMwQERGRQ2MwQ0RERA6NwQwRERE5NAYzRERE5NAYzBAREZFDYzBDREREDo3BDBERETk0BjNOav3hTOw7W2DvZhAREdlcr1g1u7c5mV2GJzceAwCkvzLHzq0hIiKyLfbMOKGcsmp7N4GIiKjHMJhxQiqVyt5NICIi6jEMZpyQmsEMERH1IgxmnJDaLJYRQtivIURERD2AwYwTMu+ZMTKWISIiJ8dgxgmZjzI1GI32awgREVEPYDDjhMx7ZhoM7JohIiLnxmDGCVkEMxxnIiIiJ8dgxgmZDzMZGMwQEZGTYzDj5JgzQ0REzo7BjBMymvXGsGeGiIicHYMZJ2QevzABmIiInB2DGSdkXiiPPTNEROTsGMw4IYueGQYzRETk5BjMOCEje2aIiKgXYTDjhMyDmXoDZzMREZFzYzDjhMzXlmTPDBEROTsGM07IPIBhzgwRETk7BjNOiDkzRETUmzCYcUKWs5mYM0NERM6NwYwTYp0ZIiLqTRjMOCHWmSEiot6EwYwTssiZ4XIGRETk5BjMOCHzYIY9M0RE5OwYzDghwQRgIiLqRRjMOCFOzSYiot6EwYwTsiiax5wZIiJycgxmnBCXMyAiot6EwYwTYgIwERH1Ji72bgB1H4NRoLKuwaLOjIEJwERE5OTYM+NEbn5vP0Ys/xl55TXyNvbMEBGRs2Mw40QSM0sAADtS8uRtzJkhIiJnx2DGSdTUG+Tffdxd5d/rOZuJiKw4mV2GKat2YM2+NHs3heiSMZhxEoWVdfLvdQ1NeTLMmSEia6795x5kFFVh+f9O2rspRJeMwYyTKCivlX+/UFIt/86cGSJqjsPP5GwYzDiJgoqmYOZiaVMCMN+0iKi52gZD+zsRORAGM07CPJixqADMYIaImqmptxx+FoLvE+TYGMw4iXyzYSZz7JkhoubMJwwAnChAjo/BjJMoqKizup1rMxFRc7UNlj0zdQZOFCDHxmDGSeRXtNYzwzcpIrLUvGemroHvE+TYGMw4iYJWhpmYM0NEzTUPZpgQTI6OwYyTaK1nhsNMRNRc8wRg9syQo2Mw4ySknpkIX3eL7eyZIaLmahqa98wwmCHHxmDGCdQ2GFBW0wAAGBGpt7iNOTNE1Fwtc2bIyTCYcQKFjTOZXDUqxIb5WNzGnhkiaq55TwxzZsjRMZhxAlLBvABPXYthJtaZIaLmWiYAs2eGHBuDGScgFcwL9NYiwo85M0TUNiYAk7NhMOMEpJ6ZIC/2zBBR+9gzQ87GZsHMypUrMXHiRHh4eMDX19fqPg899BDGjh0LnU6HUaNGWd1HCIHXX38dgwYNgk6nQ0REBFauXGmrZjskqfpvoJcOIT5uUKmabmPPDBE1x54ZcjYutjpwXV0d5s2bhwkTJuCjjz5qdb877rgDBw8exLFjx6ze/vDDD+Pnn3/G66+/jri4OBQVFaGoqMhWzXZITcNMOmhd1Aj21iG3zLSNs5mIqDlOzSZnY7NgZsWKFQCANWvWtLrPP//5TwBAfn6+1WAmOTkZ7733Hk6cOIHBgwcDAPr169f9jXVw0jBToJcOABDu6y4HM1xAjoiaq2XPDDkZRefM/O9//0NMTAy+//579OvXD3379sWSJUva7Zmpra1FWVmZxY8zk3tmvLQATMGMhDkzRNRc856ZOk7NJgen6GDm3LlzOH/+PDZs2IBPP/0Ua9aswZEjR3Drrbe2eb+XX34Zer1e/omKiuqhFtuHnADsbeqZmTwgUL6NOTNE1BwTgMnZdCqYefrpp6FSqdr8SUlJ6bbGGY1G1NbW4tNPP8XkyZMxdepUfPTRR9ixYwdOnTrV6v2WLl2K0tJS+SczM7Pb2qREUgJwUOMw022X98H7/28MAObMEFFLHGYiZ9OpnJnHH38cixcvbnOfmJiYS2mPhbCwMLi4uGDQoEHyttjYWABARkaGnEfTnE6ng06n67Z2KFldgxGl1fUAmnJmAMBL5wqAC00SUUvsmSFn06lgJigoCEFBQbZqSwtXXnklGhoakJqaiv79+wMATp8+DQCIjo7usXYoWWGlaYjJRa2C3t1V3q5Rm+ZnM2eGiJprkTNjYDBDjs1ms5kyMjJQVFSEjIwMGAwGJCQkAAAGDBgALy8vAMDZs2dRUVGBnJwcVFdXy/sMHToUWq0WM2bMwJgxY3DHHXfgrbfegtFoxAMPPICrr77aoremN5OSfwO8tFCrmwrMuGgYzBCRdc3rzDRfeJLI0dgsmHn++efxySefyP8fPXo0AGDHjh2YOnUqAGDJkiXYtWtXi33S0tLQt29fqNVq/O9//8Nf/vIXTJkyBZ6enpg9ezbeeOMNWzXb4TRP/pVIPTNMACai5povLMmeGXJ0Ngtm1qxZ02aNGQDYuXNnu8cJDw/HV1991T2NckIF5U3Vf825SMEM36SIqJkWPTPMmSEHp+ip2dS+/GYF8yTsmSGi1jABmJwNgxkH17z6r8RFbbq0zJkhoua4NhM5GwYzDq559V+JlADMnhkiaq55wi97ZsjRMZhxcK0lALtwajYRtUKamv3MtUMAcDkDcnwMZhxc8+q/kqacGX7jIqImBqOQF6D1cTPVpuIwEzk6BjMOTs6ZadEzw5wZImrJfFq2VGiTw0zk6BjMOLC6BiNKqlouZQBwNhMRWWee/OvNnhlyEgxmHJi0lIFGrYKv2VIGQFPOjBCAkQENETWSpmW7alRw15o+AtgzQ46OwYwDkwrmBXhaLmUAABpN0//rmTdDRI2kYMbNRQOtRgOAPTPk+BjMOLDWZjIBTT0zAPNmiKiJNMykc9VA5yr1zHA2Ezk2BjMOrLXqv0BTzgzAvBkiaiJNy3ZzVUOrMX0EsGeGHB2DGQfWWvVfAHBVN11ag4HBDBGZ1Db2zLiZ9cxwoUlydAxmHJhc/ddb2+I2tVoFVWPnDHtmiEhirWem3iA4UYAcGoMZB9ZawTwJqwATUXO15gnALk0fAeydIUfGYMaBFZS3ngAMsAowEbXUlACshs5FI2+vref7BDkuBjMOrK2cGYBVgImoJfOp2a5mJRxqDZzRRI6LwYwDay+YkXpm6pkATESN5GDGVQOVSgWdC2c0keNjMOOg6g1GFMtLGbRMAAaYM0NELUnVfqWZTFLeDKsAkyNjMOOgChuTfzVqFfw8rAczzJkhouZqzKZmA2DPDDkFBjMOShpisraUgYQ9M0TUnDw120UKZkz/smeGHBmDGQfVVvVfiUtjDQnWmSEiiZQz03yYiT0z5MgYzDioArlgXhvBDHtmiKgZeZjJxXKYieszkSNjMOOgmnpmrOfLAGY5M5zNRESN5KJ5zXpmqusYzJDjYjDjoArK267+CzQFM+yZISJJ03IGpp6ZfoGeAIDfM0rs1SSiS8ZgxkFJCcCtVf8FABcNZzMRkaWmhSZNb//ThwQDALan5NqtTUSXisGMg2qvYB4AaBorAHOYiYgkzXtmpg4KhkatwuncCmQWVdmzaURdxmDGQXUkmHGR68wwmCEiE3ltpsYEYL2HK8ZF+wEAtiWzd4YcE4MZB5Uvz2ZqPwGYOTNEJGk+NRsAZsSGAAC2peTZpU1El4rBjAOyXMqgIz0zzJkhIhPzhSYl02NNeTO/nitERW2DXdpFdCkYzDigosr2lzIAmormsWeGiCQ1zRKAAaB/kBf6BXqi3iCw53S+vZpG1GUMZhyQNMTk76mVh5KsYc4Mkf0IIRRZiE5atkBKAJZIs5o41ESOiMGMA+pI8i/AnBkie7rv898R+9xPuO3DA/h4bxqyipUxU6ipaJ5lMBPfONS0IyWP7xnkcFzs3QDqPDn5t43qvwB7ZojsJb2gEj8l5QAAfj1XhF/PFeGF709iWLgPZg4NxazhIRgc4g2VqvWeVVtpmppt+V32sr7+8HZzQWFlHRKzSjCmj1+Pt42oqxjMOKCCivar/wJmPTMGJgAT9aRNv2cBAC7v649Zw0OxJSkHh9OLkJRdhqTsMrz5y2n08ffArGEhmDksFGP6+LU5ZNxdDEaB+sa6UzoXy54ZV40aVw0KwvfHLmJbci6DGXIoDGYcUEeq/wLsmSGyB6NR4KvfLwAA/t+EaNwwMhx3TuqHwopabEvOw88nc7D7TAEyiqrwnz1p+M+eNAR6aTEjNgQzh4VgYv/AFkNA3UWayQS07JkBTENNpmAmD0/MGmKTNhDZAoMZB9TxnJnGCsAMZoh6zK9phbhQUg1vNxfMHBoibw/w0mH+ZVGYf1kUKmsbsPt0Pn4+mYttybkoqKjD2kOZWHsoE55aDaYODsbMYSGYNiQYPm6u3dY2i2DGpWXANHVQMNQqICWnHFnFVYj08+i2xyayJQYzDkgOZtoomAc09cwwmY+o52w8Yhpium5EeKs9LJ46F8yOC8PsuDDUG4w4eK4IW5Jy8PPJHOSW1eKH4xfxw/GLcNWocEVMAGYNC8XVQ0MQ4uN2SW2raZzJpNWoobYyrOXnqcXYaD8cSi/GjpQ8/GlC30t6PKKewmDGATUlALczzCQtNMm1mYh6RGVtA346YUr8vXVsZIfu46pRY9LAQEwaGIgVNwzD8QuljYFNLs7mVWDPmQLsOVOAv31zAqOifDFrWChmDQtBTJBXp9tXa6X6b3PTh4TgUHoxfklmMEOOg8GMA5ISgNsNZuSeGSYAE/WEzccvoqrOgH6BnhjTx7fT91erVRgZ5YuRUb548pohSM2vwM9Jufj5ZA6OZpQgIdP08+pPKRgQ7IWZQ0Mwa1goRkTqOzQzqqlgXus5OTNig/HqTyk4kFqIytoGeOr4MUHKx2epg2kwGFFc1TibqZ0EYObMEPWsrxpnMd06NrJbpl33D/LCfVO9cN/U/sgtq8HWk7nYkpSDA6mFOJtXgbN5FXh3ZypCfdwwc1gIZg4NxfgYf7hqrPe8tDYt29yAYC9E+bsjs6gae88WYNaw0Es+DyJbYzDjYIoq6yAEoFahzaUMgKZhJubMENleZlEVfj1XBJUKuGl0RLcfP8THDf/vimj8vyuiUVpdj52n8vBzUi52nMpDTlkNPj1wHp8eOA8fNxfEx4Zg5tAQXDU4CB7aprd5eZFJK8m/EpVKhfghIVizPx3bk/MYzJBDYDDjYPIrpKUMdO3WpdBwajZRj9nUOB37yv6BCPd1t+lj6d1dMXdUBOaOikBNvQH7Uwuw5UQufknORWFlHb4+egFfH70AnYsakwcGYuawUMQPCUatlXWZrImPDcaa/enYlpIHo1FYTRYmUhIGMw6mo9V/AbM6MyyaR2RTQgh5iOmWsd3fK9MWN1cNpg8JwfQhITAYBX7PKMaWEznYcjIHmUXV+CU5D78k50GtgjzV2tq0bHPj+wXAU6tBQUUtjl8oxcgo3x44E6KuYzDjYOTqv+3kywDsmSHqKYfSi5FRVAUvnYtdh2U0ahUu6+uPy/r649k5sUjJKZcTiJOyy5BRZFofqr2kXq2LGlMGBeHHEznYlpzLYIYUj8GMg5Gr/7YzkwlgnRminrLxSCYA4Nq4UIscFXtSqVSIDfNBbJgPHp4xEJlFVdh6Mhe/ZxTjj+Oj271/fGyIKZhJycNjMwf3QIuJuk4ZrzrqsAJpmKlDPTOczURka1V1Dfjh2EUAwK1jo+zcmtZF+Xvgjkn9cAf6dWj/qYODoFIBSdlluFhajTC9bfOAiC5F21lgpDhNSxm0nzPjytlMRDa3JSkHlXUG9PH3wGV9nWdxxkAvHUY3Di9tT8mzb2OI2sFgxsHkd3BdJoA5M0Q9QVq+4JYx3VNbRkniY01rS21LZjBDysZgxsEUlHes+i/ACsBEtnahpBr7UwsBADeP6dlZTD0hPjYYALDvbAGq6wzt7E1kPwxmHIycANyZnBmuzURkE1//ngUhgCti/BHl73wrTA8O8UaErztqG4zYn1pg7+YQtYrBjANpMBhRVNWVnhkGM0TdzVRbxlQoT8mJv5dCpVLJvTO/cKiJFIzBjAMpqmpaysDfs/0EYClnpp7BDFG3+z2jGGkFlfDQajB7uPOW/J8+xBTMbE/JhRB8LyFlYjDjQKTqv/6e2naXMgDM12ZizgxRd9t4xNQrM3t4mFOvLH1FTAA8tBrkltUiKbvM3s0hsorBjAORqv92ZIgJMJvNxJwZom5VU2/A94nZAHp++YKe5uaqwaQBgQA4q4mUi8GMA5EK5nUk+RdgzgyRrfx8MhfltQ2I8HXHFf0C7N0cm5shTdFOybVzS4isYzDjQAo6UWMGAFxYAZjIJppqy0T0ihWlpw4JAgAcyypFXlmNnVtD1BKDGQfSmeq/AKBpowJwXYMR2SXV3dc4ol4ip7QGe8/kAwBuGRtp59b0jGBvN3mxSVYDJiViMONApATgjvfMWK8AvPdMAca/9AsmvrIdu0/nd28jiZzc10cvwCiAy/r6ITrA097N6THxjbOatjGYIQViMONAupoA3Hw207rDmSiuqgcApBVUdmMLiZybqbaMaYjp1l7SKyOR6s3sPVOAmnpWAyZlYTDjQDpT/RdoPWemuLJO/p35NEQdl5hVirN5FXBzVePauDB7N6dHDQ3zQZjeDdX1Bhw4V2jv5hBZYDDjQDqbANza1OziqqZghjVoiDpu45FMAMA1w0Lh7eZq59b0LJVKJRfQ25bMWU2kLAxmHITBKFDU2KMS6N2xBODWpmaXNA4xAUA9a9AQdUhNvQH/S7wIoPck/jYnDTVtT85jNWBSFAYzDqKwshZGAahUgL9HB2czyQnAlr0vlj0zfEMi6ohtyXkora5HmN4NE/sH2rs5djGxfyDcXNXILq1B8sVyezeHSMZgxkEUlJsCEH8PLVw0HbtsLlamZtc2GFBV15S8x5wZoo6REn9vGh3RoeVEnJF5NeDtLKBHCsJgxkF0NvkXsJ4AbD7EBJhW4iaituWV12DX6d5VW6Y18Y3VgLmKNikJgxkH0dnkX8AsZ8YsL8Z8iAngMBNRR3x7NBsGo8CYPr7oH+Rl7+bYlZQEnJhVIte+IrI3BjMOorPVfwHznBmzYKayWc8MgxmiNgkhmpYv6OW9MgAQ4uOGuAg9hAB2nGLvDCkDgxkH0dnqv4D1nJnmPTMcZiJqW1J2GU7llkProsZ1I8Lt3RxFkHpntnOoiRTCZsHMypUrMXHiRHh4eMDX19fqPg899BDGjh0LnU6HUaNGWd1ny5YtuOKKK+Dt7Y2goCDccsstSE9Pt1WzFUuu/tuJnBmpZ6bebDZTi2CGPTNEbZJ6ZWYODYHevXfVlmmNtIr2njP5qG1gNWCyP5sFM3V1dZg3bx7uu+++Nve74447sGDBAqu3paWlYe7cuZg+fToSEhKwZcsWFBQU4Oabb7ZFkxVNTgDuVM6M6fIKARgbg5bmCcDMmSFqXV2DEd8mXADQ+5YvaMuwcB8Ee+tQWWfAwXNF9m4OEVxsdeAVK1YAANasWdPqPv/85z8BAPn5+Th27FiL248cOQKDwYAXX3wR6sYP5r/+9a+YO3cu6uvr4erae74lycNMXeiZAUw9MFq1ymIpA2k7EVm3PSUPxVX1CPbWYfLAIHs3RzHUahXiY4Px5W+Z2JaciymD+Lch+1J0zszYsWOhVquxevVqGAwGlJaW4rPPPsOMGTPaDGRqa2tRVlZm8ePomhaZ7HgCsItZMCP1wEgLTErHYc4MUevk2jJjem9tmdZMH2IaatqWwmrAZH+KDmb69euHn3/+Gc888wx0Oh18fX2RlZWF9evXt3m/l19+GXq9Xv6JiorqoRbbhmkpgy4MM2nMe2ZMQUtJleXK2+yZIbKusKIWO1JMCa63juEQU3OTBgRC56JGVnE1TudW2Ls51Mt1Kph5+umnoVKp2vxJSUnptsbl5OTgrrvuwqJFi3Do0CHs2rULWq0Wt956a5vfBJYuXYrS0lL5JzMzs9vaZA9FlXVNSxl4dqZnpunyNvXMWAYzzJkhsu7bhGw0GAVGRuoxMMTb3s1RHHetBhP7BwAAtrEaMNlZp3JmHn/8cSxevLjNfWJiYi6lPRb+/e9/Q6/XY9WqVfK2zz//HFFRUTh48CCuuOIKq/fT6XTQ6Treg6F0UvJvZ5YyAADzXvGGZgnAUiVhLjRJZB1ry7QvPjYEO07lY1tyHu6fOsDezaFerFPBTFBQEIKCei7Rq6qqSk78lWg0GgCA0dh7cj26Uv0XAFQqFVzUKjQYhdwDU9TYMyMFM4Ze9Hck6qiT2WU4ebEMWo0a17O2TKukejO/ZxSjqLKuUz3HRN3JZjkzGRkZSEhIQEZGBgwGAxISEpCQkICKiqax1bNnzyIhIQE5OTmorq6W96mrM33gzpkzB4cOHcILL7yAM2fO4Pfff8ef//xnREdHY/To0bZquuI0zWTq/BuFeRVgg1GgtLpZAjCHmYhakBJ/42OD4ccP6FaF+7pjaJiPqRpwCgvokf3YbGr2888/j08++UT+vxR87NixA1OnTgUALFmyBLt27WqxT1paGvr27Yvp06fjiy++wKpVq7Bq1Sp4eHhgwoQJ+Omnn+Du7m6rpitOV3tmANOMplqYZi2VVddDSjWSE4A5zERkod5gxDdHWVumo+Jjg3HyYhm2p+RxSI7sxmbBzJo1a9qsMQMAO3fubPc4t912G2677bbuaZSDapqW3flgxrxnRkr+9dK5wN3VNFzHBGAiS7tO5aOwsg6BXjrWT+mA+NgQvLP9LHadzkddgxFaF0VPkiUnxWedAyhoHGYK6kTBPImUMGwwCrnGjK+Hq1mQw5wZInNS4u+No8Lh2omE+95qRIQegV5aVNQ24FA6qwGTffCV6gDyL2GYSQ5aDEKuMePnoZVr0DBnhqhJcWWdPM2YQyYdo1arMG2wKRH4l2RO0Sb7YDDjAJpWzO58IqKrumnlbPOeGakGDXNmiJp8l5iNeoPAsHAfxIb52Ls5DiO+ceHJbcmsBkz2wWDGAVxSzoymaTjJomfGLMghIhNpFhMTfztn8sBAaDVqZBRVITWf1YCp5zGYUTiLpQy6kjOjbsqZKaqUgpmmnJl65swQAQBO55bjWFYpXNQq3DCStWU6w1PngiukasDJnKJNPY/BjMIVV3VtKQOJ5Wwm0zCTn6fWIjGYiICvGhN/pw8JRkAXekF7u/jGAnoMZsgeGMwonFRjxs9D26WZFebDSdaGmZgzQ2Sqw7SpsbYME3+7RqoGfPh8kfxeQ9RTGMwo3KUk/wJNPTP1BqNcZ4ZTs4ks7TlbgPzyWvh7auWZOdQ5Uf4eGBziDaMAdp7Kt3dzqJdhMKNwl1L9F2jeM9M4zGTWy8NhJqKm2jI3jAxn0bdLEB/bONTEpQ2oh/FVq3AF5V2fyQRYrwDs56G12E7Um5VW1WNrkqk+CmcxXRopmNl5Kg/1Bvb6Us9hMKNwUs9MV2YyAbCoJ2NZZ4Y5M0QA8L9j2agzGDEk1BvDwllb5lKMivKDv6cW5TUNOJxebO/mUC/CYEbhLqX6LwC50m9FbT3qGkzflPw8tcyZIWokDTHdOjYSKpXKzq1xbBq1ClMHm9az2sZqwNSDGMwoXHclAEvHcdWo4KnVMGeGCMDZvAokZJZAo1Zh7qgIezfHKcxorAa8nXkz1IMYzCicXP23y8NMKovj+HpooVKpzGY5CZYfp15Lqvg7dVBQl4dyydLkgYFw1ahwrqAS51gNmHoIgxmFk3NmupwAbLrE0nCVv4eph0cKcgCAnTPUGxmMAl//bqotw8Tf7uPt5orx/UzVgNk7Qz2FwYyCGc2WIOh6AnBjz0zjMJOvh6tpu6YpmGHeDPVG+84WIKesBnp3V0yPZW2Z7iQV0OMq2tRTGMwoWHFVnZzT0pWlDICmhSbzzSoJA02znADOaKLeSRpimjsqHDoXjZ1b41ykKdqH0otRWl1v59ZQb8BgRsGaAhDXLi1lALTsmfHzNPXMaNTmPTMMZqh3Kaupx08ncgAAt4zhEFN3iw7wxIBgLxiMArtOsxow2R6DGQW71IJ5QFPQUlbTAMCUAAxY5sxwRhP1NpuPXURtgxEDg70wIlJv7+Y4Jal3ZjuHmqgHMJhRsEtdygCwDFoAUy8PAKjVKkg3NbBSJ/UyUm2ZW1hbxmbih5imaO84lc/3GLI5BjMKdqnVfwHApdnwlNQzA5hVB2bPDPUi6QWVOHy+GGoVcNNo1paxlTF9fOHr4YrS6nr8nlFi7+aQk2Mwo2CXWv0XsNYzYxbMaJoWoSTqLaTE38kDgxDi42bn1jgvF40aUwexGjD1DAYzCiZX//Xu2kwmwDLRF2gaZjK/jQvCUW9hNApsYm2ZHhPfWA2Yq2iTrTGYUTC5+m939sx4alvcxp4Z6i1+PVeICyXV8HZzwdVDQ+zdHKc3ZVAQXNQqnM2rwPnCSns3h5wYgxkFk6ZTd7X6L9BUAVhiOczEnBnqXaTE3+tHhsPNlbVlbE3v7orL+voDALYls3eGbIfBjIJ1SwKwWc+MSmV6c2l+G4vmUW9QUduAHxtry3CIqedIU7S3pTBvhmyHwYxCGY0ChZXdV2cGAHzcXC3+L/3O5QyoN9h8/CKq6w2ICfTE6Chfezen15DyZg6eK0J5DasBk20wmFEo86UMAry6ngBs3jNjnvwLQK4qzJwZ6g2+Ym0Zu+gX6ImYQE80GAV2ny6wd3PISTGYUSgp+df3EpYyACzrzJjXmAHMe2YYzJBzyyyqwsG0IqhUwM1jWFump3GoiWyNwYxCdUf1X6DtnhnmzFBvIdWWmTQgEGF6dzu3pveZ3lgNeOepfPYEk00wmFEoOfn3EoMZnWvTJTZP/gWaiuYxZ4acmdEo5GCGi0rax7i+fvBxc0FRZR0SMovt3RzqZt8lZsvDuPbCYEahmgrmXVowIyXfAYC/p+WxpGnb/KZEzuxQehEyi6rhpXPBrGGh9m5Or+SqUeOqwY1DTZyi7VSq6hrw0JdH8fiGRJRU1dmtHQxmFKppKYOuJ/8CQISvO96+bRRGROox/zLLb6UucgVgBjPkvKTaMnPiwuCuZW0Ze5kRy2DGGV0orpZ/L66y32w1F7s9MrWpoPzSp2VL5o6KwNxRLZMeWQGYnF1VXQM2H78IALh1HIeY7OmqQUHQqFU4lVuOzKIqRPl72LtJ1A2yzIIZey6Nw54ZhequnJm2MGeGnN1PJ3JQWWdAdIAHxkX72bs5vZqvhxZjG6/Bdq7V5DQyi6vk343Cfl+MGcwoVHdU/22PlDPD2UzkrMwTf1lbxv7ih0hTtBnMOIuMwqZgxp69/AxmFKq7pma3xZXDTOTELpRUY39qIQDgptGsLaME0oSEX1MLUVHbYOfWUHf47940+Xd7dvIzmFEgo1E0rZjtfWkJwG1h0TxyZl//ngUhgAkxAczPUIj+QZ6IDvBAncGIvWdYDdjZGDjMROZKquubljLwZM4MUWcJIeRZTLdwUUnFUKlUiG8soLctmdWAnYFZXVa79vJzNpMCSUNMendXaF1sF2+6MGeGnNSR88VIL6yCh1aD2cNZW0ZJ4mOD8fG+NOw4lQejUUCtZi6To9l9Oh8PrT2K6joDzOMXJgCThYJy2yf/ApyaTc5LSvy9Ni4Mnjp+Z1OSy/r6w1vngoKKOiRmldi7OdQFb2w9jZKqetQ2WPbqG5kATOa6q2Bee6ScmXoOM5ETqak34PtEU20ZLl+gPFoXNaYMCgLAKdqOKCGzBImZJdBq1PjpkcnY9/R0+TZ7Xk8GMwokL2Vgw5lMQNOK2gYOM5ET2ZKUg/LaBkT6uWN8P397N4eskFbR/oXVgB3OpwfSAQDXjQjDkFAfRPg2LdyalF1mp1YxmFEkeSaTrYMZzmYiJyQl/t48JpL5GAo1dXAwVCog+WIZskuq278DKcZvaUUAgFutJNbbs5QTgxkF6omCeYD51GwOM5FzyCmtwb6zpim/t4xhbRml8vfUYkwfUzVgFtBzLEWVpi/bEX7u7ezZsxjMKFBPLGUAAK4a9syQc9l0NAtGAVze1x/RAZ72bg61QRpq2s4p2g6jpt6AqjoDAMDPs2VOpz2rbDOYUSC5+q8NC+YBTcsZMGeGnIEQAl81DjFZ6wInZZHqzexLLURVHasBO4LCxl4ZV40K3lZmCdpzVJfBjAL1WAIwc2bIiSRkliA1vxJurmrMjmNtGaUbFOKFSD931DUYse9sob2bQx1Q1JjP6e+ptdoLo2bPDEmMRoHCnkoAZgVgciJSbZnZw8Pg7eZq59ZQe0zVgBsXnuRQk0MoqpKCGeufTfZMt2cwozCl1fVyT0mAjevMsGgeOYuaegO+S8gGwNoyjkRaeHJ7Sp5dC65Rx1Q1Lg7qpdNYvd3DjgUqGcwojPlSBjoX60+Y7iLlzNQzZ4Yc3LbkPJTVNCBc74YJ/QPs3RzqoPEx/vDUapBXXosT2aX2bg61Q/qkUDXrg3nl5jjEhvngmWuH9HyjGjGYUZieqv4LsGeGnMfGI5kAgJvGRMglB0j5dC4aTB5oqga8jQX0HEezl9htl/fBjw9PRpjeftO1GcwoTE8l/wJNOTMZRVWoqTfY/PGIbCGvrAa7TucD4BCTI5reOEV7WwrzZpTOjutItovBjMLI1X9tXDAPaOqZOXK+GLd/9JvNH4/IFr5JuACjAMZG+yEmyMvezaFOmtZYDfjEhTLklNbYuznUBtE40KTEvk8GMwrTUwXzgKacGQD4Lb3I5o9H1N2EEPLyBeyVcUxB3jqMjPQFwIUnHYU9ly1oDYMZhSko75mlDICmYSYiR3XiQhlO51ZA56LGnBFh9m4OddEMqRowh5oUjcNM1GEFdkgAJnJUUuLvzGGh0Luztoyjmt5YDXjv2QLm7ylYa7OZlIDBjMI0zWbqiWEm5T0hiTqqtsGAbxNNtWW4fIFjiw3zRrjeDTX1RuxPLbB3c6gdHGaidhWU90z1XwBypWEiR7QjJQ8lVfUI8dFh0oBAezeHLoFKpZJnNf3CKdqKJRQ8zsRgRkGEECislBaZtH0wc1lff/l3rYZPBXIsG49cAADcNDqSvYxOQK4GnJyn6A9NYs8MtaO0ul6uxtsTOTNxkXr85/ZxAAAD3zzIgRRU1GLnKdM3+FvHRti5NdQdJsQEwN1Vg5yyGiRll9m7OdQG5sxQm74+avqm6ePmYvOlDCSjonwBmKoA89sQOYpvE7LRYBQYGeWLAcHe9m4OdQM3Vw0mDTQNF3KKtjIp+SOCwYxC1DUYseJ/JwEAZTUNPfa45jOauKoBOQqptsytY9gr40y4irayyUXzlNcxw2BGKQ6ft0/ROrVZMNNgNNqlDUSdkZRdiuSLZdBq1Lh+ZLi9m0PdaHpjMJOYVYq8clYDpo5jMKMQ0toyALD+ngk99rgWPTOMZcgBfNWY+DtjaDB8PWyfW0Y9J9jHDSMi9QBMs9VIWTjMRO3adcoUzLx92yhc3s+/nb27j4Y9M+RA6g1GfJtgCmZYW8Y5xTcW0OMq2sojBTMqBY4zMZhRgNyyGqTklEOlAiYPDOrRx9awZ4YcyM5T+SisrEOglw5Tevi1Qj0jvrHezJ4zrAasVMoLZWwczKxcuRITJ06Eh4cHfH19W9yemJiIhQsXIioqCu7u7oiNjcXbb7/dYr+dO3dizJgx0Ol0GDBgANasWWPLZve43Y1DTCMi9PD37Nluc42KPTPkOKTlC24aHQ4X1kZySsPCfRDio0N1vQG/niu0d3PIjIJHmWwbzNTV1WHevHm47777rN5+5MgRBAcH4/PPP0dSUhKeffZZLF26FP/617/kfdLS0jBnzhxMmzYNCQkJeOSRR7BkyRJs2bLFlk3vUVK+zFWDev6bplqtkjPTWWuGlKyosk6esnsLh5iclkqlktdq4lCTskjlOxQ4ygQXWx58xYoVANBqT8odd9xh8f+YmBgcOHAAmzZtwoMPPggAeP/999GvXz+88cYbAIDY2Fjs3bsXb775JmbNmmW7xvcQg1FgzxnTWiRXDbZPt7lGpUKDEDBwbjYp2HcJF1BvEBge4YMhoT72bg7Z0IzYYHz5Wwa2p+ThBSEUmaPRmynxaiiun7a0tBT+/k0JsAcOHMCMGTMs9pk1axYOHDjQ6jFqa2tRVlZm8aNUx7JKUFpdDx83F4yM9LVLG6S8GQYzpGRf/d6Y+DuGvTLObmL/QOhc1LhQUo2UnHJ7N4caKfkTQlHBzP79+7Fu3Trcfffd8racnByEhIRY7BcSEoKysjJUV1dbPc7LL78MvV4v/0RFRdm03ZdCGmKaNDDQbjkAUjDDlBlSqlM55Th+oRSuGhVuGMVCec7OXauRFw9lNWAFcabZTE8//TRUKlWbPykpKZ1uyIkTJzB37lwsW7YMM2fO7PT9zS1duhSlpaXyT2Zm5iUdz5bsmS8jkYIZJgCTUn31u6ni7/QhwT2eJE/20bSKNqsBK43yQpku5Mw8/vjjWLx4cZv7xMTEdOqYJ0+eRHx8PO6++2787W9/s7gtNDQUubmWT+bc3Fz4+PjA3d3d6vF0Oh10OtuvOn2pSqrqkJhZAgCYooBgxsgEYFKgBoMRmxqHmG7hEFOvET8kBM/iBBIyS1BQUYtAL+W/pzs7oeCBpk4HM0FBQQgK6r4P3qSkJEyfPh2LFi3CypUrW9w+YcIEbN682WLb1q1bMWFCz1XJtZW9ZwtgFMCgEC+E6a0HZj3BRe6ZUe4TlXqvPWcKUFBRiwBPLaY1lrsn5xeqd8OwcB8kZZdhR0oe5o1TbrpAb9FUNM++7bDGpkkaGRkZSEhIQEZGBgwGAxISEpCQkICKigoApqGladOmYebMmXjssceQk5ODnJwc5Oc3lfa/9957ce7cOTz55JNISUnBu+++i/Xr1+PRRx+1ZdN7hFT1155DTACgVjEBmJRLWlTyhlHhcGVtmV4lPtaUL8m8GaVRXjRj03eG559/HqNHj8ayZctQUVGB0aNHY/To0Th8+DAAYOPGjcjPz8fnn3+OsLAw+eeyyy6Tj9GvXz/88MMP2Lp1K0aOHIk33ngD//3vfx1+WrYQArvPSMGMfb9tal1MT4PKWlbbJGUprarH1pOmYWYuX9D7SKto7z6dj9oGvj/Zm5K/7to0mFmzZg2EEC1+pk6dCgBYvny51dvT09MtjjN16lQcPXoUtbW1SE1NbTdnxxGcyi1Hblkt3F01GNfXz65tGRLqDQBIyCy2azuImvvuWDbqDEYMCfXGsHC9vZtDPSwuQo8gbx0q6wz4La3I3s3p9XrtMBO1ThpiuiLGH26uGru2RVrY8rc0BjOkLF81DjGxV6Z3UqtVmD7Y1DvDasDKocBYhsGMvShhSrbksr6mYObw+SIYmTdDCnE2rwIJmSVwUatw42jWlumtpIUnt6XkyuX0yT6UPJuJwYwdVNY24HC6qRfkqsH2n50xPEIPd1cNSqrqcTa/wt7NIQLQVFtm6uAgTsvtxSYNDITWRY3MomqcyeP7kz1xmIks/HquEHUGI6L83dE3wMPezYGrRo3RfXwBgOPSpAgGo8Cm3znERICH1gUT+wcA4FCTUqgUONDEYMYOzIeYlFIWWhpqOpTOYIbsb+/ZAuSW1cLXw5W1ZUie1bSN1YDtSrmDTAxm7KIpmFHOm7SUBHyIPTOkAFLi79yR4dC52DdBnuxvemO9md8zilFUWWfn1vRijeNMCvkOboHBTA9LL6jE+cIquGpUmNDYdaoEo/v4wkWtQnZpDbKKq+zdHOrFymrqsSUpBwBwC4eYCECErzuGhHrDKICdpzjUZG8MZkgulDc22g9euk6vJmEzHloXDIsw1fH4w38O4tF1Cfh4bxoOpxehqq7Bzq2j3uSHYxdR22DEoBAvxEWwtgyZzGjsndnGasB2o+RhJuV8mvYSTUsYKGeISXLLmAgkZpYgo6gKGUVV+PqoaXE/tQoYGOyNuEg9RkTqERehR2yYj93r45BzkpYvuGVMpGJyysj+pscG4187zmL3qXzUNRjlyuXUc+TZTApMAGYw04NqGwzYn1oIQBn1ZZq7fUJfXBsXhuMXSnE8qxTHskpx/EIJcstqcSq3HKdyy+UPGhe1CoNCvE3BTaQeIyJ8MTjUm28wdEnSCipx5Hwx1CrgJtaWITMjI30R4KlFYWUdDqcXYeKAQHs3qfdSXizDYKYnHUkvRnW9AUHeOsSGedu7OVYFeukwbXAwppnVv8ktqzEFNxdKcTyrBMeySlFYWYeTF8tw8mIZ1h7KBABoNWoMCfNGXITUg+OLgSFeXByQOkxK/J0yKAjBPm52bg0piUatwrQhwdh4JAvbUvIYzNiBkosWMpjpQdIspikDlTMluyNCfNwQMtQNM4aaxqyFELhYWiP33Jj+LUVJVT2ONfbo/N9B0311LmoMDffBiAg94iJ9MSJSj/5BXtCoHef8qWcYWVuG2hEvBTPJufjbnFiHeh91BlIoo8S/OoOZHiRPyR6svCGmzlCpVAj3dUe4rzuuGR4KwBTgZBVXm4KZCyU4nmUaqiqvbcDRjBIczSgBcB4A4O6qwfAIH8RF+MrDVP0CPKFmgNOrHThXiOzSGvi4ucjJnkTmJg8KgqtGhfTCKpwrqET/IC97N4kUgsFMD8ktq0FKTjlUKmCyE3aPqlQqRPl7IMrfA3NGhAEwfdNOL6zE8QuN+TdZpTiRXYqqOgMOpRfjUHrTwpZeOhcMj/DBiEhfeZiqj78Hv3n1ItIQ0/Ujw5lcTlZ56VxwRUwA9pwpwLbkXAYzPaxpOQPlvS8zmOkhUq/MiEhf+Hlq7dyanqFWqxAT5IWYIC/MHWVK5jQYBc7lV8hDU8eySpCUXYaK2gb8eq4Iv55rKtqnd3dFXISUYGz6N8LXXZEvJLo0FbUN+PEEa8tQ++KHBDcGM3m4e0p/ezenV+EwEylqlWx70qhVGBjijYEh3vKHVoPBiDN5FY1JxqYhquSL5SitrsfeswXYe7ZAvr+/p9YswViPEZG+CPHRMcBxcJuPX0R1vQExQZ4YHeVr7+aQgsXHhmD5/07i8PlilFbVQ+/hau8mkQIwmOkBBqPA3jOmD+TeHsxY46JRIzbMB7FhPph/WRQAoK7BiNO55RZJxqdyylFUWYddp/Pl4BAAgrx1cs+NNIsqyJurLDsSacr/rWNZW4baFuXvgUEhXjidW4Gdp/PkXl+yPaHg5QwYzPSAxKwSlFbXw8fNBSMjWdG0I7QuagyP0GN4hB5AHwBATb0BKTnl8vTw4xdKcTq3HPnltdiWkmdRGTRM79bUg9OYh+PfS4b3HE1GYRV+SyuCirVlqIOmDwnB6dwKbEtmMGMPCoxlGMz0BKnq7+SBQXBhzZUuc3PVYFSUL0aZDUNU1xlw8mJTgvGxC6VIza/AxdIaXCytwc8nm1bZjfRzl3tuRkSaAiW9O7uo7e2rxunYkwYEIkzvbufWkCOYERuM93elYuepPDQYjHxfJQYzPYH5MrbjrtVgbLQ/xkb7y9sqahuQdKG0aRbVhVKkFVQiq7gaWcXV2Hw8R963b4AHRjTWv4mL0GNYhF5Ra2Y5O6NRyMEMa8tQR43u4wc/D1cUV9Xj8PliXBHT9UV7k7JL8cym43hg2gDMHBbaja10PpzN1IsVV9bhWFYJAGDyIOebkq1EXjoXjI8JwHizN7jS6nokXZCqGJsSjTOLqpFeWIX0wip8l5gNwDQW3D/IyyIHZ2iYHu5aThW2hd/Si5BVXA1vnQtmDuUHCXWMRq3CtMHB2HT0Aran5F1SMPPKjylIzCrFY+sT8dMjPoj08+jGljoX0TifSXmhDIMZm9t7tgBGAQwO8WYXuh3p3V0xcUCgRQn04so60zpUjVPEj2eVIru0BmfzKnA2rwKbzBbaHBTibZGDMyTUm7VQuoGU+DtnRBgDRuqU6bGmYOaX5Fw8c21sl46RfLEMexonZ1TUNuCJDcfwf0vGs4CnA2IwY2POUvXXGfl5ajFlUBCmmA3/5ZfX4sSFpkU2E7NKkV9ei5SccqTklGOD2UKbg0O9LXJwBoVwoc3OqKxtwObjFwFwiIk6b8qgILioVTiXX4m0gkr0C/Ts9DH+uycNAHBZXz+cuFCGA+cK8emBdCy+sl93N9cpCAUXmmEwY0NCCOxmvoxDCfLWYdqQYEwbYrnQpinBuATHGgOdoso6JGWXISm7DF+iaaHN2DBveRXxuEg9BgZ7MTmxFT+dyEFVnQF9AzwwNtrP3s0hB+Pj5orL+/ljf2ohtiXnYsnkmE7dP7esBt8lmnpfn7k2FiculOK5b5Pwyk8pmDIoCDGsLtwqlQKjGQYzNpSSU4688lq4u2owri/frB1ViI8brh7qhqvNFtrMLq2xmCJ+LKsUpdX1SMwqRWJWKYAMAKaFNoeFWy7TEMOFNgE0zWK6ZQxry1DXxMeGYH9qIban5HU6mPlkfzrqDQLjov0wuo8fRkX54ueTudhzpgCPrU/Exnsn8ItIM8pdM5vBjE1JQ0wT+gdA58J8AGehUqkQ4euOCF93XDPctA6VEAKZRdVyBeNjWaU4ccG00ObvGSX4PaNEvr+HVoPh4eZF/vTo28sW2swqrsL+1EIAwE1jWCeEuiZ+SDD+/v1J/JZWhLKaevi4dazUQlVdA/7voOkLhxQEqVQqrLp1BGa+uRsJmSX4YPc5PDBtgM3a7oiaZjPZtx3WMJixIam+DIeYnJ9KpUKfAA/0CfDAdSPCAbS90OZv6UX4Lb1pHSpvnQuGywnGpmGqKH/nXYfq699N3fsT+wdw9gh1Wd9AT/QP8kRqfiV2n86XX3vt2XA4C6XV9YgO8JB7XAEgTO+OFTcMw2PrE/HWL6cxbXAwhob72Kr5DkuJ70oMZmyksrYBh8+bPqymMJjplTqz0GZ5bQMOnCvEgXOF8v317q5ma1CZZlGF690cPsARQlgMMRFdivjYEKTmn8O25LwOBTMGo8BHe02Jv3dO6tdiyPem0RHYkpSDLUm5eGx9Ar598Er2rDcSCh5oYjBjIwdSC1FvEOjj74G+AfzmSSadXWhzz5kCeeooAAR4as1WETfNogrxcbPX6XTJkfPFSC+sgqdWg9lxrC1DlyZ+SDA+3H0OO07lwWAU7eajbT2Zg4yiKvh6uFqdRadSqbDypjgcTi9GSk453vrlDJ66Zoitmu9QOMzUC5lX/XX0b9JkW51ZaLOwsg47T+Vj56mmhTaDvXUWU8TjIvUI9FLuQptSbZlr48LgoeVbEF2asdF+0Lu7oqSqHr9nFOOyvv5t7v/h7nMAgP83PrrV51+glw4rb4rDvZ8fwQe7UjEjNtiiynhvx9lMvQiXMKBL0ZmFNvPKa/FLch5+SW5aaDNc79aYYGyaRRUXoYefAhbarK4z4Idjptoyt7C2DHUDF40aUwcH4duEbGxLzmszmDlyvhi/Z5RAq1Hj9onRbR73muGhuHl0BDYdvYDH1ydi88OTGXwrGK+MDaQXVCKjqAquGhUm9O96mW0ic9YW2qyqa0DyxbIWC21ml9Ygu7QGW5KaFtqM8neX69+MaFyHqqcX2vz5ZA7KaxsQ5e+Oy9v5Bk3UUdOHBDcGM7l4enbrQ0L/3WPqlZk7KhzB3u0Pzy67YRgOnCtEemEVXvkxBS/MHd5tbXZEonGcSYmDDQxmbEDqlRkX7Q9PLlpINuShdenwQpuZRdXILKrGD41VdwGgX6BnU4JxDyy0KQ0x3Tw6sldNRSfbmjooGBq1CmfyKpBRWIU+VvIUMwqrsCXJtMjsXVM6VpNG7+6KVbeOwJ8++g2fHjiPmUNDMWkg19hjMNNLcAkDsqfOLLSZVmAqBd8TC21eLK3G3rOmZGbOYqLupPdwxbhoPxxMK8K2lFz82cpyBB/vS4NRmIb+B4V4d/jYkwcG4U9XROOzX8/jiY2J+OmRKT3eo6kUQrmTmRjMdLfaBgMONBYDmzKQwQwpw6UstKlRqzAw2OuSF9rc9PsFCAFc3s/f6jdnoksxIzYEB9OKsD0lr0UwU1JVh/WHTcuO3NXJSsEAsPTaIdhzJh/phVVY8b8k/GP+qO5ossNpimWU1zXDYKabHU4vRnW9AUHeOsSGdTz6J+pp7S20eaxxLaq2F9r0lYeo2lpo07y2DBeVJFuYHhuMlZuT8eu5QpTX1MPbrBrw/x3MQFWdAUNCvXHlgM7nMXpoXfDG/JGY9/4BbPr9AmYNC8WsYb23rACHmXoBTskmR9Z8oU0hBHLLak09N2Y5OBYLbf5muq/WxTTF3HyIakCQaaHNo5klOJdfCXdXDa6NC7PjGZKz6h/khX6BnkgrqMTeMwWY3fg8q2sw4pP96QBMvTJdfV8eG+2Pu6f0x/u7UvHMpuMYG+2n6BIItsBhpl6ESxiQM1GpVAjVuyFUH4qZjd9EhRC4UFItz5463tiLU1bTgMTMEiRmlsj3d3NVY1i4HrUNBgDA7OGhNk0wpt5t+pBgfLQ3Db8k58nBzHeJ2cgrr0WIjw7Xj+zYcgetefTqgdh5Kg8pOeV49uvjeP//je1VX1qlCsBKPGO+q3SjnNIanMoth0oFTBrAjHdyTiqVCpF+Hoj085A/MIQQyCiqslim4cSFMlTUNuDI+WL5vqwtQ7YUH2sKZnY2VgNWq5qmYy+e2K/VYdCO0rlo8Mb8kbjx3/uwJSkXXx+9gJt7YTK7EuM3BjPdaHfjENPISF9FFCgj6ikqlQrRAZ6IDvCUv/0ajQJphZXyKuIBXlpMZN0lsqHL+vrD280FhZV1SMgsQVVdA1JyyuGh1eAPl/fplscYFq7Hw/ED8frPp7HsuyRcEROAcF/3bjm20il5mOnSwlSywKq/RE3UahX6B3nhxtEReP76oXhg2oBe1SVPPc9Vo5bff7en5OI/e0wLSs4fFwW9R/dNp773qv4YFeWL8poGPPXVMbmYnLOTzlKJyxkwmOkmDQYj9pwxBTNcJZuIyD7iY03J6+/uTMXu0/lQq0yrY3cnF40ab8wfCTdXNfacKcDnBzO69fhKp8TvJAxmukliVinKahqgd3fFyEi9vZtDRNQrTR0UDLWqaUhk9vAwRPl3f12j/kFe8mraL/2QjPSCym5/DMVRcA8Ug5luIg0xTRoYCBcN/6xERPbg56m1CF6WTO7eXhlziyb0xYSYAFTXG/DXDYkwGJX7Yd8dmoaZlIefut2E+TJERMqgM5u1NLqPn80eR61W4bV5I+Clc8Hh88X4T+PMKWenxNw3BjPdoLiyDseySgBwCQMiInt7ds5QDAn1xqb7J9r8sSL9PPD8dUMBAP/4+TRScsps/pj2ouBRJgYz3WHP2QIIAQwJ9Uaovv1l5YmIyHauGhSEnx6ZgjE27JUxN29cJOKHBKPOYMRj6xJR12DskcftaQLKjWYYzHQDVv0lIuq9VCoVXr4lDn4erjh5sQzvbD9j7ybZlAJHmRjMXCohBHZzSjYRUa8W7O2GF2+MA2CaFp5gtqyHs+AwkxNLvliO/PJauLtqMK5vz3RpEhGR8swZEYYbRobDYBR4bH0CauoN9m5St2LRPCcmzWKa2D8AOheNnVtDRET29MLcYQj21uFcfiVe/SnF3s2xCQ4zOaFdp/MAAFcN5hATEVFv5+uhxau3jgAArN6Xjv2pBXZuUffhMJOTqqhtwOF004rAnJJNREQAMG1wMBY2Lmz5xIZjKK+pt3OLuoc0m0mBHTMMZi7Fr6mFaDAKRAd4oG+gp72bQ0RECvHsnFhE+bvjQkk1/v79SXs3p1txmMnJSIXyLu/rb9+GEBGRonjpXPDGvFFQqYD1h7Pwy8lcezfp0nGYyTmF6N0wLtoPY6I5i4mIiCxd3s8fSxpX7H5603EUVdbZ/DGTL5bhfKFtFr2UZzMpsGuGwcwl+OP4aGy8b6I8NkpERGTu8ZmDMTDYCwUVtXjumxMQNsyi3Zaci2v/uQcLP/zVZo8BMGeGiIioV3Fz1eAf80fBRa3CD8cv4rvEbJs8zsnsMvzly6MQAsivqLXJY9gyELtUDGaIiIhsKC5SjwenDwAAPP9tEnLLarr1+HllNVjyySFU1ZmK9Nkq5hBNVfMUh8EMERGRjT0wbQDiIvQora7HkxuPdVsvR3WdAXd9ehjZpTUI8tZ1yzHbwwrAREREvZCrRo1/zB8JrYsau07nY+2hzEs+ptEo8PiGBCRmlcLXwxX/WjgagO0mHSl3kInBDBERUY8YGOKNJ2YOBgC8+P1JZBZVXdLx/rH1NDYfz4GrRoUP/t9Ym9c7kzqTFDiZicEMERFRT7ljUj9c3tcflXUGPL4hEUZj1/o7vjqShX/tOAsAePnmERgfEyAP/tg6UVeBsQyDGSIiop6iUavw+ryR8NBq8FtaET7el9bpY/yWVoSnNx0DANw/tT9uHRvZ3c20Sih4oInBDBERUQ/qE+CBv80ZCgBYteUUzuSWd/i+5wsrcc9nh1FvEJg9PBR/bRy2AiB3mdgsZ4bDTERERCRZeHkUpg4OQl2DEY+tT0S9wdjufUqr63HHmkMorqrHiEg9/jF/FNTqno8sOJuJiIiIoFKp8OotI6B3d8XxC6X4d2P+S2vqDUY88H+/IzW/EmF6N/z39nFw12osj9kYZCi4tp3NMJghIiKygxAfN7wwdxgA4F/bz+J4VqnV/YQQWPZdEvaeLYCHVoP/LhqHYB+3FvvZevhHSizudcNMK1euxMSJE+Hh4QFfX98WtycmJmLhwoWIioqCu7s7YmNj8fbbb1vss2nTJlx99dUICgqCj48PJkyYgC1bttiy2URERD3ihpHhmBMXhgajwGPrE1BTb2ixz0d70/DFwQyoVMA/bxuNYeF6q8cyjzGsHceZ2TSYqaurw7x583DfffdZvf3IkSMIDg7G559/jqSkJDz77LNYunQp/vWvf8n77N69G1dffTU2b96MI0eOYNq0abj++utx9OhRWzadiIjI5lQqFf5+43AEeulwJq8Cb/x8yuL2X07mYuXmZADAs9fGYsbQkFaP5eehRaSfOwBg45Gsbm+rglczgEr0wMpRa9aswSOPPIKSkpJ2933ggQeQnJyM7du3t7rPsGHDsGDBAjz//PMdevyysjLo9XqUlpbCx8eno80mIiLqEduSc3HnJ4ehUgFr77oC42MCcDK7DLe+vx9VdQYsvDwKL90UB1U7Yzyf7E/Hsu+SEB3gge2PT4WmGxOEn/vmBD779Tweih+Ix64e1G3HbUtHP78VlzNTWloKf3//Vm83Go0oLy9vc5/a2lqUlZVZ/BARESlVfGwI5o+LhBDAXzcmIq2gUl488soBAXhh7vB2AxkAmDcuEn4erjhfWIWfTuT0QMuVQVHBzP79+7Fu3Trcfffdre7z+uuvo6KiAvPnz291n5dffhl6vV7+iYqKskVziYiIus1z1w1FhK87MouqMfvt3cgurUFMkCfe/cNYuGo69nHtoXXB7RP6AgDe35XardWApaJ5Shxm6nQw8/TTT0OlUrX5k5KS0umGnDhxAnPnzsWyZcswc+ZMq/t88cUXWLFiBdavX4/g4OBWj7V06VKUlpbKP5mZl76gFxERkS15u7nitXkjAAA19Ub4erji40WXQe/h2qnjLJrYF26uahy/UIoD5wq7rX1KLprn0tk7PP7441i8eHGb+8TExHTqmCdPnkR8fDzuvvtu/O1vf7O6z9q1a7FkyRJs2LABM2bMaPN4Op0OOl3PLIVORETUXSb2D8RT1wzBhsOZeOWWEV1aPNLfU4sF46LwyYHz+GDXOUzsH2iDlipLp4OZoKAgBAUFdVsDkpKSMH36dCxatAgrV660us+XX36JO+64A2vXrsWcOXO67bGJiIiU5r6p/XHf1P6XdIwlk2Pw2a/nset0Pk5ml2Fo+KVPfmmazaS8rhmb5sxkZGQgISEBGRkZMBgMSEhIQEJCAioqKgCYhpamTZuGmTNn4rHHHkNOTg5ycnKQn58vH+OLL77A7bffjjfeeAPjx4+X9ykttV5ciIiIqLeL8vfAnBHhAIAPd6d2yzGVPMxk02Dm+eefx+jRo7Fs2TJUVFRg9OjRGD16NA4fPgwA2LhxI/Lz8/H5558jLCxM/rnsssvkY3z44YdoaGjAAw88YLHPww8/bMumExERObR7pphSPv537CKyiqvs3Brbsmkws2bNGgghWvxMnToVALB8+XKrt6enp8vH2Llzp9V91qxZY8umExERObThEXpMGhAIg1Hgo71p3XBEJ5rNRERERI7hnqtMvTNrf8tEcWXdJR2r1w4zERERkf1MGhCIYeE+qK434LNfz9u7OTbDYIaIiMhJqVQq3HOVaWbUmv3pl7QAZVPPjPK6ZhjMEBERObFrh4ci0s8dRZV12HAJC1AK2Hwpxy5jMENEROTEXDRq3DXZlDvzn93nYDAqNyjpKgYzRERETk5agDKjqAo/nrjYpWMwAZiIiIjsxkPrgkUT+wIAPth17pIWoOx1FYCJiIhIGW6fYLYAZWrnF6BU8uAUgxkiIqJeQFqAEgDe332u0/fnMBMRERHZ3ZLJMVCrgN2NC1B2hQJjGQYzREREvYX5ApQfdHIBSk7NJiIiIkWQFqD8/thFZBZ1YgFKDjMRERGREgyP0GPywK4vQMnZTERERGR390wxLXGw7lDHF6BU7iATgxkiIqJe58oBAZ1egFKqTcNhJiIiIrK75gtQVtd1fQFKJWAwQ0RE1AuZL0C58Uhmu/tzmImIiIgUxWIByj1paDAY29y/qWie8saZGMwQERH1UvPHRckLUP6UlNOh+ygvlGEwQ0RE1Gu5azUdXoCSw0xERESkSOYLUO5vYwFKzmYiIiIiRbJYgHJX+0scKDCWYTBDRETU2y2ZHAONWoU9ZwqQlF1qdR8OMxEREZFiRfl7YE5cGADgw93nrO/E2UxERESkZHd3cAFKBcYyDGaIiIio/QUohYIHmhjMEBEREYCmBSjXHspAUbMFKOWieT3dqA5gMENEREQAmhagrKk34rMDrSxAqcBxJgYzREREBMByAcpPDlguQNlGPT27YzBDREREsmuHhyLKv+UClFLOjPL6ZRjMEBERkRnzBSg/3HOuxQKUChxlYjBDREREluaNNS1AmVlUjR9PmBag5DATEREROQyLBSh3p0KIponZKgUONDGYISIiohakBShPXCizWICSw0xERETkEJovQMlhJiIiInI45gtQpuSUAeBsJiIiInIg5gtQZhVXA+AwExERETkYaQFKJWMwQ0RERK2SFqCUcDYTERERORxpAUoAikyaYTBDREREbbpyQACGR/gAAHzcXO3cmpZc7N0AIiIiUjaVSoX/3n4ZtqfkIT422N7NaYHBDBEREbUrVO+GP4zvY+9mWMVhJiIiInJoDGaIiIjIoTGYISIiIofGYIaIiIgcGoMZIiIicmgMZoiIiMihMZghIiIih8ZghoiIiBwagxkiIiJyaAxmiIiIyKExmCEiIiKHxmCGiIiIHBqDGSIiInJovWLVbCEEAKCsrMzOLSEiIqKOkj63pc/x1vSKYKa8vBwAEBUVZeeWEBERUWeVl5dDr9e3ertKtBfuOAGj0Yjs7Gx4e3tDpVLZuzndpqysDFFRUcjMzISPj4+9m2NzPF/n1ZvOFeD5OrPedK6A7c9XCIHy8nKEh4dDrW49M6ZX9Myo1WpERkbauxk24+Pj0yteNBKer/PqTecK8HydWW86V8C259tWj4yECcBERETk0BjMEBERkUNjMOPAdDodli1bBp1OZ++m9Aier/PqTecK8HydWW86V0A559srEoCJiIjIebFnhoiIiBwagxkiIiJyaAxmiIiIyKExmCEiIiKHxmCGiIiIHBqDGTt7+eWXcdlll8Hb2xvBwcG48cYbcerUKYt9pk6dCpVKZfFz7733WuyTkZGBOXPmwMPDA8HBwXjiiSfQ0NBgsc/OnTsxZswY6HQ6DBgwAGvWrLH16VlYvnx5i/MYMmSIfHtNTQ0eeOABBAQEwMvLC7fccgtyc3MtjuEI5ynp27dvi/NVqVR44IEHADj+dd29ezeuv/56hIeHQ6VS4ZtvvrG4XQiB559/HmFhYXB3d8eMGTNw5swZi32Kiorwxz/+ET4+PvD19cWdd96JiooKi32OHTuGyZMnw83NDVFRUVi1alWLtmzYsAFDhgyBm5sb4uLisHnz5h493/r6ejz11FOIi4uDp6cnwsPDcfvttyM7O9viGNaeE6+88orizre9a7t48eIW53HNNddY7OMs1xaA1dexSqXCa6+9Ju/jKNe2I585Pfle/O9//xt9+/aFm5sbxo8fj99++61rJybIrmbNmiVWr14tTpw4IRISEsS1114r+vTpIyoqKuR9rrrqKnHXXXeJixcvyj+lpaXy7Q0NDWL48OFixowZ4ujRo2Lz5s0iMDBQLF26VN7n3LlzwsPDQzz22GPi5MmT4p133hEajUb89NNPPXauy5YtE8OGDbM4j/z8fPn2e++9V0RFRYlt27aJw4cPiyuuuEJMnDjR4c5TkpeXZ3GuW7duFQDEjh07hBCOf103b94snn32WbFp0yYBQHz99dcWt7/yyitCr9eLb775RiQmJoobbrhB9OvXT1RXV8v7XHPNNWLkyJHi119/FXv27BEDBgwQCxculG8vLS0VISEh4o9//KM4ceKE+PLLL4W7u7v44IMP5H327dsnNBqNWLVqlTh58qT429/+JlxdXcXx48d77HxLSkrEjBkzxLp160RKSoo4cOCAuPzyy8XYsWMtjhEdHS1eeOEFi2tu/lpXyvm2d20XLVokrrnmGovzKCoqstjHWa6tEMLiPC9evCg+/vhjoVKpRGpqqryPo1zbjnzm9NR78dq1a4VWqxUff/yxSEpKEnfddZfw9fUVubm5nT4vBjMKk5eXJwCIXbt2yduuuuoq8fDDD7d6n82bNwu1Wi1ycnLkbe+9957w8fERtbW1QgghnnzySTFs2DCL+y1YsEDMmjWre0+gDcuWLRMjR460eltJSYlwdXUVGzZskLclJycLAOLAgQNCCMc5z9Y8/PDDon///sJoNAohnOe6CiFafAAYjUYRGhoqXnvtNXlbSUmJ0Ol04ssvvxRCCHHy5EkBQBw6dEje58cffxQqlUpcuHBBCCHEu+++K/z8/OTzFUKIp556SgwePFj+//z588WcOXMs2jN+/Hhxzz33dOs5mrP2gdfcb7/9JgCI8+fPy9uio6PFm2++2ep9lHi+rQUzc+fObfU+zn5t586dK6ZPn26xzRGvrRAtP3N68r348ssvFw888ID8f4PBIMLDw8XLL7/c6fPgMJPClJaWAgD8/f0ttv/f//0fAgMDMXz4cCxduhRVVVXybQcOHEBcXBxCQkLkbbNmzUJZWRmSkpLkfWbMmGFxzFmzZuHAgQO2OhWrzpw5g/DwcMTExOCPf/wjMjIyAABHjhxBfX29RRuHDBmCPn36yG10pPNsrq6uDp9//jnuuOMOi5XbneW6NpeWloacnByLtun1eowfP97ievr6+mLcuHHyPjNmzIBarcbBgwflfaZMmQKtVivvM2vWLJw6dQrFxcXyPkr8G5SWlkKlUsHX19di+yuvvIKAgACMHj0ar732mkXXvCOd786dOxEcHIzBgwfjvvvuQ2FhoXybM1/b3Nxc/PDDD7jzzjtb3OaI17b5Z05PvRfX1dXhyJEjFvuo1WrMmDGjS+fbK1bNdhRGoxGPPPIIrrzySgwfPlze/oc//AHR0dEIDw/HsWPH8NRTT+HUqVPYtGkTACAnJ8fiSQVA/n9OTk6b+5SVlaG6uhru7u62PDUAwPjx47FmzRoMHjwYFy9exIoVKzB58mScOHECOTk50Gq1Ld74Q0JC2j0H6ba29unJ87Tmm2++QUlJCRYvXixvc5brao3UPmttM297cHCwxe0uLi7w9/e32Kdfv34tjiHd5ufn1+rfQDqGPdTU1OCpp57CwoULLVYSfuihhzBmzBj4+/tj//79WLp0KS5evIh//OMfABznfK+55hrcfPPN6NevH1JTU/HMM89g9uzZOHDgADQajVNf208++QTe3t64+eabLbY74rW19pnTU+/FxcXFMBgMVvdJSUnp9LkwmFGQBx54ACdOnMDevXsttt99993y73FxcQgLC0N8fDxSU1PRv3//nm5ml82ePVv+fcSIERg/fjyio6Oxfv16u33o9pSPPvoIs2fPRnh4uLzNWa4rWaqvr8f8+fMhhMB7771ncdtjjz0m/z5ixAhotVrcc889ePnll+2+tk1n3HbbbfLvcXFxGDFiBPr374+dO3ciPj7eji2zvY8//hh//OMf4ebmZrHdEa9ta585jojDTArx4IMP4vvvv8eOHTsQGRnZ5r7jx48HAJw9exYAEBoa2iLTXPp/aGhom/v4+PjYLZDw9fXFoEGDcPbsWYSGhqKurg4lJSUt2tjeOUi3tbWPPc/z/Pnz+OWXX7BkyZI293OW6wo0tc9a28zbnpeXZ3F7Q0MDioqKuuWaS7f3JCmQOX/+PLZu3WrRK2PN+PHj0dDQgPT0dACOd76SmJgYBAYGWjx3ne3aAsCePXtw6tSpdl/LgPKvbWufOT31XhwYGAiNRtNt58tgxs6EEHjwwQfx9ddfY/v27S26Ia1JSEgAAISFhQEAJkyYgOPHj1u8eUhvpEOHDpX32bZtm8Vxtm7digkTJnTTmXReRUUFUlNTERYWhrFjx8LV1dWijadOnUJGRobcRkc9z9WrVyM4OBhz5sxpcz9nua4A0K9fP4SGhlq0raysDAcPHrS4niUlJThy5Ii8z/bt22E0GuXAbsKECdi9ezfq6+vlfbZu3YrBgwfDz89P3kcJfwMpkDlz5gx++eUXBAQEtHufhIQEqNVqeUjGkc7XXFZWFgoLCy2eu850bSUfffQRxo4di5EjR7a7r1KvbXufOT31XqzVajF27FiLfYxGI7Zt29a18+10yjB1q/vuu0/o9Xqxc+dOiyl9VVVVQgghzp49K1544QVx+PBhkZaWJr799lsRExMjpkyZIh9DmiY3c+ZMkZCQIH766ScRFBRkdZrcE088IZKTk8W///3vHp+y/Pjjj4udO3eKtLQ0sW/fPjFjxgwRGBgo8vLyhBCm6YB9+vQR27dvF4cPHxYTJkwQEyZMcLjzNGcwGESfPn3EU089ZbHdGa5reXm5OHr0qDh69KgAIP7xj3+Io0ePyrN3XnnlFeHr6yu+/fZbcezYMTF37lyrU7NHjx4tDh48KPbu3SsGDhxoMX23pKREhISEiD/96U/ixIkTYu3atcLDw6PFdFYXFxfx+uuvi+TkZLFs2TKbTN9t63zr6urEDTfcICIjI0VCQoLFa1ma3bF//37x5ptvioSEBJGamio+//xzERQUJG6//XbFnW9b51peXi7++te/igMHDoi0tDTxyy+/iDFjxoiBAweKmpoa+RjOcm0lpaWlwsPDQ7z33nst7u9I17a9zxwheu69eO3atUKn04k1a9aIkydPirvvvlv4+vpazJLqKAYzdgbA6s/q1auFEEJkZGSIKVOmCH9/f6HT6cSAAQPEE088YVGPRAgh0tPTxezZs4W7u7sIDAwUjz/+uKivr7fYZ8eOHWLUqFFCq9WKmJgY+TF6yoIFC0RYWJjQarUiIiJCLFiwQJw9e1a+vbq6Wtx///3Cz89PeHh4iJtuuklcvHjR4hiOcJ7mtmzZIgCIU6dOWWx3huu6Y8cOq8/dRYsWCSFM07Ofe+45ERISInQ6nYiPj2/xdygsLBQLFy4UXl5ewsfHR/z5z38W5eXlFvskJiaKSZMmCZ1OJyIiIsQrr7zSoi3r168XgwYNElqtVgwbNkz88MMPPXq+aWlprb6WpbpCR44cEePHjxd6vV64ubmJ2NhY8dJLL1kEAEo537bOtaqqSsycOVMEBQUJV1dXER0dLe66664WH0DOcm0lH3zwgXB3dxclJSUt7u9I17a9zxwheva9+J133hF9+vQRWq1WXH755eLXX3/t0nmpGk+OiIiIyCExZ4aIiIgcGoMZIiIicmgMZoiIiMihMZghIiIih8ZghoiIiBwagxkiIiJyaAxmiIiIyKExmCEiIiKHxmCGiIiIHBqDGSIiInJoDGaIiIjIof1/tZeLyNQSjWMAAAAASUVORK5CYII=",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "df_w_bins[df_w_bins[\"MedHouseVal_bins\"] == 5][\"Longitude\"].plot(\n",
-    "    title=\"Longitude feature values before calibration\"\n",
-    ");"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "144a8779",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAHICAYAAAC772uFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB/DUlEQVR4nO3dd3iTVfsH8G+SNunem5bSltECZZUhexQZouBi6U9BRNyKOHExFAeu19etr4KKyhLBzd4gslqgpay2tIxuundyfn+kedq06aRpRr+f68oFffLkyXky75xzn/vIhBACRERERBZKbuoGEBEREV0PBjNERERk0RjMEBERkUVjMENEREQWjcEMERERWTQGM0RERGTRGMwQERGRRWMwQ0RERBaNwQwRERFZNAYzZLF27doFmUyGXbt2tcn9jRo1CqNGjWrVY37//fcIDw+Hra0t3NzcWvXY7dHKlSshk8mQnJxs6qZcl8rKSjz33HMICgqCXC7HrbfeauomXbfk5GTIZDKsXLlS2rZ48WLIZDK9/WQyGR577LE2bl1dnTp1wuzZs03dDGoiBjNUL90Xw5EjR0zdlCb78ccf8Z///MfUzWiShIQEzJ49G2FhYfjqq6/w5ZdfGuV+Dhw4gMWLFyM3N9cox6fW98033+Cdd97BnXfeiW+//RZPPfUU4uPjsXjxYosP1MwJ3xvWw8bUDSBqqREjRqCkpARKpVLa9uOPP+LUqVOYP3++6RrWRLt27YJGo8GHH36Izp07G+1+Dhw4gCVLlmD27Nns/bEQO3bsQIcOHfDBBx9I29avX48lS5Zg1KhR6NSpk+ka14pefvllvPDCCya7/4beG2fOnIFczt/7loLPFFksuVwOOzs7i/3AycjIAACLDTCKiopM3QSrlZGR0WavC1M+jzY2NrCzs2u147XmuahUKtja2rba8ci4LPNbgMzK8ePHMXHiRLi4uMDJyQnR0dH4559/9PbRDVnt378fCxYsgLe3NxwdHXHbbbchMzNTb1+NRoPFixcjICAADg4OGD16NOLj4+uMYdfOmRk1ahT++OMPXLx4ETKZDDKZTPoFW18uRX15N19++SXCwsJgb2+PgQMHYu/evQbPvaysDIsWLULnzp2hUqkQFBSE5557DmVlZQ0+Zp06dcKiRYsAAN7e3pDJZFi8eLF0/V9//YXhw4fD0dERzs7OmDRpEuLi4vSOceLECcyePRuhoaGws7ODn58f5syZg+zsbGmfxYsX49lnnwUAhISESI9LcnKywRwGndrt0eU2xMfH46677oK7uzuGDRsmXb9q1SpERUXB3t4eHh4emDFjBlJTUxt8DNavXw+ZTIbdu3fXue6LL76ATCbDqVOnmnyu9al9LjqGciJyc3Mxf/58BAUFQaVSoXPnznj77beh0Wj09lu9ejWioqLg7OwMFxcXREZG4sMPP2y0Le+++y6GDBkCT09P2NvbIyoqCuvXr5eu1z0nO3fuRFxcnPR8rVy5ElOnTgUAjB49Wtpe83XblNfM7Nmz4eTkhAsXLuCmm26Cs7Mz7r777gbbfPnyZdx///0ICAiASqVCSEgIHn74YZSXlwMAcnJy8MwzzyAyMhJOTk5wcXHBxIkTERsb2+jjYShnRueHH35At27dYGdnh6ioKOzZs8fgbQ29Jq/3vQEYfn0kJiZi6tSp8PDwgIODA2644Qb88ccfevvoPlPWrl2LZcuWITAwEHZ2doiOjsb58+cbfUyoZTjMRNclLi4Ow4cPh4uLC5577jnY2triiy++wKhRo7B7924MGjRIb//HH38c7u7uWLRoEZKTk/Gf//wHjz32GNasWSPts3DhQixfvhy33HILxo8fj9jYWIwfPx6lpaUNtuWll15CXl4eLl26JHXPOzk5Nfucvv76azz44IMYMmQI5s+fj8TEREyePBkeHh4ICgqS9tNoNJg8eTL27duHefPmISIiAidPnsQHH3yAs2fPYuPGjfXex3/+8x989913+OWXX/DZZ5/ByckJvXr1AqBNCp41axbGjx+Pt99+G8XFxfjss88wbNgwHD9+XArQtm7disTERNx3333w8/NDXFwcvvzyS8TFxeGff/6BTCbD7bffjrNnz+Knn37CBx98AC8vLwDaAKp2ENkUU6dORZcuXfDGG29ACAEAWLZsGV555RVMmzYNc+fORWZmJj766COMGDECx48fr7eHYdKkSXBycsLatWsxcuRIvevWrFmDHj16oGfPnk0+1+tVXFyMkSNH4vLly3jwwQfRsWNHHDhwAAsXLsTVq1elXKytW7di5syZiI6Oxttvvw0AOH36NPbv348nn3yywfv48MMPMXnyZNx9990oLy/H6tWrMXXqVPz++++YNGkSvL298f3332PZsmUoLCzEm2++CQDo0qULnnjiCfz3v//Fiy++iIiICACQ/m3qawbQJhePHz8ew4YNw7vvvgsHB4d623vlyhUMHDgQubm5mDdvHsLDw3H58mWsX78excXFUCqVSExMxMaNGzF16lSEhIQgPT0dX3zxBUaOHIn4+HgEBAQ0+7nYvXs31qxZgyeeeAIqlQqffvopJkyYgH///Vd6TegYek1e73vDkPT0dAwZMgTFxcV44okn4OnpiW+//RaTJ0/G+vXrcdttt+nt/9Zbb0Eul+OZZ55BXl4eli9fjrvvvhuHDh1q9uNBTSCI6rFixQoBQBw+fLjefW699VahVCrFhQsXpG1XrlwRzs7OYsSIEXWONXbsWKHRaKTtTz31lFAoFCI3N1cIIURaWpqwsbERt956q979LF68WAAQs2bNkrbt3LlTABA7d+6Utk2aNEkEBwfXey5JSUl622sfo7y8XPj4+Ig+ffqIsrIyab8vv/xSABAjR46Utn3//fdCLpeLvXv36h3z888/FwDE/v37DT5mOosWLRIARGZmprStoKBAuLm5iQceeEBv37S0NOHq6qq3vbi4uM4xf/rpJwFA7NmzR9r2zjvvGDz3pKQkAUCsWLGiznEAiEWLFtVp68yZM/X2S05OFgqFQixbtkxv+8mTJ4WNjU2d7bXNnDlT+Pj4iMrKSmnb1atXhVwuF0uXLm32uRp6nmufi05wcLDe6+m1114Tjo6O4uzZs3r7vfDCC0KhUIiUlBQhhBBPPvmkcHFx0WtzU9U+j/LyctGzZ08xZswYve0jR44UPXr00Nu2bt26Oq93IZr3mpk1a5YAIF544YUmtffee+8Vcrnc4GeA7n1cWloq1Gq13nVJSUlCpVLpPYeGXm+611VNAAQAceTIEWnbxYsXhZ2dnbjtttvq3Lb2a1KI639vCFH39TF//nwBQO/9XlBQIEJCQkSnTp2kx0D3mRIREaH3GfLhhx8KAOLkyZN17ouuH4eZqMXUajW2bNmCW2+9FaGhodJ2f39/3HXXXdi3bx/y8/P1bjNv3jy9X9HDhw+HWq3GxYsXAQDbt29HZWUlHnnkEb3bPf7440Y8k2pHjhxBRkYGHnroIb3E4tmzZ8PV1VVv33Xr1iEiIgLh4eHIysqSLmPGjAEA7Ny5s9n3v3XrVuTm5mLmzJl6x1QoFBg0aJDeMe3t7aX/l5aWIisrCzfccAMA4NixY82+76Z46KGH9P7esGEDNBoNpk2bptdePz8/dOnSpdHHYPr06cjIyNAbLlm/fj00Gg2mT58ubWuLc123bh2GDx8Od3d3vXMZO3Ys1Gq1NMzh5uaGoqIibN26tdn3UfM8rl27hry8PAwfPvy6zqE5rxmdhx9+uNHjajQabNy4Ebfccgv69+9f53rd+1ilUkl5a2q1GtnZ2XByckK3bt1afF6DBw9GVFSU9HfHjh0xZcoUbN68GWq1Wm/f2q9JwDivlz///BMDBw7UG151cnLCvHnzkJycjPj4eL3977vvPr3PkOHDhwPQDlVR6+MwE7VYZmYmiouL0a1btzrXRUREQKPRIDU1FT169JC2d+zYUW8/d3d3ANoPdgBSUFN7do+Hh4e0rzHp7r9Lly56221tbfUCNgA4d+4cTp8+XW+3tC7BtznOnTsHAFJAVJuLi4v0/5ycHCxZsgSrV6+uc195eXnNvu+mCAkJ0fv73LlzEELUebx0GkugnDBhAlxdXbFmzRpER0cD0A4x9enTB127dpX2a4tzPXfuHE6cONHo8/nII49g7dq1mDhxIjp06IBx48Zh2rRpmDBhQqP38fvvv+P1119HTEyMXl7V9QyTNec1A2iTbgMDAxs9bmZmJvLz8+sM69Smm5H36aefIikpSS/Y8PT0bPR+DDH0euratSuKi4uRmZkJPz8/aXvt1yRgnNfLxYsX6wybA9VDfRcvXtR7rBr7rKPWxWCG2pRCoTC4XVSNdRtLfV8WtX/lNYdGo0FkZCTef/99g9fXzK9pzjEBbQ5EzQ9sHRub6rfstGnTcODAATz77LPo06cPnJycoNFoMGHChDoJq4a05DGp+YtX116ZTIa//vrL4HPbWM6SSqXCrbfeil9++QWffvop0tPTsX//frzxxht6+13vuRpS+zw1Gg1uvPFGPPfccwb31wVXPj4+iImJwebNm/HXX3/hr7/+wooVK3Dvvffi22+/rff+9u7di8mTJ2PEiBH49NNP4e/vD1tbW6xYsQI//vhji85B126gaa8ZQL8npTW88cYbeOWVVzBnzhy89tpr8PDwgFwux/z581v83DRH7dckYJzXS3OZ6rOuvWIwQy3m7e0NBwcHnDlzps51CQkJkMvlzf5CDw4OBgCcP39e7xdXdnZ2k37R1PcFrftVVLs4lq4npvb9nzt3Tu+XbkVFBZKSktC7d29pW1hYGGJjYxEdHd0qCai6YwLaL8yxY8fWu9+1a9ewfft2LFmyBK+++qq0XfcrvabrfUwaa68QAiEhIXo9Kc0xffp0fPvtt9i+fTtOnz4NIYTeEFNzztUQd3f3OudYXl6Oq1ev1jmXwsLCBh93HaVSiVtuuQW33HILNBoNHnnkEXzxxRd45ZVX6q0Z9PPPP8POzg6bN2+GSqWStq9YsaJJ51Hf89jU10xzeXt7w8XFRZpRVp/169dj9OjR+Prrr/W25+bmSkm1zWXouT179iwcHBzq7TnTaY33hiHBwcH1ftbprifTYc4MtZhCocC4ceOwadMmvSnP6enp+PHHHzFs2LA6XdyNiY6Oho2NDT777DO97R9//HGTbu/o6GiwG1n3gV9zeqdara5Tdbd///7w9vbG559/Lk09BbRTu2t/IU6bNg2XL1/GV199Vef+SkpKWlTzYvz48XBxccEbb7yBioqKOtfrZiDpfvXV/pVnqPqxo6MjgLpBi4uLC7y8vOpMef3000+b3N7bb78dCoUCS5YsqdMWIUSTpk6PHTsWHh4eWLNmDdasWYOBAwfqBbLNOVdDwsLC6pzjl19+WadnZtq0aTh48CA2b95c5xi5ubmorKwEgDrnJJfLpZloDU3JVygUkMlkevebnJzc4Ky3mup7Hpv6mmku3TIKv/32m8Eq4LrnQ6FQ1Hlu1q1bh8uXL7fofgHg4MGDerktqamp2LRpE8aNG1dvj4dOa7w3DLnpppvw77//4uDBg9K2oqIifPnll+jUqRO6d+/e6DHIeNgzQ4365ptv8Pfff9fZ/uSTT+L111/H1q1bMWzYMDzyyCOwsbHBF198gbKyMixfvrzZ9+Xr64snn3wS7733HiZPnowJEyYgNjYWf/31F7y8vBr9JRUVFYU1a9ZgwYIFGDBgAJycnHDLLbegR48euOGGG7Bw4ULk5OTAw8MDq1evlr6gdGxtbfH666/jwQcfxJgxYzB9+nQkJSVhxYoVdXJm7rnnHqxduxYPPfQQdu7ciaFDh0KtViMhIQFr167F5s2bDSZONsTFxQWfffYZ7rnnHvTr1w8zZsyAt7c3UlJS8Mcff2Do0KH4+OOP4eLighEjRmD58uWoqKhAhw4dsGXLFiQlJRl8TADt1PUZM2bA1tYWt9xyCxwdHTF37ly89dZbmDt3Lvr37489e/bg7NmzTW5vWFgYXn/9dSxcuBDJycm49dZb4ezsjKSkJPzyyy+YN28ennnmmQaPYWtri9tvvx2rV69GUVER3n333TqPSVPP1ZC5c+fioYcewh133IEbb7wRsbGx2Lx5c51eg2effRa//vorbr75ZsyePRtRUVEoKirCyZMnsX79eiQnJ8PLywtz585FTk4OxowZg8DAQFy8eBEfffQR+vTpI+VPGDJp0iS8//77mDBhAu666y5kZGTgk08+QefOnXHixIlGz6NPnz5QKBR4++23kZeXB5VKhTFjxsDHx6dJr5mWeOONN7BlyxaMHDlSKj9w9epVrFu3Dvv27YObmxtuvvlmLF26FPfddx+GDBmCkydP4ocffqjzfmmOnj17Yvz48XpTswFgyZIljd62td4btb3wwgv46aefMHHiRDzxxBPw8PDAt99+i6SkJPz8888WW7zTaphmEhVZAt001/ouqampQgghjh07JsaPHy+cnJyEg4ODGD16tDhw4IDBY9We4mloenVlZaV45ZVXhJ+fn7C3txdjxowRp0+fFp6enuKhhx5q8LaFhYXirrvuEm5ubgKA3jTtCxcuiLFjxwqVSiV8fX3Fiy++KLZu3Wpwuuunn34qQkJChEqlEv379xd79uwRI0eO1JuaLYR2au3bb78tevToIVQqlXB3dxdRUVFiyZIlIi8vr8HH19DU7JrnNn78eOHq6irs7OxEWFiYmD17tt501UuXLonbbrtNuLm5CVdXVzF16lRx5coVg1ORX3vtNdGhQwchl8v1pqIWFxeL+++/X7i6ugpnZ2cxbdo0kZGRUe/UbENtFUKIn3/+WQwbNkw4OjoKR0dHER4eLh599FFx5syZBh8DHd3zIJPJpNdVTU09V0NTs9VqtXj++eeFl5eXcHBwEOPHjxfnz5+vM/VWCO1U24ULF4rOnTsLpVIpvLy8xJAhQ8S7774rysvLhRBCrF+/XowbN074+PgIpVIpOnbsKB588EFx9erVRs/z66+/Fl26dBEqlUqEh4eLFStWGJyebGhqthBCfPXVVyI0NFQoFIo6r9umvGZmzZolHB0dG21nTRcvXhT33nuv8Pb2FiqVSoSGhopHH31UmnZcWloqnn76aeHv7y/s7e3F0KFDxcGDB+u8X5ozNfvRRx8Vq1atkh6rvn371nmPNvSabI33hqHXx4ULF8Sdd94p3NzchJ2dnRg4cKD4/fff9fbRfS6tW7dOb3tDpRDo+smEYDYSmb/c3Fy4u7vj9ddfx0svvWTq5hARkRlhvxiZnZKSkjrbdOPdo0aNatvGEBGR2WPODJmdNWvWYOXKlbjpppvg5OSEffv24aeffsK4ceMwdOhQUzePiIjMDIMZMju9evWCjY0Nli9fjvz8fCkp+PXXXzd104iIyAwxZ4aIiIgsGnNmiIiIyKIxmCEiIiKLxmCG2o3ly5cjPDzcqGuzrFy5EjKZTK8isrF16tQJs2fPlv7etWsXZDKZ3krUTaW77fr161uvgc3QqVMn3HzzzSa577Yye/ZsdOrUydTNqNf1vH5aU3x8PGxsbBpdToEIYDBD7UR+fj7efvttPP/886zU2Up+/PHHJi8pQNRc3bt3x6RJk/TWVyKqD2czUbvwzTffoLKyEjNnzjR1U4xuxIgRKCkpgVKpNOr9/Pjjjzh16hTmz59v1PuxNl999VWbrdxs6R566CHcdNNNuHDhgrS+GpEh/IlK7cKKFSswefJk2NnZmbopRieXy2FnZ8ceKDNla2urt2o21W/s2LFwd3fHt99+a+qmkJnjpx1ZvaSkJJw4cQJjx46tc11RURGefvppBAUFQaVSoVu3bnj33XfrrLgrk8nw2GOPYePGjejZsydUKhV69OhhcAHOmmbNmgUvLy+DqxmPGzcO3bp1a/D2586dwx133AE/Pz/Y2dkhMDAQM2bMMLgyuE59OQ+ffPIJQkNDYW9vj4EDB2Lv3r0YNWqUwarKGo0Gy5YtQ2BgIOzs7BAdHY3z589L148aNQp//PEHLl68CJlMBplM1qQ8kFWrVmHgwIFwcHCAu7s7RowYgS1bttTZb9++fRg4cCDs7OwQGhqK7777Tu/6nJwcPPPMM4iMjISTkxNcXFwwceJExMbGGnws1q5d2+D5NPcxKisrw6JFi9C5c2eoVCoEBQXhueeea3DVbJ3aOTPJycmQyWR499138eWXXyIsLAwqlQoDBgzA4cOHGzzWkSNHIJPJDH7Zb968GTKZDL///jsA4OLFi3jkkUfQrVs32Nvbw9PTE1OnTm1SflftvCyd63lsdAvUurm5wcnJCd26dcOLL76ot4+trS1GjRqFTZs2NdpGat84zERW78CBAwCAfv366W0XQmDy5MnYuXMn7r//fvTp0webN2/Gs88+i8uXL+ODDz7Q23/fvn3YsGEDHnnkETg7O+O///0v7rjjDqSkpMDT09Pgfd9zzz347rvvsHnzZr3E1rS0NOzYsQOLFi2qt93l5eUYP348ysrK8Pjjj8PPzw+XL1/G77//jtzcXLi6ujb5Mfjss8/w2GOPYfjw4XjqqaekFa7d3d0RGBhYZ/+33noLcrkczzzzDPLy8rB8+XLcfffdOHToEADtKsN5eXm4dOmS9Dg5OTk12IYlS5Zg8eLFGDJkCJYuXQqlUolDhw5hx44dGDdunLTf+fPnceedd+L+++/HrFmz8M0330irWPfo0QMAkJiYiI0bN2Lq1KkICQlBeno6vvjiC4wcORLx8fEICAho1vk05zHSaDSYPHky9u3bJ60kffLkSXzwwQc4e/YsNm7c2OTnpaYff/wRBQUFePDBByGTybB8+XLcfvvtSExMhK2trcHb9O/fH6GhoVi7di1mzZqld92aNWvg7u6O8ePHAwAOHz6MAwcOYMaMGQgMDERycjI+++wzjBo1CvHx8XBwcGhRu2tq6mMTFxeHm2++Gb169cLSpUuhUqlw/vx57N+/v84xo6KisGnTJuTn58PFxeW620hWypSrXBK1hZdfflkAEAUFBXrbN27cKACI119/XW/7nXfeKWQymTh//ry0DYBQKpV622JjYwUA8dFHH0nbaq/arFarRWBgoJg+fbrefbz//vtCJpOJxMTEett9/Phxg6vv1lZ7dd/aq4mXlZUJT09PMWDAAFFRUSHtt3LlSgFAb2Vj3W0jIiKkVZGFEOLDDz8UAMTJkyelbZMmTdJblbwh586dE3K5XNx2221CrVbrXafRaPTOBYDYs2ePtC0jI0OoVCrx9NNPS9tKS0vrHCcpKUmoVCqxdOnSZp9Pcx6j77//XsjlcrF37169+//8888FALF///4GH4tZs2bpPW661ZQ9PT1FTk6OtH3Tpk0CgPjtt98aPN7ChQuFra2t3m3LysqEm5ubmDNnjrStuLi4zm0PHjwoAIjvvvtO2mZoNXpDK0gLIeqsjN3Ux+aDDz5ocBX2mn788UcBQBw6dKjRfan94jATWb3s7GzY2NjU6Tn4888/oVAo8MQTT+htf/rppyGEwF9//aW3fezYsXpJiL169YKLiwsSExPrvW+5XI67774bv/76KwoKCqTtP/zwA4YMGYKQkJB6b6vredm8eTOKi4sbP9F6HDlyBNnZ2XjggQdgY1PdGXv33XfD3d3d4G3uu+8+vQTi4cOHA0CD59qQjRs3QqPR4NVXX62TyyOTyfT+7t69u3R/AODt7Y1u3brp3bdKpZKOo1arkZ2dLQ1VHDt2rNnn05zHaN26dYiIiEB4eDiysrKky5gxYwAAO3fubPoDU8P06dP17qupj/n06dNRUVGBDRs2SNu2bNmC3NxcTJ8+Xdpmb28v/b+iogLZ2dno3Lkz3NzcDD5mLdHUx8bNzQ0AsGnTpkaToXWPSVZWVqu0kawTgxlqty5evIiAgAA4OzvrbY+IiJCur6ljx451juHu7o5r1641eD/33nsvSkpK8MsvvwAAzpw5g6NHj+Kee+5p8HYhISFYsGAB/ve//8HLywvjx4/HJ5980mC+jCG68+jcubPedhsbm3rzXGqfq+4LpbFzrc+FCxcgl8vRvXv3RvdtyuOs0WjwwQcfoEuXLlCpVPDy8oK3tzdOnDhh8PFp7Hya8xidO3cOcXFx8Pb21rt07doVAJCRkdHoORrS0se8d+/eCA8Px5o1a6Rta9asgZeXlxREANrV6F999VUpP0z3mOXm5jb7NVWfpj4206dPx9ChQzF37lz4+vpixowZWLt2rcHARlTlr9UOeolqYs4MWT1PT09UVlaioKCgTuDSHAqFwuB20cjyZt27d0dUVBRWrVqFe++9F6tWrYJSqcS0adMavc/33nsPs2fPxqZNm7BlyxY88cQTePPNN/HPP/8YzHVpLS0917a67zfeeAOvvPIK5syZg9deew0eHh6Qy+WYP3++wS/E1jwfjUaDyMhIvP/++wavDwoKavYxgetr4/Tp07Fs2TJkZWXB2dkZv/76K2bOnKnXy/T4449jxYoVmD9/PgYPHgxXV1fIZDLMmDGj0d6R+gIJtVqt1+6mPjb29vbYs2cPdu7ciT/++AN///031qxZgzFjxmDLli16x9QFc15eXo0+DtR+MZghqxceHg5AO6upV69e0vbg4GBs27atTpCTkJAgXd9a7r33XixYsABXr17Fjz/+iEmTJtU7xFNbZGQkIiMj8fLLL+PAgQMYOnQoPv/88yavIq47j/Pnz2P06NHS9srKSiQnJ+s9Js3RnF/KYWFh0Gg0iI+PR58+fVp0fzWtX78eo0ePxtdff623PTc3t0Vfes15jMLCwhAbG4vo6Giz6S2YPn06lixZgp9//hm+vr7Iz8/HjBkz9PZZv349Zs2ahffee0/aVlpaitzc3EaP7+7ubnC/ixcvIjQ0VPq7OY+NXC5HdHQ0oqOj8f777+ONN97ASy+9hJ07d+rNPExKSoJcLpd6d4gM4TATWb3BgwcD0OZF1HTTTTdBrVbj448/1tv+wQcfQCaTYeLEia3WhpkzZ0Imk+HJJ59EYmIi/u///q/R2+Tn56OyslJvW2RkJORyeZOmAOv0798fnp6e+Oqrr/SO98MPP7R42AgAHB0dmzw8ceutt0Iul2Pp0qV1egFa0juiUCjq3G7dunW4fPlys48FNO8xmjZtGi5fvoyvvvqqznFKSkpQVFTUojZcj4iICERGRmLNmjVYs2YN/P39MWLECL19DD1mH330EdRqdaPHDwsLwz///IPy8nJp2++//47U1FS9/Zr62OTk5NS5Xhfk1n5tHz16FD169GjW7D1qf9gzQ1YvNDQUPXv2xLZt2zBnzhxp+y233ILRo0fjpZdeQnJyMnr37o0tW7Zg06ZNmD9/fqtWHPX29saECROwbt06uLm5YdKkSY3eZseOHXjssccwdepUdO3aFZWVlfj++++hUChwxx13NPm+lUolFi9ejMcffxxjxozBtGnTkJycjJUrVyIsLKzFvQtRUVFYs2YNFixYgAEDBsDJyQm33HKLwX07d+6Ml156Ca+99hqGDx+O22+/HSqVCocPH0ZAQADefPPNZt33zTffjKVLl+K+++7DkCFDcPLkSfzwww96vQTN0ZzH6J577sHatWvx0EMPYefOnRg6dCjUajUSEhKwdu1abN68Gf37929RO67H9OnT8eqrr8LOzg73339/nUTrm2++Gd9//z1cXV3RvXt3HDx4ENu2bau3rEBNc+fOxfr16zFhwgRMmzYNFy5cwKpVq+q8R5r62CxduhR79uzBpEmTEBwcjIyMDHz66acIDAzEsGHDpONVVFRg9+7deOSRR1rnQSLrZappVERt6f333xdOTk51pqcWFBSIp556SgQEBAhbW1vRpUsX8c477+hNFxZCOzX70UcfrXPc2lNWa0/Nrmnt2rUCgJg3b16T2pyYmCjmzJkjwsLChJ2dnfDw8BCjR48W27Zta7ANhqbWCiHEf//7XxEcHCxUKpUYOHCg2L9/v4iKihITJkyoc9va08F104dXrFghbSssLBR33XWXcHNzEwCaNE37m2++EX379hUqlUq4u7uLkSNHiq1bt+qdy6RJk+rcrvYU4NLSUvH0008Lf39/YW9vL4YOHSoOHjxYZ7/mnE9THyMhhCgvLxdvv/226NGjh3QuUVFRYsmSJSIvL6/Bx6C+qdnvvPNOnX0BiEWLFjV4PJ1z584JAAKA2LdvX53rr127Ju677z7h5eUlnJycxPjx40VCQkKTXz/vvfee6NChg1CpVGLo0KHiyJEjdR5vIZr22Gzfvl1MmTJFBAQECKVSKQICAsTMmTPF2bNn9Y71119/CQDi3LlzTXoMqP2SCdEGGX1EJpaXl4fQ0FAsX74c999/v0nasGnTJtx6663Ys2eP3tRjU9FoNPD29sbtt99ucFiA+BiZ2q233gqZTCbNBCSqD3NmqF1wdXXFc889h3feecdki/x99dVXCA0N1etGbyulpaV18iW+++475OTkGFzOoD3iY2ReTp8+jd9//x2vvfaaqZtCFoA9M0RGtnr1apw4cQJvvvkmPvzwwzpF+trCrl278NRTT2Hq1Knw9PTEsWPH8PXXXyMiIgJHjx41+grbloCPEZHlYjBDZGQymQxOTk6YPn06Pv/8c73aH20lOTkZTzzxBP7991/k5OTAw8MDN910E9566y34+Pi0eXvMER8jIsvFYIaIiIgsGnNmiIiIyKIxmCEiIiKL1i6K5mk0Gly5cgXOzs5mU36ciIiIGiaEQEFBAQICAuoUgqypXQQzV65cafHib0RERGRaqampDS6u2y6CGd0igqmpqXBxcTFxa4iIiKgp8vPzERQUpLcYsCHtIpjRDS25uLgwmCEiIrIwja7C3kbtICIiIjIKBjNERERk0RjMEBERkUVjMENEREQWjcEMERERWTQGM0RERGTRGMwQERGRRWMwQ0RERBaNwQwRERFZNAYzREREZNEYzBAREZFFYzBDREREFo3BjJVaeyQV+89nmboZRERERtcuVs1ub+Kv5OO59ScAAMlvTTJxa4iIiIyLPTNWKC2/xNRNICIiajMMZqyQTCYzdROIiIjaDIMZKyRnMENERO0IgxkrJK8RywghTNcQIiKiNsBgxgrV7JnRMJYhIiIrx2DGCtUcZarUaEzXECIiojbAYMYK1eyZqVSza4aIiKwbgxkrpBfMcJyJiIisHIMZK1RzmEnNYIaIiKwcgxkrx5wZIiKydgxmrJCmRm8Me2aIiMjaMZixQjXjFyYAExGRtWMwY4VqFspjzwwREVk7BjNWSK9nhsEMERFZOQYzVkjDnhkiImpHGMxYoZrBTIWas5mIiMi6MZixQjXXlmTPDBERWTsGM1aoZgDDnBkiIrJ2DGasEHNmiIioPWEwY4X0ZzMxZ4aIiKwbgxkrxDozRETUnjCYsUKsM0NERO0JgxkrpJczw+UMiIjIyjGYsUI1gxn2zBARkbUzWjCzbNkyDBkyBA4ODnBzczO4zxNPPIGoqCioVCr06dPH4D5CCLz77rvo2rUrVCoVOnTogGXLlhmr2VZBMAGYiIjaERtjHbi8vBxTp07F4MGD8fXXX9e735w5c3Do0CGcOHHC4PVPPvkktmzZgnfffReRkZHIyclBTk6OsZptFTg1m4iI2hOjBTNLliwBAKxcubLeff773/8CADIzMw0GM6dPn8Znn32GU6dOoVu3bgCAkJCQ1m+sldErmsecGSIisnJmnTPz22+/ITQ0FL///jtCQkLQqVMnzJ07t9GembKyMuTn5+td2hMuZ0BERO2JWQcziYmJuHjxItatW4fvvvsOK1euxNGjR3HnnXc2eLs333wTrq6u0iUoKKiNWmwemABMRETtSbOCmRdeeAEymazBS0JCQqs1TqPRoKysDN999x2GDx+OUaNG4euvv8bOnTtx5syZem+3cOFC5OXlSZfU1NRWa5M5U2sE8ksr9OrMqJkATEREVq5ZOTNPP/00Zs+e3eA+oaGh19MePf7+/rCxsUHXrl2lbREREQCAlJQUKY+mNpVKBZVK1WrtsBS3f3YAsam5mD+2i7SNPTNERGTtmhXMeHt7w9vb21htqWPo0KGorKzEhQsXEBYWBgA4e/YsACA4OLjN2mEpYlNzAQA7EzKkbcyZISIia2e02UwpKSnIyclBSkoK1Go1YmJiAACdO3eGk5MTAOD8+fMoLCxEWloaSkpKpH26d+8OpVKJsWPHol+/fpgzZw7+85//QKPR4NFHH8WNN96o11tDQGmFWvq/i72t9P8KzmYiIgPir+TjoVVHMWdoJ8weylmiZNmMFsy8+uqr+Pbbb6W/+/btCwDYuXMnRo0aBQCYO3cudu/eXWefpKQkdOrUCXK5HL/99hsef/xxjBgxAo6Ojpg4cSLee+89YzXbYmUXlUv/L6+szpNhzgwRGXLTf/cCABb/Fs9ghiye0YKZlStXNlhjBgB27drV6HECAgLw888/t06jrFhWQZn0/8u5JdL/mTNDRLVx+JmsjVlPzaamyyqsDmau5pVK/+eHFhHVVlapbnwnIgvCYMZK1Axm9CoAM5gholpKK/SHn4Xg5wRZNgYzViKzxjBTTeyZIaLaak4YADhRgCwfgxkrkVVYbnA712YiotrKKvV7ZsrVnChAlo3BjJXILKyvZ4YfUkSkr3bPTHklPyfIsjGYsRJZ9QwzMWeGiGqrHcwwIZgsHYMZK1FfzwyHmYiottoJwOyZIUvHYMZK6HpmOrjZ621nzwwR1VZaWbtnhsEMWTYGM1agrFKN/NJKAECvQFe965gzQ0S1lTFnhqwMgxkrkF01k8lWIUOEv4vedeyZIaLaavfEMGeGLB2DGSugK5jn6aiqM8zEOjNEVFvdBGD2zJBlYzBjBXQF87yclejgzpwZImoYE4DJ2jCYsQK6nhlvJ/bMEFHj2DND1obBjBXQVf/1clLB18UOMln1deyZIaLa2DND1obBjBWoHmZSQWkjh4+zSrqOs5mIqDZOzSZrw2DGCuiGmbyctEFMQI2hJi4gR0S1lbFnhqwMgxkrIPXMOCkB6AczzJkhotpq98yUc2o2WTgGM1ZASgCuGl4a3tlLuo45M0RUGxOAydowmLECugRg76phphkDO+Lz/+sHgDkzRFQXh5nI2jCYsXDllRrklVQAqM6ZAQAnlS0ALjRJRHWxZ4asDYMZC5ddpB1ispHL4GpvK21XyLXzs5kzQ0S11cmZUTOYIcvGYMbC6ZJ/PZ2UkMurC8zYKBjMEJFhtevM1F54ksjSMJixcLWTf3V0PTNMACai2movLMmeGbJ0DGYsXFZBdfXfmmx0wQw/pIioljo9M8yZIQvHYMbCZdYqmKfDnhkiqg8TgMnaMJixcLWr/+rYyLVPLXNmiKg2rs1E1obBjIWrXf1XR5cAzJ4ZIqqtdsIve2bI0jGYsXD1JQDbcGo2EdVDNzX7xZvCAXA5A7J8DGYsXO3qvzrVOTP8xUVE1dQaIS1A62KnrU3FYSaydAxmLJyUM1OnZ4Y5M0RUV81p2bpCmxxmIkvHYMaClVdqkFtcdykDgLOZiMiwmsm/zuyZISvBYMaC6ZYyUMhlcKuxlAFQnTMjBKBhQENEVXTTsm0VMtgrtV8B7JkhS8dgxoLpCuZ5OuovZQAACkX13xXMmyGiKrpgxs5GAaVCAYA9M2T5GMxYsPpmMgHVPTMA82aIqJpumEllq4DKVtczw9lMZNkYzFiw+qr/AtU5MwDzZoiomm5atp2tHEqF9iuAPTNk6RjMWLD6qv8CgK28+qlVqxnMEJFWWVXPjF2NnhkuNEmWjsGMBZOq/zor61wnl8sgq+qcYc8MEekY6pmpUAtOFCCLxmDGgtVXME+HVYCJqLaymgnANtVfAeydIUvGYMaCZRXUnwAMsAowEdVVnQAsh8pGIW0vq+DnBFkuBjMWrKGcGYBVgImorppTs21rlHAoU3NGE1kuBjMWrLFgRtczU8EEYCKqIgUztgrIZDKobDijiSwfgxkLVaHW4Jq0lEHdBGCAOTNEVJeu2q9uJpMub4ZVgMmSMZixUNlVyb8KuQzuDoaDGebMEFFtpTWmZgNgzwxZBQYzFko3xGRoKQMd9swQUW3S1GwbXTCj/Zc9M2TJGMxYqIaq/+rYVNWQYJ0ZItLR5czUHmZizwxZMgYzFipLKpjXQDDDnhkiqkUaZrLRH2bi+kxkyRjMWKjqnhnD+TJAjZwZzmYioipS0bxaPTMl5QxmyHIxmLFQWQUNV/8FqoMZ9swQkU71cgbanpkQL0cAwLGUXFM1iei6MZixULoE4Pqq/wKAjYKzmYhIX/VCk9qP/zHhPgCAHQnpJmsT0fViMGOhGiuYBwCKqgrAHGYiIp3aPTOjuvpAIZfhbHohUnOKTdk0ohZjMGOhmhLM2Eh1ZhjMEJGWtDZTVQKwq4Mt+ge7AwC2n2bvDFkmBjMWKlOazdR4AjBzZohIp/bUbAAYG+ELANiekGGSNhFdLwYzFkh/KYOm9MwwZ4aItGouNKkzJkKbN/NPYjYKyypN0i6i68FgxgLlFDW+lAFQXTSPPTNEpFNaKwEYAMK8nRDi5YgKtcDes5mmahpRizGYsUC6ISYPR6U0lGQIc2aITEcIYZaF6HTLFugSgHV0s5o41ESWiMGMBWpK8i/AnBkiU3p41TFEvPI3Znx5EN/sS8Kla+YxU6i6aJ5+MBNdNdS0MyGDnxlkcWxM3QBqPin5t4HqvwB7ZohMJTmrCH/HpQEA/knMwT+JOVj6ezx6BLhgXHc/jO/pi26+zpDJ6u9ZNZbqqdn6v2UHdPKAs50NsovKEXspF/06urd524haisGMBcoqbLz6L1CjZ0bNBGCitrTh2CUAwMBOHhjf0w+b49JwJDkHcVfyEXclHx9sO4uOHg4Y38MX43r4oV9H9waHjFuLWiNQUVV3SmWj3zNjq5BjZFdv/H7iKrafTmcwQxaFwYwFakr1X4A9M0SmoNEI/HzsMgDg/wYHY3LvANw/LATZhWXYfjoDW+LTsOdcFlJyivHV3iR8tTcJXk5KjI3wxbgevhgS5lVnCKi16GYyAXV7ZgDtUJM2mMnAs+PDjdIGImNgMGOBmp4zU1UBmMEMUZv5Jykbl3NL4Gxng3HdfaXtnk4qTBsQhGkDglBUVok9ZzOxJT4d20+nI6uwHKsPp2L14VQ4KhUY1c0H43r4YnS4D1zsbFutbXrBjE3dgGlUVx/IZUBCWgEuXStGoLtDq903kTExmLFAUjDTQME8oLpnhsl8RG1n/VHtENPNvQLq7WFxVNlgYqQ/Jkb6o0KtwaHEHGyOS8OW+DSk55fhj5NX8cfJq7BVyHBDqCfG9/DDjd194etid11tK62ayaRUyCE3MKzl7qhEVLA7Didfw86EDNwzuNN13R9RW2EwY4GqE4AbGWbSLTTJtZmI2kRRWSX+PqVN/L0zKrBJt7FVyDGsixeGdfHCksk9cPJyXlVgk47zGYXYey4Le89l4eWNp9AnyA3je/hhfA9fhHo7Nbt9ZQaq/9Y2JtwXh5OvYdtpBjNkORjMWCBdAnCjwYzUM8MEYKK28OfJqyguVyPEyxH9Oro1+/ZyuQy9g9zQO8gNz00Ix4XMQmyJS8eW+DQcT8lFTKr28vbfCejs44Rx3X0xvocfegW6NmlmVHXBvPpzcsZG+ODtvxNw8EI2isoq4aji1wSZP75KLUylWoNrxVWzmRpJAGbODFHb+rlqFtOdUYGtMu06zNsJD49ywsOjwpCeX4qt8enYHJeGgxeycT6jEOczCvHprgvwc7HDuB6+GNfdD4NCPWCrMNzzUt+07Jo6+zghyMMeqTkl2Hc+C+N7+F33eRAZG4MZC5NTVA4hALkMDS5lAFQPMzFnhsj4UnOK8U9iDmQy4La+HVr9+L4udvi/G4LxfzcEI6+kArvOZGBLXDp2nslAWn4pvjt4Ed8dvAgXOxtER/hiXHdfjOzmDQdl9ce8tMikgeRfHZlMhuhwX6w8kIwdpzMYzJBFYDBjYTILdUsZqBqtS6Hg1GyiNrOhajr20DAvBLjZG/W+XO1tMaVPB0zp0wGlFWocuJCFzafSse10OrKLyvHL8cv45fhlqGzkGN7FC+N6+CE63AdlBtZlMiQ6wgcrDyRje0IGNBphMFmYyJwwmLEwTa3+C9SoM8OieURGJYSQhpjuiGr9XpmG2NkqMCbcF2PCfaHWCBxLuYbNp9KwOT4NqTkl2HY6A9tOZ0AugzTV2tC07JoGhXjCUalAVmEZTl7OQ+8gtzY4E6KWYzBjYaTqv43kywDsmSFqK4eTryElpxhOKhuTDsso5DIM6OSBAZ088NKkCCSkFUgJxHFX8pGSo10fqrGkXqWNHCO6euOvU2nYfjqdwQyZPQYzFkaq/tvITCaAdWaI2sr6o6kAgJsi/fRyVExJJpMhwt8FEf4ueHJsF6TmFGNrfDqOpVzD3YOCG719dISvNphJyMCCcd3aoMVELWce7zpqsizdMFOTemY4m4nI2IrLK/HHiasAgDujgkzcmvoFeThgzrAQzEFIk/Yf1c0bMhkQdyUfV/NK4O9q3DwgouvRcBbYdVi2bBmGDBkCBwcHuLm5GdzniSeeQFRUFFQqFfr06WNwn82bN+OGG26As7MzvL29cccddyA5OdlYzTZ71UsZNJ4zY8vZTERGtzkuDUXlanT0cMCATtazOKOXkwp9q4aXdiRkmLYxRI0wWjBTXl6OqVOn4uGHH25wvzlz5mD69OkGr0tKSsKUKVMwZswYxMTEYPPmzcjKysLtt99ujCZbhMwmrssEMGeGqC3oli+4o1/r1JYxJ9ER2rWltp9mMEPmzWjDTEuWLAEArFy5st59/vvf/wIAMjMzceLEiTrXHz16FGq1Gq+//jrkVUMmzzzzDKZMmYKKigrY2rbeAmyWIqugadV/AVYAJjK2y7klOHAhGwBwe7+2ncXUFqIjfPDO5jPYfz4LJeVq2CuNs5o30fUyWs9Ma4iKioJcLseKFSugVquRl5eH77//HmPHjm0wkCkrK0N+fr7exVpICcDNyZnh2kxERvHLsUsQArgh1ANBHta3wnQ3X2d0cLNHWaUGBy5kmbo5RPUy62AmJCQEW7ZswYsvvgiVSgU3NzdcunQJa9eubfB2b775JlxdXaVLUJD5JuU1R6Vag5zilvTMMJgham3a2jLaQnnmnPh7PWQyGaIjfAAA2zjURGasWcHMCy+8AJlM1uAlISGh1RqXlpaGBx54ALNmzcLhw4exe/duKJVK3HnnnRCi/i/ohQsXIi8vT7qkpqa2WptMKae4eikDD8fGE4B1OTMVDGaIWt2xlGtIyiqCg1KBiT2tt+T/mHBtMLMjIb3Bz10iU2pWzszTTz+N2bNnN7hPaGjo9bRHzyeffAJXV1csX75c2rZq1SoEBQXh0KFDuOGGGwzeTqVSQaVqvOfC0uiq/3o4KhtdygCouTYTc2aIWtv6o9pemYk9/a16ZekbQj3hoFQgPb8McVfy0bODq6mbRFRHs96B3t7e8Pb2NlZb6iguLpYSf3UUCm0CmqYdfkHrqv82ZYgJqDGbiTkzRK2qtEKN32OvAGj75Qvamp2tAsM6e2FLfDq2n85gMENmyWg5MykpKYiJiUFKSgrUajViYmIQExODwsJCaZ/z588jJiYGaWlpKCkpkfYpL9d+aU+aNAmHDx/G0qVLce7cORw7dgz33XcfgoOD0bdvX2M13WzpCuY1JfkXYM4MkbFsiU9HQVklOrjZ44YQT1M3x+jG6qZoJ6SbuCVEhhmtb/TVV1/Ft99+K/2tCz527tyJUaNGAQDmzp2L3bt319knKSkJnTp1wpgxY/Djjz9i+fLlWL58ORwcHDB48GD8/fffsLdvf9Uos5pRYwYAbFgBmMgoqmvLdGgXK0qPCtf2yJ+4lIeM/FL4uNiZuEVE+owWzKxcubLBGjMAsGvXrkaPM2PGDMyYMaN1GmXhmlP9FwAUDVQALq/UIKuwDAFu7S8oJLoeaXml2HcuEwBwR1SgiVvTNnyc7dA7yA2xqbnYkZCBGQM7mrpJRHrMemo26dMlADe9Z8ZwBeB957Iw6I1tGPLWDuw5m9m6jSSycr8cvwyNAAZ0ckewp6Opm9NmoqtmNW3n0gZkhhjMWJCWJgDXns205kgqrhVXAACSsopasYVE1k1bW0Y7xHRnO+mV0dHVm9l3LgulFWoTt4ZIH4MZC9Kc6r9A/Tkz14rKpf8zn4ao6WIv5eF8RiHsbOW4KdLf1M1pU939XeDvaoeSCjUOJmabujlEehjMWJDmJgDXNzX7WnF1MMMaNERNt/6otgDnhB5+cLZrX2vDyWQyqYDe9tOc1UTmhcGMhVBrBHKqelS8nJuWAFzf1OzcqiEmAKhgDRqiJimtUOO32KsA2k/ib226oaYdpzNYDZjMCoMZC5FdVAaNAGQywMOhibOZpARg/d4X/Z4ZfiARNcX20xnIK6mAv6sdhoR5mbo5JjEkzAt2tnJcySvF6asFpm4OkYTBjIXIKtAGIB4OStgomva02RiYml1WqUZxeXXyHnNmiJpGl/h7W98OTVpOxBrpqgED2rWaiMwFgxkL0dzkX8BwAnDNISZAuxI3ETUso6AUu8+2r9oy9YmuqgbMVbTJnDCYsRDNTf4FauTM1MiLqTnEBHCYiagpNh2/ArVGoF9HN4R5O5m6OSalSwKOvZQr1b4iMjUGMxaiudV/gZo5MzWCmaJaPTMMZogaJISoXr6gnffKAICvix0iO7hCCGDnGfbOkHlgMGMhmlv9FzCcM1O7Z4bDTEQNi7uSjzPpBVDayHFzrwBTN8cs6HpndnCoicwEgxkLIVX/bUbOjK5npqLGbKY6wQx7ZogapOuVGdfdF6727au2TH10q2jvPZeJskpWAybTYzBjIaQE4GblzGifXiEATVXQUjsBmDkzRPUrr9RgU8xlAO1v+YKG9AhwgY+zCkXlahxKzDF1c4gYzFgKaZipBT0zQHUPTM2lDGpuJ6K6diRk4FpxBXycVRjexdvUzTEbcrlMKqDHasBkDhjMWIjqRSabngBsUyOY0fXA6BaY1B2HOTNE9ZNqy/Rrv7Vl6jMmXDvUtD2B1YDJ9BjMWADtUgYtGGZS1OyZ0QYtucX6K2+zZ4bIsOzCMuxM0Ca43tmPQ0y1DevsBZWNHJeuleBseqGpm0PtHIMZC5BTVF69lIFjc3pmqp/e6p4Z/WCGOTNEhm2KuYJKjUDvQFd08XU2dXPMjr1SgSFhngCA7awGTCbGYMYC6JJ/m7OUAQDU7BWvrJUArKskzIUmiQxjbZnG6aoBb+cUbTIxBjMWoCXVfwFAJpPVWTk7p6pnRhfMqDXMmSGqLf5KPuKv5kOpkOMW1papl67ezLGUa8ipNbmAqC0xmLEA1TOZmj7EpFOzCrBaI5BXUisBmMNMRHXoEn+jI3zg3oyh3fYmwM0e3f1dtNWAE9g7Q6bDYMYCtLRnBqie0VSp1iC/pAK6SQdSAjCHmYj0VKg12HictWWaSjdFeweDGTIhBjMWoHpadvODmZo9M7rkXyeVDextFQCYAExU2+4zmcguKoeXkwojurK2TGN0eTO7z2aivJLD1mQaDGYsQFbVMJN3Mwrm6egShtUaIdWYcXOwrRHk8MOHqCZd4u+tfQJg24yE+/aqVwdXeDkpUVhWicPJrAZMpsF3qgXIvI5hJiloUQupxoy7g1KqQcOcGaJq14rKpWnGnMXUNHK5DKO7aYeatrEaMJkIgxkLUL1idvMTEW1rzGaq2TOjq0HDnBmiar/GXkGFWqBHgAsi/F1M3RyLUXOKNqsBkykwmLEA15Uzo6geTtLrmak1ZZuIqmcxMfG3eYZ38YJSIUdKTjEuZLIaMLU9BjNmTm8pg5bkzMirc2Z0dSDca+TMVDBnhggAcDa9ACcu5cFGLsPk3qwt0xyOKhvcoKsGzAJ6ZAIMZszcteKWLWWgoz+bSTvM5O6o1EsMJiLg56rE3zHhPvBsQS9oexcdrltFm8EMtT0GM2ZOV2PG3UHZopkVNYeTDA0zMWeGSFuHaUNVbRkm/raMrhrwkYs50mcNUVthMGPmrif5F6jumalQa6Q6M5yaTaRv7/ksZBaUwcNRKc3MoeYJ8nBAN19naASw60ymqZtD7QyDGTN3PdV/gdo9M1XDTDV6eTjMRFRdW2Zy7wAobfix2FK6asDbWQ2Y2hjftWYuq6DlM5kAwxWA3R2UetuJ2rO84gpsjdPWR+EspuujC2Z2nclAhZq9vtR2GMyYOV3PTEtmMgHQqyejX2eGOTNEAPDbiSsoV2sQ7ueMHgGsLXM9+gS5w8NRiYLSShxJvmbq5lA7wmDGzF1P9V8AUqXfwrIKad0Ud0clc2aIquiGmO6MCoRMJjNxayybQi7DqG7a9ay2sxowtSEGM2autRKAdcexVcjgqFQwZ4YIwPmMQsSk5kIhl2FKnw6mbo5VGFtVDZiraFNbYjBj5qTqvy0eZpLpHcfNQQmZTFZjlpNg+XFqt3QVf0d19W7xUC7pG97FC7YKGRKzipDIasDURhjMmDkpZ6bFCcDap1g3XOXhoO3h0QU5AMDOGWqP1BqBX45pa8sw8bf1ONvZYlCIthowe2eorTCYMWOaGksQtDwBuKpnpmqYyc3BVrtdUR3MMG+G2qP957OQll8KV3tbjIlgbZnWpCugx1W0qa0wmDFj14rLpZyWlixlAFQvNJlZo5IwUD3LCeCMJmqfdENMU/oEQGWjMHFrrItuivbh5GvIK6kwcWuoPWAwY8aqAxDbFi1lANTtmXF31PbMKOQ1e2YYzFD7kl9agb9PpQEA7ujHIabWFuzpiM4+TlBrBHafZTVgMj4GM2bsegvmAdVBS35pJQBtAjCgnzPDGU3U3vx54irKKjXo4uOEXoGupm6OVdL1zuzgUBO1AQYzZux6lzIA9IMWQNvLAwByuQy6qypZqZPaGV1tmTtYW8ZoosO1U7R3nsnkZwwZHYMZM3a91X8BwKbW8JSuZwaoUR2YPTPUjiRnFeHIxWuQy4Db+rK2jLH06+gGNwdb5JVU4FhKrqmbQ1aOwYwZu97qv4ChnpkawYyiehFKovZCl/g7vIs3fF3sTNwa62WjkGNUV1YDprbBYMaMSdV/nVs2kwnQT/QFqoeZal7HBeGovdBoBDawtkybia6qBsxVtMnYGMyYMan6b2v2zDgq61zHnhlqL/5JzMbl3BI429ngxu6+pm6O1RvR1Rs2chnOZxTiYnaRqZtDVozBjBnTTaduafVfoLoCsI7+MBNzZqh90SX+3tI7AHa2rC1jbK72thjQyQMAsP00e2fIeBjMmLFWSQCu0TMjk2k/XGpfx6J51B4UllXir6raMhxiaju6KdrbE5g3Q8bDYMZMaTQC2UWtV2cGAFzsbPX+1v2fyxlQe/DnyasoqVAj1MsRfYPcTN2cdkOXN3MoMQcFpawGTMbBYMZM1VzKwNOp5QnANXtmaib/ApCqCjNnhtqDn1lbxiRCvBwR6uWISo3AnrNZpm4OWSkGM2ZKl/zrdh1LGQD6dWZq1pgBavbMMJgh65aaU4xDSTmQyYDb+7G2TFvjUBMZG4MZM9Ua1X+BhntmmDND7YWutsywzl7wd7U3cWvanzFV1YB3nclkTzAZBYMZMyUl/15nMKOyrX6Kayb/AtVF85gzQ9ZMoxFSMMNFJU2jfyd3uNjZIKeoHDGp10zdHGplv8ZekYZxTYXBjJmqLph3fcGMLvkOADwc9Y+lm7bNX0pkzQ4n5yA1pwROKhuM7+Fn6ua0S7YKOUZ2qxpq4hRtq1JcXoknfjqOp9fFIre43GTtYDBjpqqXMmh58i8AdHCzx4cz+qBXoCumDdD/VWojVQBmMEPWS1dbZlKkP+yVrC1jKmMjGMxYo8vXSqT/Xys23Ww1G5PdMzUoq+D6p2XrTOnTAVP61E16ZAVgsnbF5ZX48+RVAMCd/TnEZEoju3pDIZfhTHoBUnOKEeThYOomUSu4VCOYMeXSOOyZMVOtlTPTEObMkLX7+1QaisrVCPZ0QP9gd1M3p11zc1Aiquo52MG1mqxG6rVi6f8aYbofxgxmzFRrVP9tjC5nhrOZyFrVTPxlbRnTiw7XTdFmMGMtUrKrgxlT9vIzmDFTrTU1uyG2HGYiK3Y5twQHLmQDAG7ry9oy5kA3IeGfC9koLKs0cWuoNfxvX5L0f1N28jOYMUMajaheMdv5+hKAG8KieWTNfjl2CUIAg0M9mZ9hJsK8HRHs6YBytQb7zrEasLVRc5iJasotqaheysCROTNEzSWEkGYx3cFFJc2GTCZDdFUBve2nWQ3YGtSoy2rSXn7OZjJDuiEmV3tbKG2MF2/aMGeGrNTRi9eQnF0MB6UCE3uytow5iY7wwTf7k7DzTAY0GgG5nLlMlmbP2Uw8sfo4SsrVqBm/MAGY9GQVGD/5F+DUbLJeusTfmyL94ajibzZzMqCTB5xVNsgqLEfspVxTN4da4L2tZ5FbXIGySv1efQ0TgKmm1iqY1xhdzkwFh5nIipRWqPF7rLa2DJcvMD9KGzlGdPUGwCnaligmNRexqblQKuT4e/5w7H9hjHSdKZ9PBjNmSFrKwIgzmYDqFbXVHGYiK7I5Lg0FZZUIdLfHoBAPUzeHDNCtor2N1YAtzncHkwEAN/fyR7ifCzq4VS/cGncl30StYjBjlqSZTMYOZjibiayQLvH39n6BzMcwU6O6+UAmA05fzceV3JLGb0Bm49+kHADAnQYS601ZyonBjBlqi4J5QM2p2RxmIuuQlleK/ee1U37v6MfaMubKw1GJfh211YBZQM+y5BRpf2x3cLdvZM+2xWDGDLXFUgYAYKtgzwxZlw3HL0EjgIGdPBDs6Wjq5lADdENNOzhF22KUVqhRXK4GALg71s3pNGWVbQYzZkiq/mvEgnlA9XIGzJkhayCEwM9VQ0yGusDJvOjqzey/kI3iclYDtgTZVb0ytgoZnA3MEjTlqC6DGTPUZgnAzJkhKxKTmosLmUWws5VjYiRry5i7rr5OCHS3R3mlBvvPZ5u6OdQEOVX5nB6OSoO9MHJr7ZlZtmwZhgwZAgcHB7i5udW5PjY2FjNnzkRQUBDs7e0RERGBDz/8sM5+u3btQr9+/aBSqdC5c2esXLnSmM02KY1GILutEoBZAZisiK62zMSe/nC2szVxa6gx2mrAVQtPcqjJIuQU64IZw99Npky3N2owU15ejqlTp+Lhhx82eP3Ro0fh4+ODVatWIS4uDi+99BIWLlyIjz/+WNonKSkJkyZNwujRoxETE4P58+dj7ty52Lx5szGbbjJ5JRVST4mnkevMsGgeWYvSCjV+jbkCgLVlLIlu4ckdCRkmLbhGTVNctTiok0ph8HoHExaoNOo9L1myBADq7UmZM2eO3t+hoaE4ePAgNmzYgMceewwA8PnnnyMkJATvvfceACAiIgL79u3DBx98gPHjxxuv8SZScykDlY3hF0xr0eXMVDBnhizc9tMZyC+tRICrHQaHeZq6OdREg0I94KhUIKOgDKeu5KFXoJupm0QN0H1TyGr1wbx1eyS+PXgRL94U3vaNqmJ2OTN5eXnw8KgudHXw4EGMHTtWb5/x48fj4MGD9R6jrKwM+fn5ehdL0VbVfwH2zJD1WH80FQBwW78OUskBMn8qGwWGd9FWA97OAnqWo9ZbbMbAjvjryeHwdzXddG2zCmYOHDiANWvWYN68edK2tLQ0+Pr66u3n6+uL/Px8lJQYLrb05ptvwtXVVboEBQUZtd2tqa2Sf4HqnJmUnGKUVqiNfn9ExpCRX4rdZzMBcIjJEo2pmqK9PYF5M+bOhOtINqrZwcwLL7wAmUzW4CUhIaHZDTl16hSmTJmCRYsWYdy4cc2+fU0LFy5EXl6edElNTb2u47UlqfqvkQvmAdU9M0cvXsO9X/9r9PsjMoaNMZehEUBUsDtCvZ1M3RxqptFV1YBPXc5HWl6pqZtDDRBVA03m2PfZ7JyZp59+GrNnz25wn9DQ0GYdMz4+HtHR0Zg3bx5efvllvev8/PyQnq4fsaenp8PFxQX29oa7tFQqFVQq4wcDxtBWBfOA6pwZAPg3Ocfo90fU2oQQ0vIF7JWxTN7OKvQOdENMai52JGTgrkEdTd0kaoQply2oT7ODGW9vb3h7e7daA+Li4jBmzBjMmjULy5Ytq3P94MGD8eeff+pt27p1KwYPHtxqbTAnWQVts5QBUD3MRGSpTl3Ox9n0Qqhs5JjUy9/UzaEWGhvhUxXMpDOYMWNWNczUHCkpKYiJiUFKSgrUajViYmIQExODwsJCANqhpdGjR2PcuHFYsGAB0tLSkJaWhszMTOkYDz30EBITE/Hcc88hISEBn376KdauXYunnnrKmE03mSwTJAATWSpd4u+4Hn5wtWdtGUs1pqoa8L7zWczfM2P1zWYyB0YNZl599VX07dsXixYtQmFhIfr27Yu+ffviyJEjAID169cjMzMTq1atgr+/v3QZMGCAdIyQkBD88ccf2Lp1K3r37o333nsP//vf/6xyWjZQczZTWwwzmd8LkqipyirV2BSrrS3D5QssW4S/MwJc7VBaocGBC1mmbg41whyHmYwazKxcuRJCiDqXUaNGAQAWL15s8Prk5GS944waNQrHjx9HWVkZLly40GjOjiXLKmib6r8ApErDRJZoZ0IGcosr4OuiwrDOXqZuDl0HmUwmzWraxinaZkuY8TiTWU3Nbu+EEMgu0i0yafxgZkCn6no+SgVfCmRZ1h+9DAC4rW8gexmtgFQN+HSGWX9pUjvsmaHmySupkKrxtkXOTGSgK766tz8AQM0PD7IgWYVl2HVG+wv+zqgOJm4NtYbBoZ6wt1UgLb8UcVcsp9Bpe9TucmaoeX45rv2l6WJnY/SlDHT6BLkB0FYB5q8hshSbYq6gUiPQO8gNnX2cTd0cagV2tgoM66IdLtyRwKEmc2TOXxEMZsxEeaUGS36LBwDkl1a22f3WnNHEVQ3IUuhqy9zZj70y1oSraJs3qWie+XXMMJgxF0cumqZonbxGMFOp0ZikDUTNEXclD6ev5kOpkOOW3gGmbg61ojFVwUzspTxkFLAaMDUdgxkzoVtbBgDWPth2BQH1emYYy5AF+Lkq8Xdsdx+4ORg/t4zajo+LHXoFugLQzlYj88JhJmrU7jPaYObDGX0wMMSjkb1bj4I9M2RBKtQabIrRBjOsLWOdoqsK6HEVbfOjC2ZkZjjOxGDGDKTnlyIhrQAyGTC8S+stFdEUCvbMkAXZdSYT2UXl8HJSYUQbv1eobURX1ZvZe47VgM2V+YUyDGbMwp6qIaZeHVzh4di23eYKGXtmyHLoli+4rW8AbFgbySr1CHCBr4sKJRVq/JOYbermUA1mPMrEYMYc6PJlRnZt+1+acrlMykxnrRkyZzlF5dKU3Ts4xGS1ZDKZtFYTh5rMi658hxmOMjGYMTW1RmDvOe1aJCO7mabbXNc7o+bcbDJjv8ZcRoVaoGcHF4T7uZi6OWREY6uGmnYksBqwOTLDWIbBjKmduJSLvJIKuNjZoHegm0naoMubYTBD5uznY1WJv/3YK2PthoR5QWUjx+XcEiSkFZi6OVTFnL8hGMyYmG6IaVgXL5PlAOiCGabMkLk6k1aAk5fzYKuQYXIfFsqzdvZKhbR4KKsBmxHOZqL6mDJfRkcXzDABmMzVz8e0FX/HhPu0eZI8mUb1KtqsBmxuzC+UYTBjUrnF5YhNzQUAjDCDYEbDsWkyQ5VqDTZUDTHdwSGmdkNXbyYmNRdZhWUmbg0B1csZmCMGMya073wWNALo6usEf1d7k7XDRuqZMd8XKrVfe89lIauwDJ6OSoyuKndP1s/P1Q49AlwgBKsBm4vqonmmbYchDGZMSFf115RDTAAg52wmMmO6RSUn9wmALWvLtCvREdreGebNmBvzi2b4yWAiQgjsOacLZkz7a1Npo30ZFJWx2iaZl7ziCmyN1+ZMcPmC9ke3ivaes5koq+Tnk6mZ889dBjMmcia9AOn5ZbC3VaB/J3eTtiXczxkAEJN6zaTtIKrt1xNXUK7WINzPGT0CXE3dHGpjkR1c4e2sQlG5Gv8m5Zi6Oe0eh5moDt0Q0w2hHrCzVZi0LbqFLf9NYjBD5uXnqiEm9sq0T3K5DGO6aXtnWA3YfJhhLMNgxlTMYUq2zoBO2mDmyMUcaJg3Q2bifEYhYlJzYSOX4da+rC3TXukWntyekM5qwCbG2Uykp6isEkeStb0gI7uZfnZGzw6usLdVILe4AuczC03dHCIA1bVlRnXzhpeTysStIVMZ1sULShs5UnNKcC6Dn0+mxGEm0vNPYjbK1RoEedijk6eDqZsDW4UcfTu6AQDHpcksqDUCG45xiIkAB6UNhoR5AuBQk7mQmeFAE4MZE6g5xGQuZaF1Q02HkxnMkOntO5+F9PwyuDnYsrYMSbOatrMasEmZ7yATgxmTqA5mzOdDWpcEfJg9M2QGdIm/U3oHQGVj2gR5Mr0xVfVmjqVcQ05RuYlb045VjTOZyW9wPQxm2lhyVhEuZhfDViHD4KquU3PQt6MbbOQyXMkrxaVrxaZuDrVj+aUV2ByXBgC4g0NMBKCDmz3C/ZyhEcCuMxxqMjUGMyQVyosKdoeTysbEranmoLRBjw7aOh53fXUIT62JwTf7knAkOQfF5ZUmbh21J3+cuIqySg26+johsgNry5DW2Kreme2sBmwy5jzMZD7fpu1E9RIG5jPEpHNHvw6ITc1FSk4xUnKK8ctx7eJ+chnQxccZkYGu6BXoisgOrojwdzF5fRyyTrrlC+7oF2g2OWVkemMifPDxzvPYcyYT5ZUaqXI5tR1pNpMZJgAzmGlDZZVqHLiQDcA86svUdu/gTrgp0h8nL+fh5KU8nLiUh5OXc5GeX4Yz6QU4k14gfdHYyGXo6uusDW4CXdGrgxu6+TnzA4auS1JWEY5evAa5DLiNtWWoht6BbvB0VCK7qBxHknMwpLOXqZvUfplfLMNgpi0dTb6Gkgo1vJ1ViPB3NnVzDPJyUmF0Nx+MrlH/Jj2/VBvcXM7DyUu5OHEpD9lF5Yi/mo/4q/lYfTgVAKBUyBHu74zIDroeHDd08XXi4oDUZLrE3xFdveHjYmfi1pA5UchlGB3ug/VHL2F7QgaDGRMw56KFDGbakG4W04gu5jMluyl8Xezg290OY7trx6yFELiaVyr13Gj/zUNucQVOVPXo/HBIe1uVjRzdA1zQq4MrIgPd0CvQFWHeTlDILef8qW1oWFuGGhGtC2ZOp+PlSREW9TlqDXShjDk+6gxm2pA0Jbub+Q0xNYdMJkOAmz0C3OwxoacfAG2Ac+laiTaYuZyLk5e0Q1UFZZU4npKL4ym5AC4CAOxtFejZwQWRHdykYaoQT0fIGeC0awcTs3ElrxQudjZSsidRTcO7esNWIUNydjESs4oQ5u1k6iaRmWAw00bS80uRkFYAmQwYboXdozKZDEEeDgjycMCkXv4AtL+0k7OLcPJyVf7NpTycupKH4nI1Didfw+Hk6oUtnVQ26NnBBb0C3aRhqo4eDvzl1Y7ohphu6R3A5HIyyEllgxtCPbH3XBa2n05nMNPGqpczML/PZQYzbUTXK9Mr0A3ujkoTt6ZtyOUyhHo7IdTbCVP6aJM51RqBxMxCaWjqxKVcxF3JR2FZJf5JzME/idVF+1ztbRHZQZdgrP23g5u9Wb6R6PoUllXir1OsLUONiw73qQpmMjBvRJipm9OucJiJzGqVbFNSyGXo4uuMLr7O0pdWpVqDcxmFVUnG2iGq01cLkFdSgX3ns7DvfJZ0ew9HZY0EY1f0CnSDr4uKAY6F+/PkVZRUqBHq7Yi+QW6mbg6ZsegIXyz+LR5HLl5DXnEFXB1sTd0kMgMMZtqAWiOw75z2C7m9BzOG2CjkiPB3QYS/C6YNCAIAlFdqcDa9QC/J+ExaAXKKyrH7bKYUHAKAt7NK6rnRzaLyduYqy5ZEN+X/zijWlqGGBXk4oKuvE86mF2LX2Qyp15eMT5jxcgYMZtpA7KVc5JVUwMXOBr0DWdG0KZQ2cvTs4IqeHVwBdAQAlFaokZBWIE0PP3k5D2fTC5BZUIbtCRl6lUH9Xe2qe3Cq8nA82snwnqVJyS7Gv0k5kLG2DDXRmHBfnE0vxPbTDGZMwQxjGQYzbUFX9Xd4F2/YsOZKi9nZKtAnyA19agxDlJSrEX+1OsH4xOU8XMgsxNW8UlzNK8WW+OpVdgPd7aWem16B2kDJ1Z5d1Kb2c9V07GGdveDvam/i1pAlGBvhg893X8CuMxmoVGv4uUoMZtoC82WMx16pQFSwB6KCPaRthWWViLucVz2L6nIekrKKcOlaCS5dK8GfJ9OkfTt5OqBXVf2byA6u6NHB1azWzLJ2Go2QghnWlqGm6tvRHe4OtrhWXIEjF6/hhtCWL9obdyUPL244iUdHd8a4Hn6t2Errw9lM7di1onKcuJQLABje1fqmZJsjJ5UNBoV6YlCND7i8kgrEXdZVMdYmGqfmlCA5uxjJ2cX4NfYKAO1YcJi3k14OTnd/V9grOVXYGP5NzsGlayVwVtlgXHd+kVDTKOQyjO7mgw3HL2NHQsZ1BTNv/ZWA2Et5WLA2Fn/Pd0Ggu0MrttS6iKr5TOYXyjCYMbp957OgEUA3X2d2oZuQq70thnT20iuBfq2oXLsOVdUU8ZOX8nAlrxTnMwpxPqMQG2ostNnV11kvByfcz5m1UFqBLvF3Ui9/BozULGMitMHMttPpePGmiBYd4/TVfOytmpxRWFaJZ9edwA9zB7GApwViMGNk1lL11xq5Oyoxoqs3RtQY/sssKMOpy9WLbMZeykNmQRkS0gqQkFaAdTUW2uzm56yXg9PVlwttNkdRWSX+PHkVAIeYqPlGdPWGjVyGxMwiJGUVIcTLsdnH+N/eJADAgE7uOHU5HwcTs/HdwWTMHhrS2s21CsKMC80wmDEiIQT2MF/Gong7qzA63Aejw/UX2tQmGOfiRFWgk1NUjrgr+Yi7ko+fUL3QZoS/s7SKeGSgK7r4ODE5sR5/n0pDcbkanTwdEBXsburmkIVxsbPFwBAPHLiQje2n0zF3eGizbp+eX4pfY7W9ry/eFIFTl/PwyqY4vPV3AkZ09UYoqwvXS2aG0QyDGSNKSCtARkEZ7G0V6N+JH9aWytfFDjd2t8ONNRbavJJXqjdF/MSlPOSVVCD2Uh5iL+UBSAGgXWizR4D+Mg2hXGgTQPUspjv6sbYMtUx0hC8OXMjGjoSMZgcz3x5IRoVaoH+wO/p2dEefIDdsiU/H3nNZWLA2FusfGswfIrWY75rZDGaMSjfENDjMEyob5gNYC5lMhg5u9ujgZo8JPbXrUAkhkJpTIlUwPnEpD6cuaxfaPJaSi2MpudLtHZQK9AyoWeTPFZ3a2UKbl64V48CFbADAbf1YJ4RaJjrcB6/9Ho9/k3KQX1oBF7umlVooLq/ED4e0Pzh0QZBMJsPyO3th3Ad7EJOaiy/2JOLR0Z2N1nZLVD2bybTtMITBjBHp6stwiMn6yWQydPR0QEdPB9zcKwBAwwtt/pucg3+Tq9ehclbZoKeUYKwdpgrysN51qH45pu3eHxLmydkj1GKdvBwR5u2IC5lF2HM2U3rvNWbdkUvIK6lAsKeD1OMKAP6u9lgyuQcWrI3Ff7adxehuPuge4GKs5lssc/xUYjBjJEVllThyUftlNYLBTLvUnIU2C8oqcTAxGwcTs6Xbu9rb1liDSjuLKsDVzuIDHCGE3hAT0fWIjvDFhcxEbD+d0aRgRq0R+HqfNvH3/mEhdYZ8b+vbAZvj0rA5Lh0L1sZg02ND2bNeRZjxQBODGSM5eCEbFWqBjh4O6OTJX56k1dyFNveey5KmjgKAp6Oyxiri2llUvi52pjqdFjl68RqSs4vhqFRgYiRry9D1iQ73wZd7ErHzTAbUGtFoPtrW+DSk5BTDzcHW4Cw6mUyGZbdF4kjyNSSkFeA/287h+Qnhxmq+ReEwUztUs+qvpf+SJuNqzkKb2UXl2HUmE7vOVC+06eOs0psiHhnoCi8n811oU1db5qZIfzgo+RFE1ycq2B2u9rbILa7AsZRrGNDJo8H9v9yTCAD4v0HB9b7+vJxUWHZbJB5adRRf7L6AsRE+elXG2zvOZmpHuIQBXY/mLLSZUVCGbaczsO109UKbAa52VQnG2llUkR1c4W4GC22WlKvxxwltbZk7WFuGWoGNQo5R3byxKeYKtp/OaDCYOXrxGo6l5EKpkOPeIcENHndCTz/c3rcDNhy/jKfXxuLPJ4cz+DZjfGaMIDmrCCk5xbBVyDA4rOVltolqMrTQZnF5JU5fza+z0OaVvFJcySvF5rjqhTaDPOyl+je9qtahauuFNrfEp6GgrBJBHvYY2MgvaKKmGhPuUxXMpOOFifUPCf1vr7ZXZkqfAPg4Nz48u2hyDxxMzEZydjHe+isBS6f0bLU2WyJRNc5kjoMNDGaMQNcr0z/YA45ctJCMyEFp0+SFNlNzSpCaU4I/qqruAkCIl2N1gnEbLLSpG2K6vW9gu5qKTsY1qqsPFHIZzmUUIiW7GB0N5CmmZBdjc5x2kdkHRjStJo2rvS2W39kL93z9L747eBHjuvthWBeuscdgpp3gEgZkSs1ZaDMpS1sKvi0W2ryaV4J957XJzJzFRK3J1cEW/YPdcSgpB9sT0nGfgeUIvtmfBI3QDv139XVu8rGHd/HGPTcE4/t/LuLZ9bH4e/6INu/RNBfCfCczMZhpbWWVahysKgY2oguDGTIP17PQpkIuQxcfp+teaHPDscsQAhgY4mHwlzPR9Rgb4YtDSTnYkZBRJ5jJLS7H2iPaZUceaGalYABYeFM49p7LRHJ2MZb8Fof3p/VpjSZbnOpYxvy6ZhjMtLIjyddQUqGGt7MKEf5Nj/6J2lpjC22eqFqLquGFNt2kIaqGFtqsWVuGi0qSMYyJ8MGyP0/jn8RsFJRWwLlGNeAfDqWguFyNcD9nDO3c/DxGB6UN3pvWG1M/P4gNxy5jfA8/jO/RfssKcJipHeCUbLJktRfaFEIgPb9M23NTIwdHb6HNf7W3Vdpop5jXHKLq7K1daPN4ai4SM4tgb6vATZH+JjxDslZh3k4I8XJEUlYR9p3LwsSq11l5pQbfHkgGoO2VaennclSwB+aNCMPnuy/gxQ0nERXsbtYlEIyBw0ztCJcwIGsik8ng52oHP1c/jKv6JSqEwOXcEmn21MmqXpz80krEpuYiNjVXur2drRw9AlxRVqkGAEzs6WfUBGNq38aE++DrfUnYdjpDCmZ+jb2CjIIy+LqocEvvpi13UJ+nbuyCXWcykJBWgJd+OYnP/y+qXf1o1VUANscz5qdKK0rLK8WZ9ALIZMCwzsx4J+skk8kQ6O6AQHcH6QtDCIGUnGK9ZRpOXc5HYVkljl68Jt2WtWXImKIjtMHMrqpqwHJZ9XTs2UNC6h0GbSqVjQLvTeuNWz/Zj81x6fjl+GXc3g6T2c0xfmMw04r2VA0x9Q50M4sCZURtRSaTIdjTEcGejtKvX41GICm7SFpF3NNJiSGsu0RGNKCTB5ztbJBdVI6Y1FwUl1ciIa0ADkoF7hrYsVXuo0eAK56M7oJ3t5zFol/jcEOoJwLc7Fvl2ObOnIeZri9MJT2s+ktUTS6XIczbCbf27YBXb+mOR0d3bldd8tT2bBVy6fN3R0I6vtqrXVByWv8guDq03nTqh0aGoU+QGwpKK/H8zyekYnLWTneW5ricAYOZVlKp1mDvOW0ww1WyiYhMIzpCm7z+6a4L2HM2E3KZdnXs1mSjkOO9ab1hZyvH3nNZWHUopVWPb+7M8TcJg5lWEnspD/mllXC1t0XvQFdTN4eIqF0a1dUHcln1kMjEnv4I8mj9ukZh3k7Satpv/HEayVlFrX4fZseMe6AYzLQS3RDTsC5esFHwYSUiMgV3R6Ve8DJ3eOv2ytQ0a3AnDA71REmFGs+si4VaY75f9q2hepjJ/PBbt5UwX4aIyDyoasxa6tvR3Wj3I5fL8M7UXnBS2eDIxWv4qmrmlLUzx9w3BjOt4FpROU5cygXAJQyIiEztpUndEe7njA2PDDH6fQW6O+DVm7sDAN7fchYJaflGv09TMeNRJgYzrWHv+SwIAYT7OcPPtfFl5YmIyHhGdvXG3/NHoJ8Re2Vqmto/ENHhPihXa7BgTSzKKzVtcr9tTcB8oxkGM62AVX+JiNovmUyGN++IhLuDLeKv5uOjHedM3SSjMsNRJgYz10sIgT2ckk1E1K75ONvh9VsjAWinhcfUWNbDWnCYyYqdvlqAzIIy2Nsq0L9T23RpEhGR+ZnUyx+TewdArRFYsDYGpRVqUzepVbXbonnLli3DkCFD4ODgADc3tzrXx8bGYubMmQgKCoK9vT0iIiLw4Ycf6u2zYcMG3HjjjfD29oaLiwsGDx6MzZs3G7PZzaKbxTQkzBMqG4WJW0NERKa0dEoP+DirkJhZhLf/TjB1c4yi3Q0zlZeXY+rUqXj44YcNXn/06FH4+Phg1apViIuLw0svvYSFCxfi448/lvbZs2cPbrzxRvz55584evQoRo8ejVtuuQXHjx83ZtObbPfZDADAyG4cYiIiau/cHJR4+85eAIAV+5Nx4EKWiVvUesx5mMmoC00uWbIEALBy5UqD18+ZM0fv79DQUBw8eBAbNmzAY489BgD4z3/+o7fPG2+8gU2bNuG3335D3759W73NzVFYVokjydoVgTklm4iIAGB0Nx/MHNgRP/2bgmfXncDf84fD2a711oYyFd1sJjPsmDG/nJm8vDx4eHjUe71Go0FBQUGD+7SVfy5ko1IjEOzpgE5ejqZuDhERmYmXJkUgyMMel3NL8Nrv8aZuTqtqd8NMzXXgwAGsWbMG8+bNq3efd999F4WFhZg2bVq9+5SVlSE/P1/vYgy6QnkDO5k+sCIiIvPhpLLBe1P7QCYD1h65hG3x6aZu0vUz42GmZgczL7zwAmQyWYOXhITmJz2dOnUKU6ZMwaJFizBu3DiD+/z4449YsmQJ1q5dCx8fn3qP9eabb8LV1VW6BAUFNbs9TeHraof+we7oF8xZTEREpG9giAfmVq3Y/cKGk8gpKjf6fZ6+mo+L2cZZ9FKazWSGXTPNzpl5+umnMXv27Ab3CQ0NbdYx4+PjER0djXnz5uHll182uM/q1asxd+5crFu3DmPHjm3weAsXLsSCBQukv/Pz840S0Nw9KBh3Dwpu9eMSEZF1eHpcN+w6k4lzGYV4ZeMpfHxXX6MFA9tPp2Pud0fg72KHAwujjXIfgHnmzDQ7mPH29oa3d+slu8bFxWHMmDGYNWsWli1bZnCfn376CXPmzMHq1asxadKkRo+pUqmgUqlarY1EREQtYWerwPvT+uC2T/fjj5NXMS7WF1P6dGj1+4m/ko/HfzoOIYDMwrJWPz6gLRJrroyaM5OSkoKYmBikpKRArVYjJiYGMTExKCwsBKAdWho9ejTGjRuHBQsWIC0tDWlpacjMzJSO8eOPP+Lee+/Fe++9h0GDBkn75OXlGbPpRERErSIy0BWPjekMAHh1UxzS80tb9fgZ+aWY++1hFJdri/QZK+YQ1VXzzI5Rg5lXX30Vffv2xaJFi1BYWIi+ffuib9++OHLkCABg/fr1yMzMxKpVq+Dv7y9dBgwYIB3jyy+/RGVlJR599FG9fZ588kljNp2IiKjVPDq6MyI7uCKvpALPrT/Rar0cJeVqPPDdEVzJK4W3c9uMSLS7CsArV66EEKLOZdSoUQCAxYsXG7w+OTlZOsauXbsM7lNf7RoiIiJzY6uQ4/1pvaG0kWP32UysPpx63cfUaASeXheD2Et5cHOwxccztbXXjDUYZL6DTGY2NZuIiMhadfF1xrPjugEAXv89Hqk5xdd1vPe3nsWfJ9Ngq5Dhi/+LMnq9M11nkhlOZmIwQ0RE1FbmDAvBwE4eKCpX4+l1sdBoWtbf8fPRS/h453kAwJu398KgUE9p8MfYibpmGMswmCEiImorCrkM707tDQelAv8m5eCb/UnNPsa/STl4YcMJAMAjo8JwZ1RgazfTIGHGA00MZoiIiNpQR08HvDypOwBg+eYzOJde0OTbXswuwoPfH0GFWmBiTz88UzVsBUDqMjFazgyHmYiIiEhn5sAgjOrmjfJKDRasjUWFWtPobfJKKjBn5WFcK65Ar0BXvD+tD+Tyto8s2t1sJiIiIqpLJpPh7Tt6wdXeFicv5+GTqvyX+lSoNXj0h2O4kFkEf1c7/O/e/rBXKvSPWRVkmHFtO6NhMENERGQCvi52WDqlBwDg4x3ncfKS4WKwQggs+jUO+85nwUGpwP9m9YePi12d/Yw9/KNLLOYwExEREUkm9w7ApEh/VGoEFqyNQWmFus4+X+9Lwo+HUiCTAf+d0Rc9AlwNHqtmjGHoONaMwQwREZGJyGQyvHZrT3g5qXAuoxDvbTmjd/22+HQs+/M0AOClmyIwtrtvvcdyd1Ai0N0eALD+6KVWb6sZr2bAYIaIiMiUPByVePuOSADA//Yl4VBiNgDt4pFPrNYuHjlzYBDuHxbS4HHkchkeGB4KAPhqbyLULaxhU5/qtZnML5xhMENERGRi0RG+mNY/EEIAz6yPRVJWkbR45NDOnlg6pSdkTQgipvYPhLuDLS5mF+PvU2lt0HLzwGCGiIjIDLxyc3d0cLNHak4JJn64B1fyShHq7YhP74qCraJpX9cOShvcO7gTAODz3RdatRqwrmie+fXLMJghIiIyC852tnhnai8AQGmFBm4Otvhm1gC4Otg26zizhnSCna0cJy/n4WDVkFVrYNE8IiIiatSQMC88PyEcoV6O+PKe/i1aPNLDUYnp/YMAAF/sTmztJpolBjNERERm5OFRYdjxzCgMDPFo8THmDg+FXAbsPpuJ+Cv5rdKu6tlM5tc1w2CGiIjIygR5OGBSrwAAwJd7LrTKMTnMRERERG3qwRHaadq/nbiKS9eKTdwa42IwQ0REZIV6dnDFsM5eUGsEvt6X1ApH5GwmIiIiamMPjtT2zqz+NxXXisqv61gcZiIiIqI2N6yzF3oEuKCkQo3v/7lo6uYYDYMZIiIiKyWTyfDgyDAAwMoDyde1AGV1z4z5dc0wmCEiIrJiN/X0Q6C7PXKKyrHuOhagFGjdtZ5aE4MZIiIiK2ajkFcvQLmn9RegNAcMZoiIiKycbgHKlJxi/HXqaouOwQRgIiIiMhkHpQ1mDekEQLvEwfUsQMkKwERERGQS9w6usQDlheYvQGnOg1MMZoiIiNqBmgtQfr6n+QtQcpiJiIiITE63AOWe61iA0gxjGQYzRERE7UXNBSi/aOYClJyaTURERGZBtwDl7yeuIjWnGQtQcpiJiIiIzEHPDq4Y3qXlC1ByNhMRERGZ3IMjtEscrDnc9AUozXeQicEMERFRuzO0s2ezF6DU1abhMBMRERGZXO0FKEvKW74ApTlgMENERNQO1VyAcv3R1Eb35zATERERmRW9BSj3JqFSrWlw/+qieeY3zsRghoiIqJ2a1j9IWoDy77i0Jt3G/EIZBjNERETtlr1S0eQFKDnMRERERGap5gKUBxpYgJKzmYiIiMgs6S1AubvxJQ7MMJZhMENERNTezR0eCoVchr3nshB3Jc/gPhxmIiIiIrMV5OGASZH+AIAv9yQa3omzmYiIiMiczWviApRmGMswmCEiIqLGF6AUZjzQxGCGiIiIAFQvQLn6cApyai1AKRXNa+tGNQGDGSIiIgJQvQBlaYUG3x+sZwFKMxxnYjBDREREAPQXoPz2oP4ClA3U0zM5BjNEREQkuamnH4I86i5AqcuZMb9+GQYzREREVEPNBSi/3JtYZwFKMxxlYjBDRERE+qZGaRegTM0pwV+ntAtQcpiJiIiILIbeApR7LkCI6onZMjMcaGIwQ0RERHXoFqA8dTlfbwFKDjMRERGRRai9ACWHmYiIiMji1FyAMiEtHwBnMxEREZEFqbkA5aVrJQA4zEREREQWRrcApTljMENERET10i1AqcPZTERERGRxdAtQAjDLpBkGM0RERNSgoZ090bODCwDAxc7WxK2py8bUDSAiIiLzJpPJ8L97B2BHQgaiI3xM3Zw6GMwQERFRo/xc7XDXoI6mboZBHGYiIiIii8ZghoiIiCwagxkiIiKyaAxmiIiIyKIxmCEiIiKLxmCGiIiILBqDGSIiIrJoDGaIiIjIojGYISIiIotm1GBm2bJlGDJkCBwcHODm5lbn+tjYWMycORNBQUGwt7dHREQEPvzww3qPt3//ftjY2KBPnz7GazQRERFZFKMuZ1BeXo6pU6di8ODB+Prrr+tcf/ToUfj4+GDVqlUICgrCgQMHMG/ePCgUCjz22GN6++bm5uLee+9FdHQ00tPTjdlsIiIisiAyIYQw9p2sXLkS8+fPR25ubqP7Pvroozh9+jR27Niht33GjBno0qULFAoFNm7ciJiYmCbff35+PlxdXZGXlwcXF5dmtp6IiIhMoanf32aXM5OXlwcPDw+9bStWrEBiYiIWLVrUpGOUlZUhPz9f70JERETWyaxWzT5w4ADWrFmDP/74Q9p27tw5vPDCC9i7dy9sbJrW3DfffBNLliyps51BDRERkeXQfW83Oogkmun5558XABq8nD59Wu82K1asEK6urg0e9+TJk8LLy0u89tpr0rbKykrRv39/8dlnn0nbFi1aJHr37t3gsUpLS0VeXp50iY+Pb7TNvPDCCy+88MKLeV5SU1Mb/N5vds5MZmYmsrOzG9wnNDQUSqVS+ruxnJn4+HiMHj0ac+fOxbJly6Ttubm5cHd3h0KhkLZpNBoIIaBQKLBlyxaMGTOm0TZrNBpcuXIFzs7OkMlkje5vKfLz8xEUFITU1NR2kQvE87Ve7elcAZ6vNWtP5woY/3yFECgoKEBAQADk8vozY5o9zOTt7Q1vb+/ralxNcXFxGDNmDGbNmqUXyACAi4sLTp48qbft008/xY4dO7B+/XqEhIQ06T7kcjkCAwNbrc3mxsXFpV28aXR4vtarPZ0rwPO1Zu3pXAHjnq+rq2uj+xg1ZyYlJQU5OTlISUmBWq2WZiB17twZTk5OOHXqFMaMGYPx48djwYIFSEtLAwAoFAp4e3tDLpejZ8+eesf08fGBnZ1dne1ERETUPhk1mHn11Vfx7bffSn/37dsXALBz506MGjUK69evR2ZmJlatWoVVq1ZJ+wUHByM5OdmYTSMiIiIrYdSp2StXroQQos5l1KhRAIDFixcbvL6hQGbx4sXNqjFjzVQqFRYtWgSVSmXqprQJnq/1ak/nCvB8rVl7OlfAfM63TYrmERERERmL2RXNIyIiImoOBjNERERk0RjMEBERkUVjMENEREQWjcGMib355psYMGAAnJ2d4ePjg1tvvRVnzpzR22fUqFGQyWR6l4ceekhvn5SUFEyaNAkODg7w8fHBs88+i8rKSr19du3ahX79+kGlUqFz585YuXKlsU9Pz+LFi+ucR3h4uHR9aWkpHn30UXh6esLJyQl33HEH0tPT9Y5hCeep06lTpzrnK5PJ8OijjwKw/Od1z549uOWWWxAQEACZTIaNGzfqXS+EwKuvvgp/f3/Y29tj7NixOHfunN4+OTk5uPvuu+Hi4gI3Nzfcf//9KCws1NvnxIkTGD58OOzs7BAUFITly5fXacu6desQHh4OOzs7REZG4s8//2zT862oqMDzzz+PyMhIODo6IiAgAPfeey+uXLmidwxDr4m33nrL7M63sed29uzZdc5jwoQJevtYy3MLwOD7WCaT4Z133pH2sZTntinfOW35WfzJJ5+gU6dOsLOzw6BBg/Dvv/+27MQaXOyAjG78+PFixYoV4tSpUyImJkbcdNNNomPHjqKwsFDaZ+TIkeKBBx4QV69elS55eXnS9ZWVlaJnz55i7Nix4vjx4+LPP/8UXl5eYuHChdI+iYmJwsHBQSxYsEDEx8eLjz76SCgUCvH333+32bkuWrRI9OjRQ+88MjMzpesfeughERQUJLZv3y6OHDkibrjhBjFkyBCLO0+djIwMvXPdunWrACB27twphLD85/XPP/8UL730ktiwYYMAIH755Re969966y3h6uoqNm7cKGJjY8XkyZNFSEiIKCkpkfaZMGGC6N27t/jnn3/E3r17RefOncXMmTOl6/Py8oSvr6+4++67xalTp8RPP/0k7O3txRdffCHts3//fqFQKMTy5ctFfHy8ePnll4Wtra04efJkm51vbm6uGDt2rFizZo1ISEgQBw8eFAMHDhRRUVF6xwgODhZLly7Ve85rvtfN5Xwbe25nzZolJkyYoHceOTk5evtYy3MrhNA7z6tXr4pvvvlGyGQyceHCBWkfS3lum/Kd01afxatXrxZKpVJ88803Ii4uTjzwwAPCzc1NpKenN/u8GMyYmYyMDAFA7N69W9o2cuRI8eSTT9Z7mz///FPI5XKRlpYmbfvss8+Ei4uLKCsrE0II8dxzz4kePXro3W769Oli/PjxrXsCDWhokdDc3Fxha2sr1q1bJ207ffq0ACAOHjwohLCc86zPk08+KcLCwoRGoxFCWM/zKoSo8wWg0WiEn5+feOedd6Rtubm5QqVSiZ9++kkIIaQFYA8fPizt89dffwmZTCYuX74shBDi008/Fe7u7tL5CqFd7LZbt27S39OmTROTJk3Sa8+gQYPEgw8+2KrnWJOhL7za/v33XwFAXLx4UdoWHBwsPvjgg3pvY47nW18wM2XKlHpvY+3P7ZQpU8SYMWP0tlnicytE3e+ctvwsHjhwoHj00Uelv9VqtQgICBBvvvlms8+Dw0xmJi8vDwDg4eGht/2HH36Al5cXevbsiYULF6K4uFi67uDBg4iMjISvr6+0bfz48cjPz0dcXJy0z9ixY/WOOX78eBw8eNBYp2LQuXPnEBAQgNDQUNx9991ISUkBABw9ehQVFRV6bQwPD0fHjh2lNlrSedZWXl6OVatWYc6cOXqLnVrL81pbUlIS0tLS9Nrm6uqKQYMG6T2fbm5u6N+/v7TP2LFjIZfLcejQIWmfESNG6C1cO378eJw5cwbXrl2T9jHHxyAvLw8ymQxubm5629966y14enqib9++eOedd/S65i3pfHft2gUfHx9069YNDz/8sN4CxNb83Kanp+OPP/7A/fffX+c6S3xua3/ntNVncXl5OY4ePaq3j1wux9ixY1t0vkZdzoCaR6PRYP78+Rg6dKje2lN33XUXgoODERAQgBMnTuD555/HmTNnsGHDBgBAWlqa3osKgPS3br2r+vbJz89HSUkJ7O3tjXlqAIBBgwZh5cqV6NatG65evYolS5Zg+PDhOHXqFNLS0qBUKut88Pv6+jZ6DrrrGtqnLc/TkI0bNyI3NxezZ8+WtlnL82qIrn2G2laz7T4+PnrX29jYwMPDQ2+f2gvK1nwM3N3d630MdMcwhdLSUjz//POYOXOm3uJ7TzzxBPr16wcPDw8cOHAACxcuxNWrV/H+++8DsJzznTBhAm6//XaEhITgwoULePHFFzFx4kQcPHgQCoXCqp/bb7/9Fs7Ozrj99tv1tlvic2voO6etPouvXbsGtVptcJ+EhIRmnwuDGTPy6KOP4tSpU9i3b5/e9nnz5kn/j4yMhL+/P6Kjo3HhwgWEhYW1dTNbbOLEidL/e/XqhUGDBiE4OBhr16412ZduW/n6668xceJEBAQESNus5XklfRUVFZg2bRqEEPjss8/0rluwYIH0/169ekGpVOLBBx/Em2++afJy8M0xY8YM6f+RkZHo1asXwsLCsGvXLkRHR5uwZcb3zTff4O6774adnZ3edkt8buv7zrFEHGYyE4899hh+//137Ny5E4GBgQ3uO2jQIADA+fPnAQB+fn51Ms11f/v5+TW4j4uLi8kCCTc3N3Tt2hXnz5+Hn58fysvLkZubW6eNjZ2D7rqG9jHleV68eBHbtm3D3LlzG9zPWp5XoLp9htpWs+0ZGRl611dWViInJ6dVnnPd9W1JF8hcvHgRW7du1euVMWTQoEGorKyU1qOztPPVCQ0NhZeXl95r19qeWwDYu3cvzpw50+h7GTD/57a+75y2+iz28vKCQqFotfNlMGNiQgg89thj+OWXX7Bjx4463ZCG6Bba9Pf3BwAMHjwYJ0+e1Pvw0H2Qdu/eXdpn+/btesfZunUrBg8e3Epn0nyFhYW4cOEC/P39ERUVBVtbW702njlzBikpKVIbLfU8V6xYAR8fH0yaNKnB/azleQWAkJAQ+Pn56bUtPz8fhw4d0ns+c3NzcfToUWmfHTt2QKPRSIHd4MGDsWfPHlRUVEj7bN26Fd26dYO7u7u0jzk8BrpA5ty5c9i2bRs8PT0bvU1MTAzkcrk0JGNJ51vTpUuXkJ2drffatabnVufrr79GVFQUevfu3ei+5vrcNvad01afxUqlElFRUXr7aDQabN++vWXn2+yUYWpVDz/8sHB1dRW7du3Sm9JXXFwshBDi/PnzYunSpeLIkSMiKSlJbNq0SYSGhooRI0ZIx9BNkxs3bpyIiYkRf//9t/D29jY4Te7ZZ58Vp0+fFp988kmbT1l++umnxa5du0RSUpLYv3+/GDt2rPDy8hIZGRlCCO10wI4dO4odO3aII0eOiMGDB4vBgwdb3HnWpFarRceOHcXzzz+vt90anteCggJx/Phxcfz4cQFAvP/+++L48ePS7J233npLuLm5iU2bNokTJ06IKVOmGJya3bdvX3Ho0CGxb98+0aVLF73pu7m5ucLX11fcc8894tSpU2L16tXCwcGhznRWGxsb8e6774rTp0+LRYsWGWX6bkPnW15eLiZPniwCAwNFTEyM3ntZN7vjwIED4oMPPhAxMTHiwoULYtWqVcLb21vce++9Zne+DZ1rQUGBeOaZZ8TBgwdFUlKS2LZtm+jXr5/o0qWLKC0tlY5hLc+tTl5ennBwcBCfffZZndtb0nPb2HeOEG33Wbx69WqhUqnEypUrRXx8vJg3b55wc3PTmyXVVAxmTAyAwcuKFSuEEEKkpKSIESNGCA8PD6FSqUTnzp3Fs88+q1ePRAghkpOTxcSJE4W9vb3w8vISTz/9tKioqNDbZ+fOnaJPnz5CqVSK0NBQ6T7ayvTp04W/v79QKpWiQ4cOYvr06eL8+fPS9SUlJeKRRx4R7u7uwsHBQdx2223i6tWresewhPOsafPmzQKAOHPmjN52a3hed+7cafC1O2vWLCGEdnr2K6+8Inx9fYVKpRLR0dF1Hofs7Gwxc+ZM4eTkJFxcXMR9990nCgoK9PaJjY0Vw4YNEyqVSnTo0EG89dZbddqydu1a0bVrV6FUKkWPHj3EH3/80abnm5SUVO97WVdX6OjRo2LQoEHC1dVV2NnZiYiICPHGG2/oBQDmcr4NnWtxcbEYN26c8Pb2Fra2tiI4OFg88MADdb6ArOW51fniiy+Evb29yM3NrXN7S3puG/vOEaJtP4s/+ugj0bFjR6FUKsXAgQPFP//806LzklWdHBEREZFFYs4MERERWTQGM0RERGTRGMwQERGRRWMwQ0RERBaNwQwRERFZNAYzREREZNEYzBAREZFFYzBDREREFo3BDBEREVk0BjNERERk0RjMEBERkUVjMENEREQW7f8BLQm4Y4WLagoAAAAASUVORK5CYII=",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "df_smoothed_w_bins[df_w_bins[\"MedHouseVal_bins\"] == 5][\"Longitude\"].plot(\n",
-    "    title=\"Longitude feature values after calibration\\n(only slight change in values)\"\n",
-    ");"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "771a9011",
-   "metadata": {},
-   "source": [
-    "# Data preparation"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "6d1a2d52",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_train, df_valid = train_test_split(df, test_size=0.2, random_state=1)\n",
-    "df_valid, df_test = train_test_split(df_valid, test_size=0.5, random_state=1)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "78a5061f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "continuous_cols = df.drop(columns=[\"MedHouseVal\"]).columns.values.tolist()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "61ef884d",
-   "metadata": {},
-   "source": [
-    "# Model with LDS & FDS"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "id": "368867f4",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n",
-      "  self._check_inputs(cat_embed_cols)\n"
-     ]
-    }
-   ],
-   "source": [
-    "# deeptabular\n",
-    "tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\n",
-    "X_tab_train = tab_preprocessor.fit_transform(df_train)\n",
-    "X_tab_valid = tab_preprocessor.transform(df_valid)\n",
-    "X_tab_test = tab_preprocessor.transform(df_test)\n",
-    "\n",
-    "# target\n",
-    "y_train = df_train[\"MedHouseVal\"].values\n",
-    "y_valid = df_valid[\"MedHouseVal\"].values\n",
-    "y_test = df_test[\"MedHouseVal\"].values\n",
-    "\n",
-    "X_train = {\"X_tab\": X_tab_train, \"target\": y_train}\n",
-    "X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "id": "1329d458",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "WideDeep(\n",
-       "  (deeptabular): TabMlp(\n",
-       "    (cont_norm): Identity()\n",
-       "    (encoder): MLP(\n",
-       "      (mlp): Sequential(\n",
-       "        (dense_layer_0): Sequential(\n",
-       "          (0): Linear(in_features=8, out_features=200, bias=True)\n",
-       "          (1): ReLU(inplace=True)\n",
-       "          (2): Dropout(p=0.1, inplace=False)\n",
-       "        )\n",
-       "        (dense_layer_1): Sequential(\n",
-       "          (0): Linear(in_features=200, out_features=100, bias=True)\n",
-       "          (1): ReLU(inplace=True)\n",
-       "          (2): Dropout(p=0.1, inplace=False)\n",
-       "        )\n",
-       "      )\n",
-       "    )\n",
-       "  )\n",
-       "  (fds_layer): FDSLayer(\n",
-       "    (pred_layer): Linear(in_features=100, out_features=1, bias=True)\n",
-       "  )\n",
-       ")"
-      ]
-     },
-     "execution_count": 15,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "deeptabular = TabMlp(\n",
-    "    column_idx=tab_preprocessor.column_idx,\n",
-    "    continuous_cols=tab_preprocessor.continuous_cols,\n",
-    ")\n",
-    "model = WideDeep(deeptabular=deeptabular, with_fds=True)\n",
-    "model"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "id": "2b42ad10",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "epoch 1: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 165.40it/s, loss=0.591, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 218.64it/s, loss=0.479, metrics={}]\n",
-      "FDS update: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 331/331 [00:00<00:00, 366.86it/s]\n",
-      "epoch 2: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:01<00:00, 182.78it/s, loss=0.497, metrics={}]\n",
-      "valid: 100%|███████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 214.30it/s, loss=0.47, metrics={}]\n",
-      "FDS update: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 331/331 [00:00<00:00, 350.68it/s]\n",
-      "epoch 3: 100%|████████████████████████████████████████████████████████████████████████| 331/331 [00:04<00:00, 81.28it/s, loss=0.52, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 146.50it/s, loss=0.452, metrics={}]\n",
-      "FDS update: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 122.88it/s]\n",
-      "epoch 4: 100%|███████████████████████████████████████████████████████████████████████| 331/331 [00:03<00:00, 99.77it/s, loss=0.508, metrics={}]\n",
-      "valid: 100%|███████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 157.48it/s, loss=0.45, metrics={}]\n",
-      "FDS update: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 147.92it/s]\n",
-      "epoch 5: 100%|███████████████████████████████████████████████████████████████████████| 331/331 [00:03<00:00, 93.21it/s, loss=0.591, metrics={}]\n",
-      "valid: 100%|███████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 154.60it/s, loss=0.45, metrics={}]\n",
-      "FDS update: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 145.73it/s]\n",
-      "predict: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 243.46it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "test RMSE: 0.7417540528440087\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Optimizers\n",
-    "deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n",
-    "# LR Scheduler\n",
-    "deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n",
-    "# Hyperparameters\n",
-    "trainer = Trainer(\n",
-    "    model,\n",
-    "    objective=\"huber\",\n",
-    "    lr_schedulers={\"deeptabular\": deep_sch},\n",
-    "    initializers={\n",
-    "        \"deeptabular\": XavierNormal,\n",
-    "        \"fds_layer\": XavierNormal,\n",
-    "        # \"FDS_dropout\": XavierNormal,\n",
-    "        # \"pred_layer\": XavierNormal,\n",
-    "    },\n",
-    "    optimizers={\"deeptabular\": deep_opt},\n",
-    "    metrics=[],\n",
-    "    with_lds=True,\n",
-    "    lds_kernel=\"gaussian\",\n",
-    "    lds_ks=5,\n",
-    "    lds_sigma=2,\n",
-    "    lds_granularity=100,\n",
-    "    lds_reweight=False,\n",
-    "    lds_y_max=None,\n",
-    "    lds_y_min=None,\n",
-    ")\n",
-    "\n",
-    "trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n",
-    "\n",
-    "print(\n",
-    "    f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "29f977db",
-   "metadata": {},
-   "source": [
-    "# Model with LDS only"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "id": "51ec337d",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "epoch 1: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 135.40it/s, loss=0.449, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 158.01it/s, loss=0.386, metrics={}]\n",
-      "epoch 2: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 137.79it/s, loss=0.377, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 156.84it/s, loss=0.399, metrics={}]\n",
-      "epoch 3: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 138.69it/s, loss=0.358, metrics={}]\n",
-      "valid: 100%|███████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 150.62it/s, loss=0.41, metrics={}]\n",
-      "epoch 4: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 131.96it/s, loss=0.339, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 146.01it/s, loss=0.321, metrics={}]\n",
-      "epoch 5: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 136.04it/s, loss=0.331, metrics={}]\n",
-      "valid: 100%|███████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 173.22it/s, loss=0.32, metrics={}]\n",
-      "predict: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 296.77it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "test RMSE: 0.6000006967500053\n"
-     ]
-    }
-   ],
-   "source": [
-    "deeptabular = TabMlp(\n",
-    "    column_idx=tab_preprocessor.column_idx,\n",
-    "    continuous_cols=tab_preprocessor.continuous_cols,\n",
-    ")\n",
-    "model = WideDeep(deeptabular=deeptabular, fds=False)\n",
-    "\n",
-    "# Optimizers\n",
-    "deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n",
-    "# LR Scheduler\n",
-    "deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n",
-    "# Hyperparameters\n",
-    "trainer = Trainer(\n",
-    "    model,\n",
-    "    objective=\"huber\",\n",
-    "    lr_schedulers={\"deeptabular\": deep_sch},\n",
-    "    initializers={\"deeptabular\": XavierNormal},\n",
-    "    optimizers={\"deeptabular\": deep_opt},\n",
-    "    metrics=[],\n",
-    "    with_lds=True,\n",
-    "    lds_kernel=\"gaussian\",\n",
-    "    lds_ks=5,\n",
-    "    lds_sigma=2,\n",
-    "    lds_granularity=100,\n",
-    "    lds_reweight=False,\n",
-    "    lds_y_max=None,\n",
-    "    lds_y_min=None,\n",
-    ")\n",
-    "\n",
-    "trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n",
-    "\n",
-    "print(\n",
-    "    f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8ee72521",
-   "metadata": {},
-   "source": [
-    "# Model without FDS or LDS"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 18,
-   "id": "0836c4e9",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "epoch 1: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 129.54it/s, loss=0.445, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 137.64it/s, loss=0.427, metrics={}]\n",
-      "epoch 2: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 135.98it/s, loss=0.374, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 148.50it/s, loss=0.389, metrics={}]\n",
-      "epoch 3: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 127.72it/s, loss=0.359, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 147.63it/s, loss=0.383, metrics={}]\n",
-      "epoch 4: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 136.54it/s, loss=0.339, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 146.55it/s, loss=0.323, metrics={}]\n",
-      "epoch 5: 100%|██████████████████████████████████████████████████████████████████████| 331/331 [00:02<00:00, 131.18it/s, loss=0.331, metrics={}]\n",
-      "valid: 100%|██████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 174.87it/s, loss=0.318, metrics={}]\n",
-      "predict: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 42/42 [00:00<00:00, 287.25it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "test RMSE: 0.6014019159826868\n"
-     ]
-    }
-   ],
-   "source": [
-    "deeptabular = TabMlp(\n",
-    "    column_idx=tab_preprocessor.column_idx,\n",
-    "    continuous_cols=tab_preprocessor.continuous_cols,\n",
-    ")\n",
-    "model = WideDeep(deeptabular=deeptabular, fds=False)\n",
-    "\n",
-    "# Optimizers\n",
-    "deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n",
-    "# LR Scheduler\n",
-    "deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n",
-    "# Hyperparameters\n",
-    "trainer = Trainer(\n",
-    "    model,\n",
-    "    objective=\"huber\",\n",
-    "    lr_schedulers={\"deeptabular\": deep_sch},\n",
-    "    initializers={\"deeptabular\": XavierNormal},\n",
-    "    optimizers={\"deeptabular\": deep_opt},\n",
-    "    metrics=[],\n",
-    "    with_lds=False,\n",
-    ")\n",
-    "\n",
-    "trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n",
-    "\n",
-    "print(\n",
-    "    f\"test RMSE: {mean_squared_error(y_test, trainer.predict(X_tab=X_tab_test), squared=False)}\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "09055e82-c289-4e5d-a697-fd32bb68d018",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.13"
-  },
-  "toc": {
-   "base_numbering": 1,
-   "nav_menu": {},
-   "number_sections": true,
-   "sideBar": true,
-   "skip_h1_title": false,
-   "title_cell": "Table of Contents",
-   "title_sidebar": "Contents",
-   "toc_cell": false,
-   "toc_position": {},
-   "toc_section_display": true,
-   "toc_window_display": false
-  },
-  "vscode": {
-   "interpreter": {
-    "hash": "bee110fa72fc220f84be99700c69baf478c6696e63cfda5b1944123ebc470d26"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
diff --git a/mkdocs/sources/examples/16_Self_Supervised_Pretraning_pt1.ipynb b/mkdocs/sources/examples/15_Self_Supervised_Pretraning_pt1.ipynb
similarity index 100%
rename from mkdocs/sources/examples/16_Self_Supervised_Pretraning_pt1.ipynb
rename to mkdocs/sources/examples/15_Self_Supervised_Pretraning_pt1.ipynb
diff --git a/mkdocs/sources/examples/16_Self_Supervised_Pretraning_pt2.ipynb b/mkdocs/sources/examples/15_Self_Supervised_Pretraning_pt2.ipynb
similarity index 100%
rename from mkdocs/sources/examples/16_Self_Supervised_Pretraning_pt2.ipynb
rename to mkdocs/sources/examples/15_Self_Supervised_Pretraning_pt2.ipynb
diff --git a/mkdocs/sources/examples/17_Usign_a_custom_hugging_face_model.ipynb b/mkdocs/sources/examples/16_Usign_a_custom_hugging_face_model.ipynb
similarity index 100%
rename from mkdocs/sources/examples/17_Usign_a_custom_hugging_face_model.ipynb
rename to mkdocs/sources/examples/16_Usign_a_custom_hugging_face_model.ipynb
diff --git a/mkdocs/sources/examples/18_feature_importance_via_attention_weights.ipynb b/mkdocs/sources/examples/17_feature_importance_via_attention_weights.ipynb
similarity index 100%
rename from mkdocs/sources/examples/18_feature_importance_via_attention_weights.ipynb
rename to mkdocs/sources/examples/17_feature_importance_via_attention_weights.ipynb
diff --git a/mkdocs/sources/examples/19_wide_and_deep_for_recsys_pt1.ipynb b/mkdocs/sources/examples/18_wide_and_deep_for_recsys_pt1.ipynb
similarity index 100%
rename from mkdocs/sources/examples/19_wide_and_deep_for_recsys_pt1.ipynb
rename to mkdocs/sources/examples/18_wide_and_deep_for_recsys_pt1.ipynb
diff --git a/mkdocs/sources/examples/19_wide_and_deep_for_recsys_pt2.ipynb b/mkdocs/sources/examples/18_wide_and_deep_for_recsys_pt2.ipynb
similarity index 100%
rename from mkdocs/sources/examples/19_wide_and_deep_for_recsys_pt2.ipynb
rename to mkdocs/sources/examples/18_wide_and_deep_for_recsys_pt2.ipynb
diff --git a/mkdocs/sources/examples/20_load_from_folder_functionality.ipynb b/mkdocs/sources/examples/19_load_from_folder_functionality.ipynb
similarity index 100%
rename from mkdocs/sources/examples/20_load_from_folder_functionality.ipynb
rename to mkdocs/sources/examples/19_load_from_folder_functionality.ipynb
diff --git a/mkdocs/sources/examples/21_Using_huggingface_within_widedeep.ipynb b/mkdocs/sources/examples/20_Using_huggingface_within_widedeep.ipynb
similarity index 100%
rename from mkdocs/sources/examples/21_Using_huggingface_within_widedeep.ipynb
rename to mkdocs/sources/examples/20_Using_huggingface_within_widedeep.ipynb
diff --git a/mkdocs/sources/pytorch-widedeep/model_components.md b/mkdocs/sources/pytorch-widedeep/model_components.md
index 1257735a..10ef9e89 100644
--- a/mkdocs/sources/pytorch-widedeep/model_components.md
+++ b/mkdocs/sources/pytorch-widedeep/model_components.md
@@ -127,9 +127,3 @@ self-supervised pre-training with tabular data.
         filters:
             - "!^_"  # exclude all members starting with _
             - "!^forward$"
-
-::: pytorch_widedeep.models.fds_layer.FDSLayer
-    selection:
-        filters:
-            - "!^_"  # exclude all members starting with _
-            - "!^forward$"
diff --git a/pytorch_widedeep/dataloaders.py b/pytorch_widedeep/dataloaders.py
index be424f80..3c17bc5c 100644
--- a/pytorch_widedeep/dataloaders.py
+++ b/pytorch_widedeep/dataloaders.py
@@ -37,16 +37,6 @@ def get_class_weights(dataset: WideDeepDataset) -> Tuple[np.ndarray, int, int]:
     return weights, minor_class_count, num_classes
 
 
-class DataLoaderDefault(DataLoader):
-    def __init__(
-        self, dataset: WideDeepDataset, batch_size: int, num_workers: int, **kwargs
-    ):
-        self.with_lds = dataset.with_lds
-        super().__init__(
-            dataset=dataset, batch_size=batch_size, num_workers=num_workers, **kwargs
-        )
-
-
 class DataLoaderImbalanced(DataLoader):
     r"""Class to load and shuffle batches with adjusted weights for imbalanced
     datasets. If the classes do not begin from 0 remapping is necessary. See
@@ -87,7 +77,6 @@ def __init__(
             "target array 'Y'"
         )
 
-        self.with_lds = dataset.with_lds
         if "oversample_mul" in kwargs:
             oversample_mul = kwargs["oversample_mul"]
             del kwargs["oversample_mul"]
diff --git a/pytorch_widedeep/losses.py b/pytorch_widedeep/losses.py
index e82fa117..63530352 100644
--- a/pytorch_widedeep/losses.py
+++ b/pytorch_widedeep/losses.py
@@ -15,18 +15,18 @@
 
 
 class MSELoss(nn.Module):
-    r"""Mean square error loss with the option of using Label Smooth
-    Distribution (LDS)
-
-    LDS is based on
-    [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).
-    """
+    r"""Mean square error loss"""
 
+    # legacy code from when we used to support FDS-LDS and this class could
+    # taked the corresponding params. At this stage probably you want to use
+    # torch.nn.MSELoss
     def __init__(self):
         super().__init__()
 
     def forward(
-        self, input: Tensor, target: Tensor, lds_weight: Optional[Tensor] = None
+        self,
+        input: Tensor,
+        target: Tensor,
     ) -> Tensor:
         r"""
         Parameters
@@ -35,8 +35,6 @@ def forward(
             Input tensor with predictions
         target: Tensor
             Target tensor with the actual values
-        lds_weight: Tensor, Optional
-            Tensor of weights that will multiply the loss value.
 
         Examples
         --------
@@ -45,28 +43,22 @@ def forward(
         >>>
         >>> target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)
         >>> input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)
-        >>> lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)
-        >>> loss = MSELoss()(input, target, lds_weight)
+        >>> loss = MSELoss()(input, target)
         """
         loss = (input - target) ** 2
-        if lds_weight is not None:
-            loss *= lds_weight
         return torch.mean(loss)
 
 
 class MSLELoss(nn.Module):
-    r"""Mean square log error loss with the option of using Label Smooth
-    Distribution (LDS)
-
-    LDS is based on
-    [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).
-    """
+    r"""Mean square log error loss"""
 
     def __init__(self):
         super().__init__()
 
     def forward(
-        self, input: Tensor, target: Tensor, lds_weight: Optional[Tensor] = None
+        self,
+        input: Tensor,
+        target: Tensor,
     ) -> Tensor:
         r"""
         Parameters
@@ -75,8 +67,6 @@ def forward(
             Input tensor with predictions (not probabilities)
         target: Tensor
             Target tensor with the actual classes
-        lds_weight: Tensor, Optional
-            Tensor of weights that will multiply the loss value.
 
         Examples
         --------
@@ -85,8 +75,7 @@ def forward(
         >>>
         >>> target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)
         >>> input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)
-        >>> lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)
-        >>> loss = MSLELoss()(input, target, lds_weight)
+        >>> loss = MSLELoss()(input, target)
         """
         assert (
             input.min() >= 0
@@ -96,25 +85,19 @@ def forward(
         assert target.min() >= 0, "All target values must be >=0"
 
         loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2
-        if lds_weight is not None:
-            loss *= lds_weight
         return torch.mean(loss)
 
 
 class RMSELoss(nn.Module):
-    r"""Root mean square error loss adjusted for the possibility of using Label
-    Smooth Distribution (LDS)
-
-    LDS is based on
-    [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).
-    """
+    r"""Root mean square error loss"""
 
+    # legacy code from when we used to support FDS-LDS and this class could
+    # taked the corresponding params. At this stage probably you want to use
+    # torch.sqrt(nn.MSELoss)
     def __init__(self):
         super().__init__()
 
-    def forward(
-        self, input: Tensor, target: Tensor, lds_weight: Optional[Tensor] = None
-    ) -> Tensor:
+    def forward(self, input: Tensor, target: Tensor) -> Tensor:
         r"""
         Parameters
         ----------
@@ -122,8 +105,6 @@ def forward(
             Input tensor with predictions (not probabilities)
         target: Tensor
             Target tensor with the actual classes
-        lds_weight: Tensor, Optional
-            Tensor of weights that will multiply the loss value.
 
         Examples
         --------
@@ -132,29 +113,19 @@ def forward(
         >>>
         >>> target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)
         >>> input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)
-        >>> lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)
-        >>> loss = RMSELoss()(input, target, lds_weight)
+        >>> loss = RMSELoss()(input, target)
         """
         loss = (input - target) ** 2
-        if lds_weight is not None:
-            loss *= lds_weight
         return torch.sqrt(torch.mean(loss))
 
 
 class RMSLELoss(nn.Module):
-    r"""Root mean square log error loss adjusted for the possibility of using Label
-    Smooth Distribution (LDS)
-
-    LDS is based on
-    [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).
-    """
+    r"""Root mean square log error loss"""
 
     def __init__(self):
         super().__init__()
 
-    def forward(
-        self, input: Tensor, target: Tensor, lds_weight: Optional[Tensor] = None
-    ) -> Tensor:
+    def forward(self, input: Tensor, target: Tensor) -> Tensor:
         r"""
         Parameters
         ----------
@@ -162,8 +133,6 @@ def forward(
             Input tensor with predictions (not probabilities)
         target: Tensor
             Target tensor with the actual classes
-        lds_weight: Tensor, Optional
-            Tensor of weights that will multiply the loss value.
 
         Examples
         --------
@@ -172,8 +141,7 @@ def forward(
         >>>
         >>> target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)
         >>> input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)
-        >>> lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)
-        >>> loss = RMSLELoss()(input, target, lds_weight)
+        >>> loss = RMSLELoss()(input, target)
         """
         assert (
             input.min() >= 0
@@ -183,8 +151,6 @@ def forward(
         assert target.min() >= 0, "All target values must be >=0"
 
         loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2
-        if lds_weight is not None:
-            loss *= lds_weight
         return torch.sqrt(torch.mean(loss))
 
 
@@ -388,7 +354,6 @@ def forward(
         self,
         input: Tensor,
         target: Tensor,
-        lds_weight: Optional[Tensor] = None,
         p: float = 1.5,
     ) -> Tensor:
         r"""
@@ -398,9 +363,6 @@ def forward(
             Input tensor with predictions
         target: Tensor
             Target tensor with the actual values
-        lds_weight: Tensor, Optional
-            If we choose to use LDS this is the tensor of weights that will
-            multiply the loss value.
         p: float, default = 1.5
             the power to be used to compute the loss. See the original
             publication for details
@@ -412,8 +374,7 @@ def forward(
         >>>
         >>> target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)
         >>> input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)
-        >>> lds_weight = torch.tensor([0.1, 0.2, 0.3, 0.4]).view(-1, 1)
-        >>> loss = TweedieLoss()(input, target, lds_weight)
+        >>> loss = TweedieLoss()(input, target)
         """
 
         assert (
@@ -425,9 +386,6 @@ def forward(
         loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (
             2 - p
         )
-        if lds_weight is not None:
-            loss *= lds_weight
-
         return torch.mean(loss)
 
 
@@ -497,19 +455,15 @@ def forward(self, input: Tensor, target: Tensor) -> Tensor:
 
 
 class L1Loss(nn.Module):
-    r"""L1 loss adjusted for the possibility of using Label Smooth
-    Distribution (LDS)
-
-    LDS is based on
-    [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).
-    """
+    r"""L1 loss"""
 
+    # legacy code from when we used to support FDS-LDS and this class could
+    # taked the corresponding params. At this stage probably you want to use
+    # torch.nn.L1Loss
     def __init__(self):
         super().__init__()
 
-    def forward(
-        self, input: Tensor, target: Tensor, lds_weight: Optional[Tensor] = None
-    ) -> Tensor:
+    def forward(self, input: Tensor, target: Tensor) -> Tensor:
         r"""
         Parameters
         ----------
@@ -517,9 +471,6 @@ def forward(
             Input tensor with predictions
         target: Tensor
             Target tensor with the actual values
-        lds_weight: Tensor, Optional
-            If we choose to use LDS this is the tensor of weights that will
-            multiply the loss value.
 
         Examples
         --------
@@ -532,8 +483,6 @@ def forward(
         >>> loss = L1Loss()(input, target)
         """
         loss = F.l1_loss(input, target, reduction="none")
-        if lds_weight is not None:
-            loss *= lds_weight
         return torch.mean(loss)
 
 
@@ -569,7 +518,6 @@ def forward(
         self,
         input: Tensor,
         target: Tensor,
-        lds_weight: Optional[Tensor] = None,
     ) -> Tensor:
         r"""
         Parameters
@@ -578,9 +526,6 @@ def forward(
             Input tensor with predictions (not probabilities)
         target: Tensor
             Target tensor with the actual classes
-        lds_weight: Tensor, Optional
-            If we choose to use LDS this is the tensor of weights that will
-            multiply the loss value.
 
         Examples
         --------
@@ -603,8 +548,6 @@ def forward(
             ValueError(
                 "Incorrect activation function value - must be in ['sigmoid', 'tanh']"
             )
-        if lds_weight is not None:
-            loss *= lds_weight
         return torch.mean(loss)
 
 
@@ -640,7 +583,6 @@ def forward(
         self,
         input: Tensor,
         target: Tensor,
-        lds_weight: Optional[Tensor] = None,
     ) -> Tensor:
         r"""
         Parameters
@@ -649,9 +591,6 @@ def forward(
             Input tensor with predictions (not probabilities)
         target: Tensor
             Target tensor with the actual classes
-        lds_weight: Tensor, Optional
-            If we choose to use LDS this is the tensor of weights that will
-            multiply the loss value.
 
         Examples
         --------
@@ -674,8 +613,6 @@ def forward(
             ValueError(
                 "Incorrect activation function value - must be in ['sigmoid', 'tanh']"
             )
-        if lds_weight is not None:
-            loss *= lds_weight
         return torch.mean(loss)
 
 
@@ -711,7 +648,6 @@ def forward(
         self,
         input: Tensor,
         target: Tensor,
-        lds_weight: Optional[Tensor] = None,
     ) -> Tensor:
         r"""
         Parameters
@@ -720,9 +656,6 @@ def forward(
             Input tensor with predictions (not probabilities)
         target: Tensor
             Target tensor with the actual classes
-        lds_weight: Tensor, Optional
-            If we choose to use LDS this is the tensor of weights that will
-            multiply the loss value.
 
         Examples
         --------
@@ -745,8 +678,6 @@ def forward(
             ValueError(
                 "Incorrect activation function value - must be in ['sigmoid', 'tanh']"
             )
-        if lds_weight is not None:
-            loss *= lds_weight
         return torch.sqrt(torch.mean(loss))
 
 
@@ -764,7 +695,6 @@ def forward(
         self,
         input: Tensor,
         target: Tensor,
-        lds_weight: Optional[Tensor] = None,
     ) -> Tensor:
         r"""
         Parameters
@@ -773,9 +703,6 @@ def forward(
             Input tensor with predictions (not probabilities)
         target: Tensor
             Target tensor with the actual classes
-        lds_weight: Tensor, Optional
-            If we choose to use LDS this is the tensor of weights that will
-            multiply the loss value.
 
         Examples
         --------
@@ -792,8 +719,6 @@ def forward(
         loss = torch.where(
             cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta
         )
-        if lds_weight is not None:
-            loss *= lds_weight
         return torch.mean(loss)
 
 
diff --git a/pytorch_widedeep/models/fds_layer.py b/pytorch_widedeep/models/fds_layer.py
deleted file mode 100644
index 81baf5b4..00000000
--- a/pytorch_widedeep/models/fds_layer.py
+++ /dev/null
@@ -1,278 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from pytorch_widedeep.wdtypes import Tuple, Union, Tensor, Literal, Optional
-from pytorch_widedeep.utils.deeptabular_utils import (
-    find_bin,
-    get_kernel_window,
-)
-
-
-class FDSLayer(nn.Module):
-    def __init__(
-        self,
-        feature_dim: int,
-        granularity: int = 100,
-        y_max: Optional[float] = None,
-        y_min: Optional[float] = None,
-        start_update: int = 0,
-        start_smooth: int = 2,
-        kernel: Literal["gaussian", "triang", "laplace"] = "gaussian",
-        ks: int = 5,
-        sigma: float = 2,
-        momentum: Optional[float] = 0.9,
-        clip_min: Optional[float] = None,
-        clip_max: Optional[float] = None,
-    ):
-        """
-        Feature Distribution Smoothing layer. Please, see
-        [Delving into Deep Imbalanced Regression](https:/arxiv.org/abs/2102.09554)
-        for details.
-
-        :information_source: **NOTE**: this is NOT an available model per se,
-         but more a utility that can be used as we run a `WideDeep` model.
-         The parameters of this extra layers can be set as the class
-         `WideDeep` is instantiated via the keyword arguments `fds_config`.
-
-        :information_source: **NOTE**: Feature Distribution Smoothing is
-         available when using ONLY a `deeptabular` component
-
-        :information_source: **NOTE**: We consider this feature absolutely
-        experimental and we recommend the user to not use it unless the
-        corresponding [publication](https://arxiv.org/abs/2102.09554) is
-        well understood
-
-        The code here is based on the code at the
-        [official repo](https://github.com/YyzHarry/imbalanced-regression)
-
-        Parameters
-        ----------
-        feature_dim: int,
-            input dimension size, i.e. output size of previous layer. This
-            will be the dimension of the output from the `deeptabular`
-            component
-        granularity: int = 100,
-            number of bins that the target $y$ is divided into and that will
-            be used to compute the features' statistics (mean and variance)
-        y_max: Optional[float] = None,
-            $y$ upper limit to be considered when binning
-        y_min: Optional[float] = None,
-            $y$ lower limit to be considered when binning
-        start_update: int = 0,
-            number of _'waiting epochs' after which the FDS layer will start
-            to update its statistics
-        start_smooth: int = 1,
-            number of _'waiting epochs' after which the FDS layer will start
-            smoothing the feature distributions
-        kernel: Literal["gaussian", "triang", "laplace", None] = "gaussian",
-            choice of smoothing kernel
-        ks: int = 5,
-            kernel window size
-        sigma: Union[int, float] = 2,
-            if a _'gaussian'_ or _'laplace'_ kernels are used, this is the
-            corresponding standard deviation
-        momentum: float = 0.9,
-            to train the layer the authors used a momentum update of the running
-            statistics across each epoch. Set to 0.9 in the paper.
-        clip_min: Optional[float] = None,
-            this parameter is used to clip the ratio between the so called
-            running variance and the smoothed variance, and is introduced for
-            numerical stability. We leave it as optional as we did not find a
-            notable improvement in our experiments. The authors used a value
-            of 0.1
-        clip_max: Optional[float] = None,
-            same as `clip_min` but for the upper limit.We leave it as optional
-            as we did not find a notable improvement in our experiments. The
-            authors used a value of 10.
-        """
-        super(FDSLayer, self).__init__()
-        assert (
-            start_update + 1 < start_smooth
-        ), "initial update must start at least 2 epoch before smoothing"
-
-        self.feature_dim = feature_dim
-        self.granularity = granularity
-        self.y_max = y_max
-        self.y_min = y_min
-        self.kernel_window = torch.tensor(
-            get_kernel_window(kernel, ks, sigma), dtype=torch.float32
-        )
-        self.half_ks = (ks - 1) // 2
-        self.momentum = momentum
-        self.start_update = start_update
-        self.start_smooth = start_smooth
-        self.clip_min = clip_min
-        self.clip_max = clip_max
-
-        self.pred_layer = nn.Linear(feature_dim, 1)
-
-        self._register_buffers()
-
-    def forward(self, features, labels, epoch) -> Union[Tuple[Tensor, Tensor], Tensor]:
-        if self.training:
-            features = self._smooth(features, labels, epoch)
-            return (features, self.pred_layer(features))
-        else:
-            return self.pred_layer(features)
-
-    def reset(self):
-        self.running_mean.zero_()
-        self.running_var.fill_(1)
-        self.running_mean_last_epoch.zero_()
-        self.running_var_last_epoch.fill_(1)
-        self.smoothed_mean_last_epoch.zero_()
-        self.smoothed_var_last_epoch.fill_(1)
-        self.num_samples_tracked.zero_()
-
-    def update_last_epoch_stats(self, epoch):
-        if epoch > self.start_update:
-            self.running_mean_last_epoch = self.running_mean
-            self.running_var_last_epoch = self.running_var
-
-            smoothed_mean_last_epoch_inp = F.pad(
-                self.running_mean_last_epoch.unsqueeze(1).permute(2, 1, 0),
-                pad=(self.half_ks, self.half_ks),
-                mode="reflect",
-            )
-            smoothed_mean_last_epoch_weight = self.kernel_window.view(1, 1, -1).to(
-                smoothed_mean_last_epoch_inp.device
-            )
-            self.smoothed_mean_last_epoch = (
-                F.conv1d(
-                    input=smoothed_mean_last_epoch_inp,
-                    weight=smoothed_mean_last_epoch_weight,
-                    padding=0,
-                )
-                .permute(2, 1, 0)
-                .squeeze(1)
-            )
-
-            smoothed_var_last_epoch_inp = F.pad(
-                self.running_var_last_epoch.unsqueeze(1).permute(2, 1, 0),
-                pad=(self.half_ks, self.half_ks),
-                mode="reflect",
-            )
-            smoothed_var_last_epoch_weight = self.kernel_window.view(1, 1, -1).to(
-                smoothed_var_last_epoch_inp.device
-            )
-            self.smoothed_var_last_epoch = (
-                F.conv1d(
-                    input=smoothed_var_last_epoch_inp,
-                    weight=smoothed_var_last_epoch_weight,
-                    padding=0,
-                )
-                .permute(2, 1, 0)
-                .squeeze(1)
-            )
-
-    def update_running_stats(self, features, labels, epoch):
-        assert self.feature_dim == features.size(
-            1
-        ), "Input feature dimension is not aligned!"
-        assert features.size(0) == labels.size(
-            0
-        ), "Dimensions of features and labels are not aligned!"
-
-        if epoch == 0:
-            if not self.y_max:
-                self.y_max = labels.max()
-            if not self.y_min:
-                self.y_min = labels.min()
-            bin_edges = torch.linspace(self.y_min, self.y_max, steps=self.granularity)
-            self.register_buffer("bin_edges", bin_edges)
-
-        if epoch >= self.start_update:
-            left_bin_edges_indices = find_bin(
-                self.bin_edges, labels.squeeze(), ret_value=False
-            )
-            for left_bin_edge_ind in torch.unique(left_bin_edges_indices):
-                inds = (left_bin_edges_indices == left_bin_edge_ind).nonzero().squeeze()
-                curr_feats = features[inds]
-                curr_num_sample = curr_feats.size(0)
-                curr_mean = torch.mean(curr_feats, 0)
-                curr_var = torch.var(
-                    curr_feats, 0, unbiased=True if curr_feats.size(0) != 1 else False
-                )
-
-                self.num_samples_tracked[left_bin_edge_ind] += curr_num_sample
-                if not self.momentum:
-                    factor = 1 - curr_num_sample / float(
-                        self.num_samples_tracked[left_bin_edge_ind]
-                    )
-                else:
-                    factor = self.momentum
-                if epoch == self.start_update:
-                    factor = 0
-                self.running_mean[left_bin_edge_ind] = (
-                    1 - factor
-                ) * curr_mean + factor * self.running_mean[left_bin_edge_ind]
-                self.running_var[left_bin_edge_ind] = (
-                    1 - factor
-                ) * curr_var + factor * self.running_var[left_bin_edge_ind]
-
-    def _smooth(self, features, labels, epoch):
-        smoothed_features = features.detach()
-
-        if epoch >= self.start_smooth:
-            left_bin_edges_indices = find_bin(
-                self.bin_edges, labels.squeeze(), ret_value=False
-            )
-            for left_bin_edge_ind in torch.unique(left_bin_edges_indices):
-                inds = (left_bin_edges_indices == left_bin_edge_ind).nonzero().squeeze()
-                smoothed_features[inds] = self._calibrate_mean_var(
-                    smoothed_features[inds], left_bin_edge_ind
-                )
-
-        return smoothed_features
-
-    def _calibrate_mean_var(self, features, left_bin_edge_ind):
-        # rescaling of the data https://stats.stackexchange.com/a/46431
-        m1 = self.running_mean_last_epoch[left_bin_edge_ind]
-        v1 = self.running_var_last_epoch[left_bin_edge_ind]
-        m2 = self.smoothed_mean_last_epoch[left_bin_edge_ind]
-        v2 = self.smoothed_var_last_epoch[left_bin_edge_ind]
-        if torch.sum(v1) < 1e-10:
-            return features
-        if (v1 == 0.0).any():
-            valid = v1 != 0.0
-            factor = v2[valid] / v1[valid]
-            if self.clip_min and self.clip_max:
-                factor = torch.clamp(factor, self.clip_min, self.clip_max)
-            if features.dim() == 1:
-                # the tensor has to be 2d
-                features = features.unsqueeze(0)
-            features[:, valid] = (features[:, valid] - m1[valid]) * torch.sqrt(
-                factor
-            ) + m2[valid]
-            return features
-
-        factor = v2 / v1
-        if self.clip_min and self.clip_max:
-            factor = torch.clamp(factor, self.clip_min, self.clip_max)
-
-        return (features - m1) * torch.sqrt(factor) + m2
-
-    def _register_buffers(self):
-        self.register_buffer(
-            "running_mean", torch.zeros(self.granularity - 1, self.feature_dim)
-        )
-        self.register_buffer(
-            "running_var", torch.ones(self.granularity - 1, self.feature_dim)
-        )
-        self.register_buffer(
-            "running_mean_last_epoch",
-            torch.zeros(self.granularity - 1, self.feature_dim),
-        )
-        self.register_buffer(
-            "running_var_last_epoch", torch.ones(self.granularity - 1, self.feature_dim)
-        )
-        self.register_buffer(
-            "smoothed_mean_last_epoch",
-            torch.zeros(self.granularity - 1, self.feature_dim),
-        )
-        self.register_buffer(
-            "smoothed_var_last_epoch",
-            torch.ones(self.granularity - 1, self.feature_dim),
-        )
-        self.register_buffer("num_samples_tracked", torch.zeros(self.granularity - 1))
diff --git a/pytorch_widedeep/models/model_fusion.py b/pytorch_widedeep/models/model_fusion.py
index 7e8c6aad..a667aae4 100644
--- a/pytorch_widedeep/models/model_fusion.py
+++ b/pytorch_widedeep/models/model_fusion.py
@@ -1,6 +1,7 @@
 import torch
 from torch import nn
 
+from pytorch_widedeep.models import TabNet
 from pytorch_widedeep.wdtypes import List, Union, Tensor, Literal, Optional
 from pytorch_widedeep.models.tabular.mlp._layers import MLP
 from pytorch_widedeep.models._base_wd_model_component import (
@@ -20,10 +21,10 @@ class ModelFuser(BaseWDModelComponent):
         List of models whose outputs will be fused
     fusion_method: Union[str, List[str]]
         Method to fuse the output of the models. It can be one of
-        ['concatenate', 'mean', 'max', 'sum', 'mult', 'head'] or a list of
-        those. If a list is provided the output of the models will be fused
-        using all the methods in the list and the final output will be the
-        concatenation of the outputs of each method
+        ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a
+        list of those, but 'dot'. If a list is provided the output of the
+        models will be fused using all the methods in the list and the final
+        output will be the concatenation of the outputs of each method
     projection_method: Optional[str]
         If the fusion_method is not 'concatenate', this parameter will
         determine how to project the output of the models to a common
@@ -110,6 +111,7 @@ def __init__(
                 "max",
                 "sum",
                 "mult",
+                "dot",
                 "head",
             ],
             List[Literal["concatenate", "mean", "max", "sum", "mult", "head"]],
@@ -182,11 +184,20 @@ def forward(self, X: List[Tensor]) -> Tensor:  # noqa: C901
             return self.head(
                 torch.cat([model(x) for model, x in zip(self.models, X)], -1)
             )
+        elif self.fusion_method == "dot":
+            assert len(X) == 2, (
+                "When using 'dot' as fusion_method, only two models "
+                " can be fused. Accordingly, only two inputs should be provided"
+            )
+            outputs = [model(x) for model, x in zip(self.models, X)]
+            return torch.bmm(outputs[1].unsqueeze(1), outputs[0].unsqueeze(2)).view(
+                -1, 1
+            )
         else:
             if isinstance(self.fusion_method, str):
                 fusion_methods = [self.fusion_method]
             else:
-                fusion_methods = self.fusion_method
+                fusion_methods = self.fusion_method  # type: ignore
 
             fused_outputs: List[Tensor] = []
             for fm in fusion_methods:
@@ -210,7 +221,7 @@ def forward(self, X: List[Tensor]) -> Tensor:  # noqa: C901
                     else:
                         # This should never happen, but avoids type errors
                         raise ValueError(
-                            "fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'head'] "
+                            "fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] "
                             "or a list of those"
                         )
                 fused_outputs.append(out)
@@ -260,12 +271,14 @@ def output_dim(self) -> int:
                 if hasattr(self, "head_hidden_dims")
                 else self.head.output_dim
             )
+        elif self.fusion_method == "dot":
+            output_dim = 1
         else:
             output_dim = 0
             if isinstance(self.fusion_method, str):
                 fusion_methods = [self.fusion_method]
             else:
-                fusion_methods = self.fusion_method
+                fusion_methods = self.fusion_method  # type: ignore
             for fm in fusion_methods:
                 if fm == "concatenate":
                     output_dim += sum([model.output_dim for model in self.models])
@@ -288,14 +301,30 @@ def output_dim(self) -> int:
         return output_dim
 
     def check_input_parameters(self):  # noqa: C901
+
+        if any(isinstance(model, TabNet) for model in self.models):
+            raise ValueError(
+                "TabNet is not supported in ModelFuser. "
+                "Please, use another model for tabular data"
+            )
+
         if isinstance(self.fusion_method, str):
             if not any(
                 x == self.fusion_method
-                for x in ["concatenate", "min", "max", "mean", "sum", "mult", "head"]
+                for x in [
+                    "concatenate",
+                    "min",
+                    "max",
+                    "mean",
+                    "sum",
+                    "dot",
+                    "mult",
+                    "head",
+                ]
             ):
                 raise ValueError(
-                    "fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'head'] "
-                    "or a list of those"
+                    "fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] "
+                    "or a list of any those but 'dot'"
                 )
 
             if (
@@ -323,14 +352,15 @@ def check_input_parameters(self):  # noqa: C901
                         "mean",
                         "sum",
                         "mult",
+                        "dot",
                         "head",
                     ]
                 )
                 for fm in self.fusion_method
             ):
                 raise ValueError(
-                    "fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'head'] "
-                    "or a list of those"
+                    "fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] "
+                    "or a list of those but 'dot'"
                 )
 
             if (
@@ -351,9 +381,11 @@ def check_input_parameters(self):  # noqa: C901
                         "projection_method must be one of ['min', 'max', 'mean']"
                     )
 
-        if "head" in self.fusion_method and isinstance(self.fusion_method, list):
+        if any(x in self.fusion_method for x in ["head", "dot"]) and isinstance(
+            self.fusion_method, list
+        ):
             raise ValueError(
-                "When using 'head' as fusion_method, no other method should be provided"
+                "When using 'head' or 'dot' as fusion_method, no other method should be provided"
             )
 
     def __repr__(self):
diff --git a/pytorch_widedeep/models/text/rnns/basic_rnn.py b/pytorch_widedeep/models/text/rnns/basic_rnn.py
index 78d4e670..87db8e16 100644
--- a/pytorch_widedeep/models/text/rnns/basic_rnn.py
+++ b/pytorch_widedeep/models/text/rnns/basic_rnn.py
@@ -2,7 +2,7 @@
 
 import numpy as np
 import torch
-from torch import nn, lstm
+from torch import nn
 
 from pytorch_widedeep.wdtypes import (
     List,
diff --git a/pytorch_widedeep/models/wide_deep.py b/pytorch_widedeep/models/wide_deep.py
index 922bbab5..7d131b44 100644
--- a/pytorch_widedeep/models/wide_deep.py
+++ b/pytorch_widedeep/models/wide_deep.py
@@ -3,8 +3,15 @@
 import torch
 from torch import nn
 
-from pytorch_widedeep.wdtypes import Dict, List, Tuple, Union, Tensor, Optional
-from pytorch_widedeep.models.fds_layer import FDSLayer
+from pytorch_widedeep.wdtypes import (
+    Dict,
+    List,
+    Tuple,
+    Union,
+    Tensor,
+    Literal,
+    Optional,
+)
 from pytorch_widedeep.utils.general_utils import alias
 from pytorch_widedeep.models._get_activation_fn import get_activation_fn
 from pytorch_widedeep.models.tabular.mlp._layers import MLP
@@ -46,7 +53,9 @@ class WideDeep(nn.Module):
     deeptabular: BaseWDModelComponent, Optional, default = None
         Currently this library implements a number of possible architectures
         for the `deeptabular` component. See the documenation of the
-        package.
+        package. Note that `deeptabular` can be a list of models. This is
+        useful when using multiple tabular inputs (e.g. for example in the
+        context of a two-tower model for recommendation systems)
     deeptext: BaseWDModelComponent | List[BaseWDModelComponent], Optional, default = None
         Currently this library implements a number of possible architectures
         for the `deeptext` component. See the documenation of the
@@ -90,25 +99,6 @@ class WideDeep(nn.Module):
         Size of the final wide and deep output layer containing the
         predictions. `1` for regression and binary classification or number
         of classes for multiclass classification.
-    with_fds: bool, default = False
-        Boolean indicating if Feature Distribution Smoothing (FDS) will be
-        applied before the final prediction layer. Only available for
-        regression problems.
-        See [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554) for details.
-
-    Other Parameters
-    ----------------
-    **fds_config: dict, default = None
-        Dictionary with the parameters to be used when using Feature
-        Distribution Smoothing. Please, see the docs for the `FDSLayer`.
-        <br/>
-        :information_source: **NOTE**: Feature Distribution Smoothing
-         is available when using **ONLY** a `deeptabular` component
-        <br/>
-        :information_source: **NOTE**: We consider Feature Distribution Smoothing
-        absolutely experimental and we recommend the user to not use it unless the
-        corresponding [publication](https://arxiv.org/abs/2102.09554) is
-        well understood
 
 
     Examples
@@ -141,7 +131,9 @@ class WideDeep(nn.Module):
     def __init__(
         self,
         wide: Optional[nn.Module] = None,
-        deeptabular: Optional[BaseWDModelComponent] = None,
+        deeptabular: Optional[
+            Union[BaseWDModelComponent, List[BaseWDModelComponent]]
+        ] = None,
         deeptext: Optional[
             Union[BaseWDModelComponent, List[BaseWDModelComponent]]
         ] = None,
@@ -158,8 +150,6 @@ def __init__(
         enforce_positive: bool = False,
         enforce_positive_activation: str = "softplus",
         pred_dim: int = 1,
-        with_fds: bool = False,
-        **fds_config,
     ):
         super(WideDeep, self).__init__()
 
@@ -171,7 +161,6 @@ def __init__(
             deephead,
             head_hidden_dims,
             pred_dim,
-            with_fds,
         )
 
         # this attribute will be eventually over-written by the Trainer's
@@ -181,9 +170,14 @@ def __init__(
         # required as attribute just in case we pass a deephead
         self.pred_dim = pred_dim
 
-        self.with_fds = with_fds
         self.enforce_positive = enforce_positive
 
+        # better to set this attribute already here
+        if isinstance(deeptabular, list):
+            self.is_tabnet = False
+        else:
+            self.is_tabnet = deeptabular.__class__.__name__ == "TabNet"
+
         # The main 5 components of the wide and deep assemble: wide,
         # deeptabular, deeptext, deepimage and deephead
         self.with_deephead = deephead is not None or head_hidden_dims is not None
@@ -209,12 +203,9 @@ def __init__(
 
         self.wide = wide
         self.deeptabular, self.deeptext, self.deepimage = self._set_model_components(
-            deeptabular, deeptext, deepimage, self.with_deephead
+            deeptabular, deeptext, deepimage
         )
 
-        if self.with_fds:
-            self.fds_layer = FDSLayer(feature_dim=self.deeptabular.output_dim, **fds_config)  # type: ignore[arg-type]
-
         if self.enforce_positive:
             self.enf_pos = get_activation_fn(enforce_positive_activation)
 
@@ -222,10 +213,7 @@ def forward(
         self,
         X: Dict[str, Union[Tensor, List[Tensor]]],
         y: Optional[Tensor] = None,
-        epoch: Optional[int] = None,
     ) -> Union[Tensor, Tuple[Tensor, Tensor]]:
-        if self.with_fds:
-            return self._forward_deep_with_fds(X, y, epoch)  # type: ignore[arg-type]
 
         wide_out = self._forward_wide(X)
         if self.with_deephead:
@@ -240,7 +228,7 @@ def forward(
 
     def _build_deephead(
         self,
-        deeptabular: Optional[BaseWDModelComponent],
+        deeptabular: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],
         deeptext: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],
         deepimage: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],
         head_hidden_dims: List[int],
@@ -252,8 +240,11 @@ def _build_deephead(
     ) -> nn.Sequential:
         deep_dim = 0
         if deeptabular is not None:
-            deep_dim += deeptabular.output_dim
-
+            if isinstance(deeptabular, list):
+                for dt in deeptabular:
+                    deep_dim += dt.output_dim
+            else:
+                deep_dim += deeptabular.output_dim
         if deeptext is not None:
             if isinstance(deeptext, list):
                 for dt in deeptext:
@@ -284,83 +275,26 @@ def _build_deephead(
 
     def _set_model_components(  # noqa: C901
         self,
-        deeptabular: Optional[BaseWDModelComponent],
+        deeptabular: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],
         deeptext: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],
         deepimage: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],
-        with_deephead: bool,
     ) -> Tuple[
-        Optional[WDModel],
+        Optional[Union[nn.ModuleList, WDModel]],
         Optional[Union[nn.ModuleList, WDModel]],
         Optional[Union[nn.ModuleList, WDModel]],
     ]:
         if deeptabular is not None:
-            self.is_tabnet = deeptabular.__class__.__name__ == "TabNet"
-        else:
-            self.is_tabnet = False
-
-        if deeptabular is not None:
-            # if FDS the FDS Layer already includes the pred layer
-            if not self.with_fds:
-                if self.is_tabnet:
-                    deeptabular_ = (
-                        nn.Sequential(
-                            deeptabular,
-                            TabNetPredLayer(deeptabular.output_dim, self.pred_dim),
-                        )
-                        if not with_deephead
-                        else deeptabular
-                    )
-                else:
-                    deeptabular_ = (
-                        nn.Sequential(
-                            deeptabular,
-                            nn.Linear(deeptabular.output_dim, self.pred_dim),
-                        )
-                        if not with_deephead
-                        else deeptabular
-                    )
-            else:
-                deeptabular_ = deeptabular
+            deeptabular_ = self._set_model_component(deeptabular, is_deeptabular=True)
         else:
             deeptabular_ = None
 
         if deeptext is not None:
-            if isinstance(deeptext, list):
-                deeptext_: Optional[Union[nn.ModuleList, WDModel]] = nn.ModuleList()
-                for dt in deeptext:
-                    deeptext_.append(
-                        nn.Sequential(dt, nn.Linear(dt.output_dim, self.pred_dim))
-                        if not with_deephead
-                        else dt
-                    )
-            else:
-                deeptext_ = (
-                    nn.Sequential(
-                        deeptext, nn.Linear(deeptext.output_dim, self.pred_dim)
-                    )
-                    if not with_deephead
-                    else deeptext
-                )
+            deeptext_ = self._set_model_component(deeptext)
         else:
             deeptext_ = None
 
         if deepimage is not None:
-            if isinstance(deepimage, list):
-                deepimage_: Optional[Union[nn.ModuleList, WDModel]] = nn.ModuleList()
-                for di in deepimage:
-                    deepimage_.append(
-                        nn.Sequential(di, nn.Linear(di.output_dim, self.pred_dim))
-                        if not with_deephead
-                        else di
-                    )
-            else:
-                deepimage_ = (
-                    nn.Sequential(
-                        deepimage, nn.Linear(deepimage.output_dim, self.pred_dim)
-                    )
-                    if not with_deephead
-                    else deepimage
-                )
+            deepimage_ = self._set_model_component(deepimage)
         else:
             deepimage_ = None
 
@@ -379,103 +313,126 @@ def _forward_wide(self, X: Dict[str, Union[Tensor, List[Tensor]]]) -> Tensor:
 
         return out
 
-    def _forward_deephead(
+    def _forward_deep(
         self, X: Dict[str, Union[Tensor, List[Tensor]]], wide_out: Tensor
     ) -> Union[Tensor, Tuple[Tensor, Tensor]]:
         if self.deeptabular is not None:
             if self.is_tabnet:
-                tab_out = self.deeptabular(X["deeptabular"])
-                deepside, M_loss = tab_out[0], tab_out[1]
+                tab_out, M_loss = self.deeptabular(X["deeptabular"])
+                wide_out.add_(tab_out)
             else:
-                deepside = self.deeptabular(X["deeptabular"])
-        else:
-            deepside = torch.FloatTensor().to(self.wd_device)
+                wide_out = self._forward_component(
+                    X, self.deeptabular, "deeptabular", wide_out
+                )
 
         if self.deeptext is not None:
-            if isinstance(self.deeptext, list):
-                deeptext_out = torch.cat(  # type: ignore[call-overload]
-                    [dt(X["deeptext"]) for dt in self.deeptext], axis=1
-                )
-            else:
-                deeptext_out = self.deeptext(X["deeptext"])
-            deepside = torch.cat([deepside, deeptext_out], axis=1)  # type: ignore[call-overload]
-        if self.deepimage is not None:
-            if isinstance(self.deepimage, list):
-                deepimage_out = torch.cat(  # type: ignore[call-overload]
-                    [di(X["deepimage"]) for di in self.deepimage], axis=1
-                )
-            else:
-                deepimage_out = self.deepimage(X["deepimage"])
-            deepside = torch.cat([deepside, deepimage_out], axis=1)  # type: ignore[call-overload]
+            wide_out = self._forward_component(X, self.deeptext, "deeptext", wide_out)
 
-        assert self.deephead is not None  # assertion to avoid type issues. TO DO: Fix
-        deepside_out = self.deephead(deepside)
+        if self.deepimage is not None:
+            wide_out = self._forward_component(X, self.deepimage, "deepimage", wide_out)
 
         if self.is_tabnet:
-            res: Union[Tensor, Tuple[Tensor, Tensor]] = (
-                wide_out.add_(deepside_out),
-                M_loss,
-            )
+            res: Union[Tensor, Tuple[Tensor, Tensor]] = (wide_out, M_loss)
         else:
-            res = wide_out.add_(deepside_out)
+            res = wide_out
 
         return res
 
-    def _forward_deep(
+    def _forward_deephead(
         self, X: Dict[str, Union[Tensor, List[Tensor]]], wide_out: Tensor
     ) -> Union[Tensor, Tuple[Tensor, Tensor]]:
+        deepside = torch.FloatTensor().to(self.wd_device)
+
         if self.deeptabular is not None:
             if self.is_tabnet:
-                tab_out, M_loss = self.deeptabular(X["deeptabular"])
-                wide_out.add_(tab_out)
+                deepside, M_loss = self.deeptabular(X["deeptabular"])
             else:
-                wide_out.add_(self.deeptabular(X["deeptabular"]))
-        if self.deeptext is not None:
-            if isinstance(self.deeptext, nn.ModuleList):
-                text_out = torch.add(  # type: ignore[call-overload]
-                    *[dt(X["deeptext"][i]) for i, dt in enumerate(self.deeptext)]
+                deepside = self._forward_component_with_head(
+                    X, self.deeptabular, "deeptabular", deepside
                 )
-                wide_out.add_(text_out)
-            else:
-                wide_out.add_(self.deeptext(X["deeptext"]))
+
+        if self.deeptext is not None:
+            deepside = self._forward_component_with_head(
+                X, self.deeptext, "deeptext", deepside
+            )
+
         if self.deepimage is not None:
-            if isinstance(self.deepimage, nn.ModuleList):
-                image_out = torch.add(  # type: ignore[call-overload]
-                    *[di(X["deepimage"][i]) for i, di in enumerate(self.deepimage)]
-                )
-                wide_out.add_(image_out)
-            else:
-                wide_out.add_(self.deepimage(X["deepimage"]))
+            deepside = self._forward_component_with_head(
+                X, self.deepimage, "deepimage", deepside
+            )
+
+        # assertion to avoid type issues
+        assert self.deephead is not None
+        deepside_out = self.deephead(deepside)
 
         if self.is_tabnet:
-            res: Union[Tensor, Tuple[Tensor, Tensor]] = (wide_out, M_loss)
+            res: Union[Tensor, Tuple[Tensor, Tensor]] = (
+                wide_out.add_(deepside_out),
+                M_loss,
+            )
         else:
-            res = wide_out
+            res = wide_out.add_(deepside_out)
 
         return res
 
-    def _forward_deep_with_fds(
+    def _forward_component(
         self,
-        X: Dict[str, Tensor],
-        y: Optional[Tensor] = None,
-        epoch: Optional[int] = None,
-    ) -> Union[Tensor, Tuple[Tensor, Tensor]]:
-        assert self.deeptabular is not None, (
-            "Feature Distribution Smoothing (FDS) is supported when using only a deeptabular component"
-            " and for regression problems."
-        )
-        res = self.fds_layer(self.deeptabular(X["deeptabular"]), y, epoch)
-        if self.enforce_positive:
-            if isinstance(res, Tuple):  # type: ignore[arg-type]
-                out: Union[Tensor, Tuple[Tensor, Tensor]] = (
-                    res[0],
-                    self.enf_pos(res[1]),
-                )
-            else:
-                out = self.enf_pos(res)
+        X: Dict[str, Union[Tensor, List[Tensor]]],
+        component: Union[nn.ModuleList, WDModel],
+        component_type: Literal["deeptabular", "deeptext", "deepimage"],
+        wide_out: Tensor,
+    ) -> Tensor:
+        if isinstance(component, nn.ModuleList):
+            component_out = torch.add(  # type: ignore[call-overload]
+                *[cp(X[component_type][i]) for i, cp in enumerate(component)]
+            )
         else:
-            out = res
-        return out
+            component_out = component(X[component_type])
+
+        return wide_out.add_(component_out)
+
+    def _forward_component_with_head(
+        self,
+        X: Dict[str, Union[Tensor, List[Tensor]]],
+        component: Union[nn.ModuleList, WDModel],
+        component_type: Literal["deeptabular", "deeptext", "deepimage"],
+        deepside: Tensor,
+    ) -> Tensor:
+        if isinstance(component, nn.ModuleList):
+            component_out = torch.cat(  # type: ignore[call-overload]
+                [cp(X[component_type][i]) for i, cp in enumerate(component)], axis=1
+            )
+        else:
+            component_out = component(X[component_type])
+
+        return torch.cat([deepside, component_out], axis=1)  # type: ignore[call-overload]
+
+    def _set_model_component(
+        self,
+        component: Union[BaseWDModelComponent, List[BaseWDModelComponent]],
+        is_deeptabular: bool = False,
+    ) -> Union[nn.ModuleList, WDModel]:
+        if isinstance(component, list):
+            component_: Optional[Union[nn.ModuleList, WDModel]] = nn.ModuleList()
+            for cp in component:
+                if self.with_deephead or cp.output_dim == 1:
+                    component_.append(cp)
+                else:
+                    component_.append(
+                        nn.Sequential(cp, nn.Linear(cp.output_dim, self.pred_dim))
+                    )
+        elif self.with_deephead or component.output_dim == 1:
+            component_ = component
+        elif is_deeptabular and self.is_tabnet:
+            component_ = nn.Sequential(
+                component, TabNetPredLayer(component.output_dim, self.pred_dim)
+            )
+        else:
+            component_ = nn.Sequential(
+                component, nn.Linear(component.output_dim, self.pred_dim)
+            )
+
+        return component_
 
     @staticmethod  # noqa: C901
     def _check_inputs(  # noqa: C901
@@ -486,7 +443,6 @@ def _check_inputs(  # noqa: C901
         deephead,
         head_hidden_dims,
         pred_dim,
-        with_fds,
     ):
         if wide is not None:
             assert wide.wide_linear.weight.size(1) == pred_dim, (
@@ -495,13 +451,39 @@ def _check_inputs(  # noqa: C901
                     wide.wide_linear.weight.size(1), pred_dim
                 )
             )
-        if deeptabular is not None and not hasattr(deeptabular, "output_dim"):
-            raise AttributeError(
-                "deeptabular model must have an 'output_dim' attribute or property. "
-                "See pytorch-widedeep.models.deep_text.DeepText"
+
+        if deeptabular is not None:
+            err_msg = (
+                "deeptabular model must have an 'output_dim' attribute or property."
             )
+            if isinstance(deeptabular, list):
+                all_have_output_dim = all(
+                    hasattr(dt, "output_dim") for dt in deeptabular
+                )
+                if not all_have_output_dim:
+                    raise AttributeError(err_msg)
+            else:
+                if not hasattr(deeptabular, "output_dim"):
+                    raise AttributeError(err_msg)
+                # the following assertion is thought for those cases where we
+                # use fusion with 'dot product' so that the output_dim will
+                # be 1 and the pred_dim is not 1
+                if deeptabular.output_dim == 1:
+                    assert pred_dim == 1, "If 'output_dim' is 1, 'pred_dim' must be 1"
+
         if deeptabular is not None:
-            is_tabnet = deeptabular.__class__.__name__ == "TabNet"
+            is_tabnet = False
+            if isinstance(deeptabular, list):
+                is_any_tabnet = any(
+                    dt.__class__.__name__ == "TabNet" for dt in deeptabular
+                )
+                if is_any_tabnet:
+                    raise ValueError(
+                        "Currently TabNet is not supported as a component of a multiple "
+                        "tabular component model."
+                    )
+            else:
+                is_tabnet = deeptabular.__class__.__name__ == "TabNet"
             has_wide_text_or_image = (
                 wide is not None or deeptext is not None or deepimage is not None
             )
@@ -517,11 +499,9 @@ def _check_inputs(  # noqa: C901
                     " components. Therefore, such importances will partially lose their 'meaning'.",
                     UserWarning,
                 )
+
         if deeptext is not None:
-            err_msg = (
-                "deeptext model must have an 'output_dim' attribute or property. "
-                "See pytorch-widedeep.models.deep_text.DeepText"
-            )
+            err_msg = "deeptext model must have an 'output_dim' attribute or property."
             if isinstance(deeptext, list):
                 all_have_output_dim = all(hasattr(dt, "output_dim") for dt in deeptext)
                 if not all_have_output_dim:
@@ -529,11 +509,11 @@ def _check_inputs(  # noqa: C901
             else:
                 if not hasattr(deeptext, "output_dim"):
                     raise AttributeError(err_msg)
+                if deeptext.output_dim == 1:
+                    assert pred_dim == 1, "If 'output_dim' is 1, 'pred_dim' must be 1"
+
         if deepimage is not None:
-            err_msg = (
-                "deepimage model must have an 'output_dim' attribute or property. "
-                "See pytorch-widedeep.models.deep_image.DeepImage"
-            )
+            err_msg = "deepimage model must have an 'output_dim' attribute or property."
             if isinstance(deepimage, list):
                 all_have_output_dim = all(hasattr(di, "output_dim") for di in deepimage)
                 if not all_have_output_dim:
@@ -541,6 +521,9 @@ def _check_inputs(  # noqa: C901
             else:
                 if not hasattr(deepimage, "output_dim"):
                     raise AttributeError(err_msg)
+                if deepimage.output_dim == 1:
+                    assert pred_dim == 1, "If 'output_dim' is 1, 'pred_dim' must be 1"
+
         if deephead is not None and head_hidden_dims is not None:
             raise ValueError(
                 "both 'deephead' and 'head_hidden_dims' are not None. Use one of the other, but not both"
@@ -554,6 +537,7 @@ def _check_inputs(  # noqa: C901
             raise ValueError(
                 "if 'head_hidden_dims' is not None, at least one deep component must be used"
             )
+
         if deephead is not None:
             if not hasattr(deephead, "output_dim"):
                 raise AttributeError(
@@ -563,27 +547,26 @@ def _check_inputs(  # noqa: C901
             deephead_inp_feat = next(deephead.parameters()).size(1)
             output_dim = 0
             if deeptabular is not None:
-                output_dim += deeptabular.output_dim
+                if isinstance(deeptabular, list):
+                    for dt in deeptabular:
+                        output_dim += dt.output_dim
+                else:
+                    output_dim += deeptabular.output_dim
             if deeptext is not None:
-                output_dim += deeptext.output_dim
+                if isinstance(deeptext, list):
+                    for dt in deeptext:
+                        output_dim += dt.output_dim
+                else:
+                    output_dim += deeptext.output_dim
             if deepimage is not None:
-                output_dim += deepimage.output_dim
+                if isinstance(deepimage, list):
+                    for di in deepimage:
+                        output_dim += di.output_dim
+                else:
+                    output_dim += deepimage.output_dim
             if deephead_inp_feat != output_dim:
                 warnings.warn(
                     "A custom 'deephead' is used and it seems that the input features "
                     "do not match the output of the deep components",
                     UserWarning,
                 )
-        if with_fds and (
-            (
-                wide is not None
-                or deeptext is not None
-                or deepimage is not None
-                or deephead is not None
-            )
-            or pred_dim != 1
-        ):
-            raise ValueError(
-                "Feature Distribution Smoothing (FDS) is supported when using only a deeptabular component"
-                " and for regression problems."
-            )
diff --git a/pytorch_widedeep/training/_base_trainer.py b/pytorch_widedeep/training/_base_trainer.py
index 971031e4..16b9f520 100644
--- a/pytorch_widedeep/training/_base_trainer.py
+++ b/pytorch_widedeep/training/_base_trainer.py
@@ -210,18 +210,6 @@ def _set_optimizer(
             elif isinstance(optimizers, Dict):
                 opt_names = list(optimizers.keys())
                 mod_names = [n for n, c in self.model.named_children()]
-                # if with_fds - the prediction layer is part of the model and
-                # should be optimized with the rest of deeptabular
-                # component/model
-                if self.model.with_fds:
-                    if "enf_pos" in mod_names:
-                        mod_names.remove("enf_pos")
-                    mod_names.remove("fds_layer")
-                    # The Tabular optimizer is always going to be just one
-                    assert isinstance(optimizers["deeptabular"], Optimizer)
-                    optimizers["deeptabular"].add_param_group(
-                        {"params": self.model.fds_layer.pred_layer.parameters()}
-                    )
                 for mn in mod_names:
                     assert mn in opt_names, "No optimizer found for {}".format(mn)
                 optimizer = MultipleOptimizer(optimizers)
@@ -384,11 +372,6 @@ def _check_inputs(
         lr_schedulers,
         custom_loss_function,
     ):
-        if model.with_fds and _ObjectiveToMethod.get(objective) != "regression":
-            raise ValueError(
-                "Feature Distribution Smooting can be used only for regression"
-            )
-
         if objective == "multitarget":
             assert custom_loss_function is not None, (
                 "When 'objective' is 'multitarget', 'custom_loss_function' must be "
diff --git a/pytorch_widedeep/training/_feature_importance.py b/pytorch_widedeep/training/_feature_importance.py
index 43b6d6f2..79a37f19 100644
--- a/pytorch_widedeep/training/_feature_importance.py
+++ b/pytorch_widedeep/training/_feature_importance.py
@@ -134,7 +134,7 @@ def _sample_data(self, loader: DataLoader) -> Tensor:
         n_iterations = self.n_samples // loader.batch_size
 
         batches = []
-        for i, (data, _, _) in enumerate(loader):
+        for i, (data, _) in enumerate(loader):
             if i < n_iterations:
                 batches.append(data["deeptabular"].to(self.device))
             else:
diff --git a/pytorch_widedeep/training/_trainer_utils.py b/pytorch_widedeep/training/_trainer_utils.py
index c10460df..dd85c1f2 100644
--- a/pytorch_widedeep/training/_trainer_utils.py
+++ b/pytorch_widedeep/training/_trainer_utils.py
@@ -22,8 +22,10 @@
     FocalR_RMSELoss,
 )
 from pytorch_widedeep.wdtypes import (
+    Any,
     Dict,
     List,
+    Tuple,
     Union,
     Compose,
     Literal,
@@ -45,7 +47,7 @@ def tabular_train_val_split(
     X_val: Optional[np.ndarray] = None,
     y_val: Optional[np.ndarray] = None,
     val_split: Optional[float] = None,
-):
+) -> Tuple[TensorDataset, Optional[TensorDataset]]:
     r"""
     Function to create the train/val split for the BayesianTrainer where only
     tabular data is used
@@ -84,7 +86,7 @@ def tabular_train_val_split(
             torch.from_numpy(y_val),
         )
     elif val_split is not None:
-        y_tr, y_val, idx_tr, idx_val = train_test_split(
+        y_tr, y_val, idx_tr, idx_val = train_test_split(  # type: ignore
             y,
             np.arange(len(y)),
             test_size=val_split,
@@ -115,7 +117,7 @@ def wd_train_val_split(  # noqa: C901
     seed: int,
     method: Literal["regression", "binary", "multiclass", "qregression"],
     X_wide: Optional[np.ndarray] = None,
-    X_tab: Optional[np.ndarray] = None,
+    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
     X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
     X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
     X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,
@@ -123,7 +125,6 @@ def wd_train_val_split(  # noqa: C901
     val_split: Optional[float] = None,
     target: Optional[np.ndarray] = None,
     transforms: Optional[Union[Transforms, Compose]] = None,
-    **lds_args,
 ):
     r"""
     Function to create the train/val split for a wide and deep model
@@ -139,13 +140,13 @@ def wd_train_val_split(  # noqa: C901
         random seed to be used during train/val split
     method: str
         'regression',  'binary' or 'multiclass'
-    X_wide: np.ndarray, Optional, default = None
+    X_wide: np.ndaaray, Optional, default = None
         wide dataset
-    X_tab: np.ndarray, Optional, default = None
+    X_tab: np.ndarray or List[np.ndarray], Optional, default = None
         tabular dataset (categorical and continuous features)
-    X_img: np.ndarray, Optional, default = None
+    X_img: np.ndarray or List[np.ndarray], Optional, default = None
         image dataset
-    X_text: np.ndarray, Optional, default = None
+    X_text: np.ndarray or List[np.ndarray], Optional, default = None
         text dataset
     X_val: Dict, Optional, default = None
         Dict with the validation set, where the keys are the component names
@@ -167,14 +168,15 @@ def wd_train_val_split(  # noqa: C901
             "if the validation set is passed as a dictionary, the training set must also be a dictionary,"
             " that includes the target"
         )
-        train_set = WideDeepDataset(**X_train, transforms=transforms, **lds_args)  # type: ignore
-        eval_set = WideDeepDataset(**X_val, transforms=transforms, is_training=False)  # type: ignore
+        train_set = WideDeepDataset(**X_train, transforms=transforms)  # type: ignore
+        eval_set = WideDeepDataset(**X_val, transforms=transforms)  # type: ignore
     elif val_split is not None:
         if not X_train:
             assert (
                 target is not None
             ), "if the validation split is specified, the target must also be specified"
             X_train = _build_train_dict(X_wide, X_tab, X_text, X_img, target)
+
         y_tr, y_val, idx_tr, idx_val = train_test_split(
             X_train["target"],
             np.arange(len(X_train["target"])),
@@ -188,61 +190,63 @@ def wd_train_val_split(  # noqa: C901
         )
         X_tr, X_val = {"target": y_tr}, {"target": y_val}
         if "X_wide" in X_train.keys():
-            X_tr["X_wide"], X_val["X_wide"] = (
-                X_train["X_wide"][idx_tr],
-                X_train["X_wide"][idx_val],
+            # the wide component will never be a list, but can still be passed
+            # to '_wd_train_val_split_component'
+            X_tr, X_val = _wd_train_val_split_component(
+                X_train, X_tr, X_val, idx_tr, idx_val, "X_wide"
             )
         if "X_tab" in X_train.keys():
-            X_tr["X_tab"], X_val["X_tab"] = (
-                X_train["X_tab"][idx_tr],
-                X_train["X_tab"][idx_val],
+            X_tr, X_val = _wd_train_val_split_component(
+                X_train, X_tr, X_val, idx_tr, idx_val, "X_tab"
             )
         if "X_text" in X_train.keys():
-            if isinstance(X_train["X_text"], list):
-                X_tr["X_text"], X_val["X_text"] = (
-                    [
-                        X_train["X_text"][i][idx_tr]
-                        for i in range(len(X_train["X_text"]))
-                    ],
-                    [
-                        X_train["X_text"][i][idx_val]
-                        for i in range(len(X_train["X_text"]))
-                    ],
-                )
-            else:
-                X_tr["X_text"], X_val["X_text"] = (
-                    X_train["X_text"][idx_tr],
-                    X_train["X_text"][idx_val],
-                )
+            X_tr, X_val = _wd_train_val_split_component(
+                X_train, X_tr, X_val, idx_tr, idx_val, "X_text"
+            )
         if "X_img" in X_train.keys():
-            if isinstance(X_train["X_img"], list):
-                X_tr["X_img"], X_val["X_img"] = (
-                    [X_train["X_img"][i][idx_tr] for i in range(len(X_train["X_img"]))],
-                    [
-                        X_train["X_img"][i][idx_val]
-                        for i in range(len(X_train["X_img"]))
-                    ],
-                )
-            else:
-                X_tr["X_img"], X_val["X_img"] = (
-                    X_train["X_img"][idx_tr],
-                    X_train["X_img"][idx_val],
-                )
-        train_set = WideDeepDataset(**X_tr, transforms=transforms, **lds_args)  # type: ignore
-        eval_set = WideDeepDataset(**X_val, transforms=transforms, is_training=False)  # type: ignore
+            X_tr, X_val = _wd_train_val_split_component(
+                X_train, X_tr, X_val, idx_tr, idx_val, "X_img"
+            )
+        train_set = WideDeepDataset(**X_tr, transforms=transforms)  # type: ignore
+        eval_set = WideDeepDataset(**X_val, transforms=transforms)  # type: ignore
     else:
         if not X_train:
             assert target is not None
             X_train = _build_train_dict(X_wide, X_tab, X_text, X_img, target)
-        train_set = WideDeepDataset(**X_train, transforms=transforms, **lds_args)  # type: ignore
+        train_set = WideDeepDataset(**X_train, transforms=transforms)  # type: ignore
         eval_set = None
 
     return train_set, eval_set
 
 
+def _wd_train_val_split_component(
+    X: Dict[str, Union[np.ndarray, List[np.ndarray]]],
+    X_tr: Dict[str, Union[np.ndarray, List[np.ndarray]]],
+    X_val: Dict[str, Union[np.ndarray, List[np.ndarray]]],
+    idx_tr: Any,  # is a numpy array but sklearn's train_test_split returns a non-sensical type
+    idx_val: Any,
+    component_type: Literal["X_wide", "X_tab", "X_text", "X_img"],
+) -> Tuple[
+    Dict[str, Union[np.ndarray, List[np.ndarray]]],
+    Dict[str, Union[np.ndarray, List[np.ndarray]]],
+]:
+    if isinstance(X[component_type], list):
+        X_tr[component_type], X_val[component_type] = (
+            [X[component_type][i][idx_tr] for i in range(len(X[component_type]))],
+            [X[component_type][i][idx_val] for i in range(len(X[component_type]))],
+        )
+    else:
+        X_tr[component_type], X_val[component_type] = (
+            X[component_type][idx_tr],
+            X[component_type][idx_val],
+        )
+
+    return X_tr, X_val
+
+
 def _build_train_dict(
     X_wide: Optional[np.ndarray],
-    X_tab: Optional[np.ndarray],
+    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]],
     X_text: Optional[Union[np.ndarray, List[np.ndarray]]],
     X_img: Optional[Union[np.ndarray, List[np.ndarray]]],
     target: np.ndarray,
diff --git a/pytorch_widedeep/training/_wd_dataset.py b/pytorch_widedeep/training/_wd_dataset.py
index f08c0275..f86ac84e 100644
--- a/pytorch_widedeep/training/_wd_dataset.py
+++ b/pytorch_widedeep/training/_wd_dataset.py
@@ -2,16 +2,11 @@
 
 import numpy as np
 import torch
-from scipy.ndimage import convolve1d
 from sklearn.utils import Bunch
 from torch.utils.data import Dataset
 from torchvision.transforms import Compose
 
-from pytorch_widedeep.wdtypes import Literal, Optional, Transforms
-from pytorch_widedeep.utils.deeptabular_utils import (
-    find_bin,
-    get_kernel_window,
-)
+from pytorch_widedeep.wdtypes import Optional, Transforms
 
 
 class WideDeepDataset(Dataset):
@@ -22,7 +17,7 @@ class WideDeepDataset(Dataset):
     ----------
     X_wide: np.ndarray
         wide input
-    X_tab: np.ndarray
+    X_tab: np.ndarray or List[np.ndarray]
         deeptabular input
     X_text: np.ndarray or List[np.ndarray]
         deeptext input
@@ -32,41 +27,16 @@ class WideDeepDataset(Dataset):
         target array
     transforms: Optional[Transforms | Compose]
         torchvision Compose object. See models/_multiple_transforms.py
-    with_lds: bool
-        Boolean indicating if Label Distribution Smoothing will be applied to
-        the dataset
-
-    Other Parameters
-    ----------------
-    **kwargs
-        Label Distribution Smoothing parameters:
-            lds_kernel: Literal['gaussian', 'triang', 'laplace'] = 'gaussian'
-                choice of kernel for Label Distribution Smoothing
-            lds_ks: int = 5
-                LDS kernel window size
-            lds_sigma: float = 2
-                standard deviation of ['gaussian','laplace'] kernel for LDS
-            lds_granularity: int = 100,
-                number of bins in the histogram used in LDS to count occurence of sample values
-            lds_reweight: bool
-                option to reweight bin frequency counts in LDS
-            lds_y_max: Optional[float] = None
-                option to restrict LDS bins by upper label limit
-            lds_y_min: Optional[float] = None
-                option to restrict LDS bins by lower label limit
     """
 
     def __init__(
         self,
         X_wide: Optional[np.ndarray] = None,
-        X_tab: Optional[np.ndarray] = None,
+        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         target: Optional[np.ndarray] = None,
         transforms: Optional[Union[Transforms, Compose]] = None,
-        is_training: bool = True,
-        with_lds: bool = False,
-        **kwargs,
     ):
         super(WideDeepDataset, self).__init__()
         self.X_wide = X_wide
@@ -85,23 +55,15 @@ def __init__(
             self.transforms_names = []
         self.Y = target
 
-        # LDS
-        self.is_training = is_training
-        self.with_lds = with_lds
-        if self.Y is not None and self.is_training:
-            # this is a hack to avoid having to run separate for loops during
-            # training whether we use lds or not
-            if self.with_lds:
-                self.weights = self._compute_lds_weights(**kwargs)
-            else:
-                self.weights = np.zeros_like(self.Y, dtype="float32")
-
     def __getitem__(self, idx: int):  # noqa: C901
         x = Bunch()
         if self.X_wide is not None:
             x.wide = self.X_wide[idx]
         if self.X_tab is not None:
-            x.deeptabular = self.X_tab[idx]
+            if isinstance(self.X_tab, list):
+                x.deeptabular = [self.X_tab[i][idx] for i in range(len(self.X_tab))]
+            else:
+                x.deeptabular = self.X_tab[idx]
         if self.X_text is not None:
             if isinstance(self.X_text, list):
                 x.deeptext = [self.X_text[i][idx] for i in range(len(self.X_text))]
@@ -119,65 +81,7 @@ def __getitem__(self, idx: int):  # noqa: C901
             return x
         else:
             y = self.Y[idx]
-            if self.is_training:
-                return x, y, self.weights[idx]
-            else:
-                return x, y
-
-    def _compute_lds_weights(
-        self,
-        lds_granularity: int = 100,
-        lds_reweight: bool = False,
-        lds_kernel: Literal["gaussian", "triang", "laplace"] = "gaussian",
-        lds_ks: int = 5,
-        lds_sigma: float = 2,
-        lds_y_min: Optional[float] = None,
-        lds_y_max: Optional[float] = None,
-    ) -> np.ndarray:
-        """Assign weight to each sample by following procedure:
-        1.      creating histogram from label values with nuber of bins = granularity
-        2[opt]. reweighting label frequencies by sqrt
-        3[opt]. smoothing label frequencies by convolution of kernel function window with frequencies list
-        4.      inverting values by n_samples / (n_classes * np.bincount(y)), see:
-        https://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_sample_weight.html
-        5.      assigning weight to each sample from closest bin value
-        """
-
-        assert self.Y is not None, "No target array provided"
-        y_max = max(self.Y) if lds_y_max is None else lds_y_max
-        y_min = min(self.Y) if lds_y_min is None else lds_y_min
-        bin_edges = np.linspace(y_min, y_max, num=lds_granularity, endpoint=True)
-        value_dict = dict(zip(bin_edges[:-1], np.histogram(self.Y, bin_edges)[0]))
-
-        if lds_reweight:
-            value_dict = dict(
-                zip(value_dict.keys(), np.sqrt(list(value_dict.values())))
-            )
-
-        if lds_kernel is not None:
-            lds_kernel_window = get_kernel_window(lds_kernel, lds_ks, lds_sigma)
-            smoothed_values = convolve1d(
-                list(value_dict.values()), weights=lds_kernel_window, mode="constant"
-            )
-
-            # to avoid 0 division
-            numerator = sum(smoothed_values)
-            denominator = (len(smoothed_values) * smoothed_values).astype("float")
-            denominator[denominator == 0] = np.inf
-
-            weigths = numerator / denominator
-            # weigths = sum(smoothed_values) / (len(smoothed_values) * smoothed_values)
-        else:
-            values = list(value_dict.values())
-            weigths = sum(values) / (len(values) * values)  # type: ignore[operator]
-        value_dict = dict(zip(value_dict.keys(), weigths))
-
-        left_bin_edges = find_bin(bin_edges, self.Y)
-        weights = np.array(
-            [value_dict[edge] for edge in left_bin_edges], dtype="float32"
-        )
-
-        return weights
+            return x, y
 
     def _prepare_images(self, imgs: np.ndarray, idx: int):
         # if an image dataset is used, make sure is in the right format to
@@ -211,7 +115,10 @@ def __len__(self):
         if self.X_wide is not None:
             return len(self.X_wide)
         if self.X_tab is not None:
-            return len(self.X_tab)
+            if isinstance(self.X_tab, list):
+                return len(self.X_tab[0])
+            else:
+                return len(self.X_tab)
         if self.X_text is not None:
             if isinstance(self.X_text, list):
                 return len(self.X_text[0])
diff --git a/pytorch_widedeep/training/trainer.py b/pytorch_widedeep/training/trainer.py
index 13617bc4..b0a2672a 100644
--- a/pytorch_widedeep/training/trainer.py
+++ b/pytorch_widedeep/training/trainer.py
@@ -1,6 +1,4 @@
 import json
-import warnings
-from inspect import signature
 from pathlib import Path
 
 import numpy as np
@@ -16,7 +14,6 @@
 from pytorch_widedeep.wdtypes import (
     Dict,
     List,
-    Tuple,
     Union,
     Tensor,
     Literal,
@@ -27,7 +24,6 @@
     LRScheduler,
 )
 from pytorch_widedeep.callbacks import Callback
-from pytorch_widedeep.dataloaders import DataLoaderDefault
 from pytorch_widedeep.initializers import Initializer
 from pytorch_widedeep.training._finetune import FineTune
 from pytorch_widedeep.utils.general_utils import alias
@@ -274,7 +270,7 @@ def __init__(
     def fit(  # noqa: C901
         self,
         X_wide: Optional[np.ndarray] = None,
-        X_tab: Optional[np.ndarray] = None,
+        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,
@@ -287,7 +283,6 @@ def fit(  # noqa: C901
         custom_dataloader: Optional[DataLoader] = None,
         feature_importance_sample_size: Optional[int] = None,
         finetune: bool = False,
-        with_lds: bool = False,
         **kwargs,
     ):
         r"""Fit method.
@@ -303,7 +298,9 @@ def fit(  # noqa: C901
             See `pytorch_widedeep.preprocessing.WidePreprocessor`
         X_tab: np.ndarray, Optional. default=None
             Input for the `deeptabular` model component.
-            See `pytorch_widedeep.preprocessing.TabPreprocessor`
+            See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple
+            tabular models are used for different columns, this should be a
+            list of numpy arrays
         X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None
             Input for the `deeptext` model component.
             See `pytorch_widedeep.preprocessing.TextPreprocessor`.
@@ -370,12 +367,6 @@ def fit(  # noqa: C901
             For details on how these routines work, please see the Examples
             section in this documentation and the Examples folder in the repo. <br/>
             Param Alias: `warmup`
-        with_lds: bool, default=False
-            Boolean indicating if Label Distribution Smoothing will be used. <br/>
-            information_source: **NOTE**: We consider this feature absolutely
-            experimental and we recommend the user to not use it unless the
-            corresponding [publication](https://arxiv.org/abs/2102.09554) is
-            well understood
 
         Other Parameters
         ----------------
@@ -388,26 +379,6 @@ def fit(  # noqa: C901
                 [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)
                 for details.
 
-            - **Label Distribution Smoothing related parameters**:<br/>
-
-                - lds_kernel (`Literal['gaussian', 'triang', 'laplace']`):
-                    choice of kernel for Label Distribution Smoothing
-                - lds_ks (`int`):
-                    LDS kernel window size
-                - lds_sigma (`float`):
-                    standard deviation of ['gaussian','laplace'] kernel for LDS
-                - lds_granularity (`int`):
-                    number of bins in histogram used in LDS to count occurence of sample values
-                - lds_reweight (`bool`):
-                    option to reweight bin frequency counts in LDS
-                - lds_y_max (`Optional[float]`):
-                    option to restrict LDS bins by upper label limit
-                - lds_y_min (`Optional[float]`):
-                    option to restrict LDS bins by lower label limit
-
-                See `pytorch_widedeep.trainer._wd_dataset` for more details on
-                the implications of these parameters
-
             - **Finetune related parameters**:<br/>
                 see the source code at `pytorch_widedeep._finetune`. Namely, these are:
 
@@ -447,9 +418,7 @@ def fit(  # noqa: C901
         folder in the repo
         """
 
-        lds_args, dataloader_args, finetune_args = self._extract_kwargs(kwargs)
-        lds_args["with_lds"] = with_lds
-        self.with_lds = with_lds
+        dataloader_args, finetune_args = self._extract_kwargs(kwargs)
 
         self.batch_size = batch_size
 
@@ -465,7 +434,6 @@ def fit(  # noqa: C901
             val_split,
             target,
             self.transforms,
-            **lds_args,
         )
         if custom_dataloader is not None:
             # make sure is callable (and HAS to be an subclass of DataLoader)
@@ -477,7 +445,7 @@ def fit(  # noqa: C901
                 **dataloader_args,
             )
         else:
-            train_loader = DataLoaderDefault(
+            train_loader = DataLoader(
                 dataset=train_set,
                 batch_size=batch_size,
                 num_workers=self.num_workers,
@@ -513,11 +481,9 @@ def fit(  # noqa: C901
 
             self.train_running_loss = 0.0
             with trange(train_steps, disable=self.verbose != 1) as t:
-                for batch_idx, (data, targett, lds_weightt) in zip(t, train_loader):
+                for batch_idx, (data, targett) in zip(t, train_loader):
                     t.set_description("epoch %i" % (epoch + 1))
-                    train_score, train_loss = self._train_step(
-                        data, targett, batch_idx, epoch, lds_weightt
-                    )
+                    train_score, train_loss = self._train_step(data, targett, batch_idx)
                     print_loss_and_metric(t, train_loss, train_score)
                     self.callback_container.on_batch_end(batch=batch_idx)
             epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, "train")
@@ -553,9 +519,6 @@ def fit(  # noqa: C901
                 # self.callback_container.on_train_end(epoch_logs)
                 break
 
-            if self.model.with_fds:
-                self._update_fds_stats(train_loader, epoch)
-
         self.callback_container.on_train_end(epoch_logs)
 
         if feature_importance_sample_size is not None:
@@ -568,7 +531,7 @@ def fit(  # noqa: C901
     def predict(  # type: ignore[override, return]
         self,
         X_wide: Optional[np.ndarray] = None,
-        X_tab: Optional[np.ndarray] = None,
+        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,
@@ -586,13 +549,13 @@ def predict(  # type: ignore[override, return]
         X_wide: np.ndarray, Optional. default=None
             Input for the `wide` model component.
             See `pytorch_widedeep.preprocessing.WidePreprocessor`
-        X_tab: np.ndarray, Optional. default=None
+        X_tab: np.ndarray or List[np.ndarray], Optional. default=None
             Input for the `deeptabular` model component.
             See `pytorch_widedeep.preprocessing.TabPreprocessor`
-        X_text: np.ndarray, Optional. default=None
+        X_text: np.ndarray or List[np.ndarray], Optional. default=None
             Input for the `deeptext` model component.
             See `pytorch_widedeep.preprocessing.TextPreprocessor`
-        X_img: np.ndarray, Optional. default=None
+        X_img: np.ndarray or List[np.ndarray], Optional. default=None
             Input for the `deepimage` model component.
             See `pytorch_widedeep.preprocessing.ImagePreprocessor`
         X_test: Dict, Optional. default=None
@@ -624,7 +587,7 @@ def predict(  # type: ignore[override, return]
     def predict_uncertainty(  # type: ignore[return]
         self,
         X_wide: Optional[np.ndarray] = None,
-        X_tab: Optional[np.ndarray] = None,
+        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,
@@ -645,13 +608,13 @@ def predict_uncertainty(  # type: ignore[return]
         X_wide: np.ndarray, Optional. default=None
             Input for the `wide` model component.
             See `pytorch_widedeep.preprocessing.WidePreprocessor`
-        X_tab: np.ndarray, Optional. default=None
+        X_tab: np.ndarray or List[np.ndarray], Optional. default=None
             Input for the `deeptabular` model component.
             See `pytorch_widedeep.preprocessing.TabPreprocessor`
-        X_text: np.ndarray, Optional. default=None
+        X_text: np.ndarray or List[np.ndarray], Optional. default=None
             Input for the `deeptext` model component.
             See `pytorch_widedeep.preprocessing.TextPreprocessor`
-        X_img: np.ndarray, Optional. default=None
+        X_img: np.ndarray or List[np.ndarray], Optional. default=None
             Input for the `deepimage` model component.
             See `pytorch_widedeep.preprocessing.ImagePreprocessor`
         X_test: Dict, Optional. default=None
@@ -721,7 +684,7 @@ def predict_uncertainty(  # type: ignore[return]
     def predict_proba(  # type: ignore[override, return]  # noqa: C901
         self,
         X_wide: Optional[np.ndarray] = None,
-        X_tab: Optional[np.ndarray] = None,
+        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,
@@ -739,13 +702,13 @@ def predict_proba(  # type: ignore[override, return]  # noqa: C901
         X_wide: np.ndarray, Optional. default=None
             Input for the `wide` model component.
             See `pytorch_widedeep.preprocessing.WidePreprocessor`
-        X_tab: np.ndarray, Optional. default=None
+        X_tab: np.ndarray or List[np.ndarray], Optional. default=None
             Input for the `deeptabular` model component.
             See `pytorch_widedeep.preprocessing.TabPreprocessor`
-        X_text: np.ndarray, Optional. default=None
+        X_text: np.ndarray or List[np.ndarray], Optional. default=None
             Input for the `deeptext` model component.
             See `pytorch_widedeep.preprocessing.TextPreprocessor`
-        X_img: np.ndarray, Optional. default=None
+        X_img: np.ndarray or List[np.ndarray], Optional. default=None
             Input for the `deepimage` model component.
             See `pytorch_widedeep.preprocessing.ImagePreprocessor`
         X_test: Dict, Optional. default=None
@@ -929,25 +892,7 @@ def _train_step(
         data: Dict[str, Union[Tensor, List[Tensor]]],
         target: Tensor,
         batch_idx: int,
-        epoch: int,
-        lds_weightt: Tensor,
     ):
-        lds_weight = (
-            None
-            if torch.all(lds_weightt == 0)
-            else lds_weightt.view(-1, 1).to(self.device)
-        )
-        if (
-            self.with_lds
-            and lds_weight is not None
-            and "lds_weight" not in signature(self.loss_fn.forward).parameters
-        ):
-            warnings.warn(
-                """LDS weights are not None but the loss function used does not"
-                " support LDS weightening. For loss functions that support LDS"
-                " weightening please read the docs""",
-                UserWarning,
-            )
 
         self.model.train()
 
@@ -966,20 +911,13 @@ def _train_step(
 
         self.optimizer.zero_grad()
 
-        if self.model.with_fds:
-            _, y_pred = self.model(X, y, epoch)
-        else:
-            y_pred = self.model(X)
+        y_pred = self.model(X)
 
         if self.model.is_tabnet:
             loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]
             score = self._get_score(y_pred[0], y)
         else:
-            loss = (
-                self.loss_fn(y_pred, y)
-                if not self.with_lds
-                else self.loss_fn(y_pred, y, lds_weight=lds_weight)
-            )
+            loss = self.loss_fn(y_pred, y)
             score = self._get_score(y_pred, y)
 
         loss.backward()
@@ -1006,7 +944,7 @@ def _eval_step(
                     X[k] = v.to(self.device)
             y = (
                 target.view(-1, 1).float()
-                if self.method not in ["multiclass", "qregression"]
+                if self.method not in ["multiclass", "qregression", "multitarget"]
                 else target
             )
             y = y.to(self.device)
@@ -1022,6 +960,7 @@ def _eval_step(
             self.valid_running_loss += loss.item()
             avg_loss = self.valid_running_loss / (batch_idx + 1)
 
+        self.model.train()
         return score, avg_loss
 
     def _get_score(self, y_pred, y):
@@ -1034,47 +973,15 @@ def _get_score(self, y_pred, y):
                 score = self.metric(y_pred, y)
             if self.method == "multiclass":
                 score = self.metric(F.softmax(y_pred, dim=1), y)
+            # TO DO: handle multitarget
             return score
         else:
             return None
 
-    def _fds_step(
-        self,
-        data: Dict[str, Tensor],
-        target: Tensor,
-        epoch: int,
-    ) -> Tuple[Tensor, Tensor]:
-        self.model.train()
-        # FDS is only supported for the deeptabular component, X will never
-        # be Dict[str, List[Tensor]]
-        X = {k: v.to(self.device) for k, v in data.items()}
-        y = target.view(-1, 1).float().to(self.device)
-        smoothed_features, _ = self.model(X, y, epoch)
-        return smoothed_features, y
-
-    def _update_fds_stats(self, train_loader: DataLoader, epoch: int):
-        train_steps = len(train_loader)
-        features_l, y_pred_l = [], []
-        with torch.no_grad():
-            with trange(train_steps, disable=self.verbose != 1) as t:
-                for _, (data, targett, _) in zip(t, train_loader):
-                    t.set_description("FDS update")
-                    deeptab_features, deeptab_preds = self._fds_step(
-                        data,
-                        targett,
-                        epoch,
-                    )
-                    features_l.append(deeptab_features)
-                    y_pred_l.append(deeptab_preds)
-        features = torch.cat(features_l)
-        y_pred = torch.cat(y_pred_l)
-        self.model.fds_layer.update_last_epoch_stats(epoch)
-        self.model.fds_layer.update_running_stats(features, y_pred, epoch)
-
     def _predict(  # type: ignore[override, return]  # noqa: C901
         self,
         X_wide: Optional[np.ndarray] = None,
-        X_tab: Optional[np.ndarray] = None,
+        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,
@@ -1195,15 +1102,6 @@ def _extract_kwargs(kwargs):
             "persistent_workers",
             "oversample_mul",
         ]
-        lds_params = [
-            "lds_kernel",
-            "lds_ks",
-            "lds_sigma",
-            "lds_granularity",
-            "lds_reweight",
-            "lds_y_max",
-            "lds_y_min",
-        ]
         finetune_params = [
             "n_epochs",
             "finetune_epochs",
@@ -1223,13 +1121,11 @@ def _extract_kwargs(kwargs):
             "deepimage_max_lr",
         ]
 
-        lds_args, dataloader_args, finetune_args = {}, {}, {}
+        dataloader_args, finetune_args = {}, {}
         for k, v in kwargs.items():
-            if k in lds_params:
-                lds_args[k] = v
             if k in dataloader_params:
                 dataloader_args[k] = v
             if k in finetune_params:
                 finetune_args[k] = v
 
-        return lds_args, dataloader_args, finetune_args
+        return dataloader_args, finetune_args
diff --git a/pytorch_widedeep/utils/deeptabular_utils.py b/pytorch_widedeep/utils/deeptabular_utils.py
index 1e0af8f0..c33a81d8 100644
--- a/pytorch_widedeep/utils/deeptabular_utils.py
+++ b/pytorch_widedeep/utils/deeptabular_utils.py
@@ -1,27 +1,15 @@
 import warnings
 
 import numpy as np
-import torch
 import pandas as pd
-from scipy.ndimage import gaussian_filter1d
 from sklearn.exceptions import NotFittedError
-from scipy.signal.windows import triang
-
-from pytorch_widedeep.wdtypes import (
-    Dict,
-    List,
-    Union,
-    Tensor,
-    Literal,
-    Optional,
-)
+
+from pytorch_widedeep.wdtypes import Dict, List, Optional
 from pytorch_widedeep.utils.general_utils import alias
 
 warnings.filterwarnings("ignore")
 pd.options.mode.chained_assignment = None
 
-__all__ = ["LabelEncoder", "find_bin", "get_kernel_window"]
-
 
 class LabelEncoder:
     r"""Label Encode categorical values for multiple columns at once
@@ -267,88 +255,3 @@ def __repr__(self) -> str:
             list_of_params.append("shared_embed={shared_embed}")
         all_params = ", ".join(list_of_params)
         return f"LabelEncoder({all_params.format(**self.__dict__)})"
-
-
-def find_bin(
-    bin_edges: Union[np.ndarray, Tensor],
-    values: Union[np.ndarray, Tensor],
-    ret_value: bool = True,
-) -> Union[np.ndarray, Tensor]:
-    """Returns indices that are the results of applying the 'searchsorted' algo
-    to 'bin_edges' and 'values' or the left edge of the bins (i.e. bin_edges[indices])
-
-    Parameters
-    ----------
-    bin_edges: Union[np.ndarray, Tensor]
-        monotonically increasing array of bin edges
-    values: Union[np.ndarray, Tensor]
-        values for which we want corresponding bins
-    ret_value: bool
-        if True, return bin values else indices
-
-    Returns
-    -------
-    left_bin_edges: Union[np.ndarray, Tensor]
-        left bin edges
-    """
-    if type(bin_edges) == np.ndarray and type(values) == np.ndarray:
-        indices: Union[np.ndarray, Tensor] = np.searchsorted(
-            bin_edges, values, side="left"
-        )
-        indices = np.where(
-            (indices == 0) | (indices == len(bin_edges)), indices, indices - 1
-        )
-        indices = np.where(indices != len(bin_edges), indices, indices - 2)
-    elif type(bin_edges) == Tensor and type(values) == Tensor:
-        bin_edges = bin_edges.to(values.device)
-        indices = torch.searchsorted(bin_edges, values, right=False)
-        indices = torch.where(
-            (indices == 0) | (indices == len(bin_edges)), indices, indices - 1
-        )
-        indices = torch.where(indices != len(bin_edges), indices, indices - 2)
-    else:
-        raise TypeError(
-            "Both input arrays must be of teh same type, either np.ndarray of Tensor"
-        )
-    return indices if not ret_value else bin_edges[indices]  # type: ignore[index]
-
-
-def _laplace(x, sigma: Union[int, float] = 2):
-    return np.exp(-abs(x) / sigma) / (2.0 * sigma)
-
-
-def get_kernel_window(
-    kernel: Literal["gaussian", "triang", "laplace"] = "gaussian",
-    ks: int = 5,
-    sigma: Union[int, float] = 2,
-) -> Union[List[float], np.ndarray]:
-    """Procedure to prepare the window of values from symetrical kernel function
-    for smoothing of the distribution in Label and Feature Distribution
-    Smoothing (LDS & FDS).
-
-    Parameters
-    ----------
-    kernel: Literal['gaussian', 'triang', 'laplace'] = 'gaussian'
-        choice of kernel for label distribution smoothing
-    ks: int = 5
-        kernel size, i.e. count of samples in symmetric window
-    sigma: Union[int,float] = 2
-        standard deviation of ['gaussian','laplace'] kernel
-
-    Returns
-    -------
-    kernel_window: list
-        list with values from the chosen kernel function
-    """
-    half_ks = (ks - 1) // 2
-    if kernel == "gaussian":
-        base_kernel = [0.0] * half_ks + [1.0] + [0.0] * half_ks
-        kernel_window = gaussian_filter1d(base_kernel, sigma=sigma)
-    elif kernel == "triang":
-        kernel_window = triang(ks) / sum(triang(ks))
-    elif kernel == "laplace":
-        kernel_window = list(map(_laplace, np.arange(-half_ks, half_ks + 1)))
-    else:
-        raise ValueError("Kernel can be only ['gaussian', 'triang', 'laplace'].")
-
-    return kernel_window
diff --git a/pytorch_widedeep/version.py b/pytorch_widedeep/version.py
index 51bbb3f2..31e744e4 100644
--- a/pytorch_widedeep/version.py
+++ b/pytorch_widedeep/version.py
@@ -1 +1 @@
-__version__ = "1.6.2"
+__version__ = "1.6.3"
diff --git a/tests/test_data_utils/test_du_tabular.py b/tests/test_data_utils/test_du_tabular.py
index 10325bf3..95f847f1 100644
--- a/tests/test_data_utils/test_du_tabular.py
+++ b/tests/test_data_utils/test_du_tabular.py
@@ -1,15 +1,10 @@
 import numpy as np
-import torch
 import pandas as pd
 import pytest
 from sklearn.exceptions import NotFittedError
 
 from pytorch_widedeep.preprocessing import TabPreprocessor
-from pytorch_widedeep.utils.deeptabular_utils import (
-    LabelEncoder,
-    find_bin,
-    get_kernel_window,
-)
+from pytorch_widedeep.utils.deeptabular_utils import LabelEncoder
 from pytorch_widedeep.preprocessing.tab_preprocessor import embed_sz_rule
 
 
@@ -317,37 +312,6 @@ def test_overlapping_cols_valueerror():
         )
 
 
-###############################################################################
-# Test get_kernel_window
-###############################################################################
-
-
-def test_get_kernel_window():
-    assert get_kernel_window().shape[0] == 5
-
-
-###############################################################################
-# Test find_bin
-###############################################################################
-
-
-@pytest.mark.parametrize(
-    "bin_edges, values",
-    [
-        (np.array([1, 2, 3, 4, 5]), np.array([-1, 0.5, 1, 2.5, 5, 6])),
-        (torch.tensor([1, 2, 3, 4, 5]), torch.tensor([-1, 0.5, 1, 2.5, 5, 6])),
-    ],
-)
-def test_find_bin(bin_edges, values):
-    if type(bin_edges) == np.ndarray and type(values) == np.ndarray:
-        assert np.array_equal(find_bin(bin_edges, values), np.array([1, 1, 1, 2, 4, 4]))
-    elif type(bin_edges) == torch.Tensor and type(values) == torch.Tensor:
-        assert torch.equal(
-            find_bin(bin_edges, values, ret_value=False),
-            torch.tensor([0, 0, 0, 1, 3, 3]),
-        )
-
-
 ###############################################################################
 # Test quantization and new cols_to_scale_params
 ###############################################################################
diff --git a/tests/test_losses/test_losses.py b/tests/test_losses/test_losses.py
index a030fe8c..efea281c 100644
--- a/tests/test_losses/test_losses.py
+++ b/tests/test_losses/test_losses.py
@@ -357,33 +357,6 @@ def test_inverse_maps():
     assert all(out)
 
 
-##############################################################################
-# Test LDS weight compatible losses
-##############################################################################
-@pytest.mark.parametrize(
-    "loss_f",
-    [
-        MSELoss,
-        MSLELoss,
-        RMSELoss,
-        RMSLELoss,
-        TweedieLoss,
-        L1Loss,
-        FocalR_L1Loss,
-        FocalR_MSELoss,
-        FocalR_RMSELoss,
-        HuberLoss,
-    ],
-)
-def test_lds_losses(loss_f):
-    lds_weight = np.array([1, 1, 1, 1]).reshape(-1, 1)
-    t_lds_weight = torch.from_numpy(lds_weight)
-    assert (
-        loss_f()(t_pred, t_true).item()
-        == loss_f()(t_pred, t_true, lds_weight=t_lds_weight).item()
-    )
-
-
 ##############################################################################
 # Test FocalR loss activation functions
 ##############################################################################
diff --git a/tests/test_model_components/test_mc_tab_mlp.py b/tests/test_model_components/test_mc_tab_mlp.py
index 5846b67b..9bf5070b 100644
--- a/tests/test_model_components/test_mc_tab_mlp.py
+++ b/tests/test_model_components/test_mc_tab_mlp.py
@@ -4,8 +4,7 @@
 import torch
 import pytest
 
-from pytorch_widedeep.models import TabMlp, WideDeep
-from pytorch_widedeep.training import Trainer
+from pytorch_widedeep.models import TabMlp
 
 colnames = list(string.ascii_lowercase)[:10]
 embed_cols = [np.random.choice(np.arange(5), 10) for _ in range(5)]
@@ -187,35 +186,3 @@ def test_act_fn_ValueError():
 #         assert x_cat.size() == torch.Size((s1, s2_cat)) and x_cont.size() == torch.Size(
 #             (s1, s2_cont)
 #         )
-
-
-###############################################################################
-# Test Feature Dsitribution Smoothing
-###############################################################################
-
-
-@pytest.mark.parametrize(
-    "with_lds",
-    [True, False],
-)
-def test_fds(with_lds):
-    # lds with model
-    model = WideDeep(
-        deeptabular=tabmlp,
-        with_fds=True,
-        momentum=None,
-        clip_min=0,
-        clip_max=10,
-    )
-    trainer = Trainer(model, objective="regression", everbose=0)
-    # n_epochs=2 to run self._calibrate_mean_var
-    trainer.fit(X_tab=X_deep, target=target, n_epochs=3, with_lds=with_lds)
-    # simply checking that runs and produces outputs
-    preds = trainer.predict(X_tab=X_deep)
-    module_names = list(model.named_modules())
-    assert module_names[-2][0] == "fds_layer"
-    assert module_names[-1][0] == "fds_layer.pred_layer"
-    assert preds.shape[0] == 10 and "train_loss" in trainer.history
-
-    trainer.model.fds_layer.reset()
-    assert float(trainer.model.fds_layer.num_samples_tracked.sum()) == 0
diff --git a/tests/test_model_functioning/test_miscellaneous.py b/tests/test_model_functioning/test_miscellaneous.py
index 937580f3..e54d407d 100644
--- a/tests/test_model_functioning/test_miscellaneous.py
+++ b/tests/test_model_functioning/test_miscellaneous.py
@@ -24,7 +24,6 @@
 from pytorch_widedeep.training import Trainer
 from pytorch_widedeep.callbacks import EarlyStopping
 from pytorch_widedeep.preprocessing import TabPreprocessor
-from pytorch_widedeep.training._wd_dataset import WideDeepDataset
 
 # Wide array
 X_wide = np.random.choice(50, (32, 10))
@@ -382,38 +381,3 @@ def test_handle_columns_with_dots():
     )
     preds = trainer.predict(X_tab=X_tab, batch_size=16)
     assert preds.shape[0] == 32 and "train_loss" in trainer.history
-
-
-###############################################################################
-# test Label Distribution Smoothing
-###############################################################################
-
-
-def test_lds_component_with_model():
-    model = WideDeep(deeptabular=tabmlp)
-    trainer = Trainer(model, objective="regression", verbose=0)
-    trainer.fit(X_tab=X_tab, target=target, with_lds=True)
-    # simply checking that runs and produces outputs
-    preds = trainer.predict(X_tab=X_tab)
-
-    assert preds.shape[0] == 32 and "train_loss" in trainer.history
-
-
-def test_lds_component_with_dataset():
-    dataset_with_lds = WideDeepDataset(X_tab=X_tab, target=target, with_lds=True)
-    # test if weights were created
-    assert dataset_with_lds.weights.shape[0] == 32
-
-
-###############################################################################
-# test Trainer _extract_kwargs
-###############################################################################
-
-
-def test_Trainer_extract_kwargs():
-    lds_args, dataloader_args, finetune_args = Trainer._extract_kwargs(
-        {"pin_memory": True, "lds_ks": 7, "n_epochs": 10}
-    )
-    assert lds_args == {"lds_ks": 7}
-    assert dataloader_args == {"pin_memory": True}
-    assert finetune_args == {"n_epochs": 10}
diff --git a/tests/test_multi_model_and_mutil_data/data_for_muti_tabular_components/test.csv b/tests/test_multi_model_and_mutil_data/data_for_muti_tabular_components/test.csv
new file mode 100644
index 00000000..480912aa
--- /dev/null
+++ b/tests/test_multi_model_and_mutil_data/data_for_muti_tabular_components/test.csv
@@ -0,0 +1,101 @@
+user_id,item_id,purchased,id_x,age,gender,location,review,id_y,price,color,category,description
+16,27,1,16,20,male,location_c,Its must health collection attack similar see.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+18,5,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,5,149.51,red,category_2,None laugh national tell as over.
+8,4,0,8,36,male,location_b,Seek what risk particularly up today.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+15,3,1,15,41,male,location_a,Age provide more price kitchen wall education.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+5,30,1,5,25,female,location_c,Talk fly art me white kitchen our.,30,716.11,red,category_3,Only night people bill fly year.
+20,21,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+30,1,1,30,44,male,location_a,Call language material science process through resource.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+13,19,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+16,30,1,16,20,male,location_c,Its must health collection attack similar see.,30,716.11,red,category_3,Only night people bill fly year.
+32,13,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+10,1,1,10,28,male,location_b,Return young national not right citizen environmental food these.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+2,15,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+1,22,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+25,20,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+14,16,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+32,4,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+18,6,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,6,804.18,green,category_1,Point thought or occur term way.
+15,27,1,15,41,male,location_a,Age provide more price kitchen wall education.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+30,6,0,30,44,male,location_a,Call language material science process through resource.,6,804.18,green,category_1,Point thought or occur term way.
+9,26,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+4,28,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+16,19,1,16,20,male,location_c,Its must health collection attack similar see.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+21,26,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+21,29,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,29,128.4,yellow,category_2,Time may involve important artist fear.
+15,5,0,15,41,male,location_a,Age provide more price kitchen wall education.,5,149.51,red,category_2,None laugh national tell as over.
+9,7,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+7,11,1,7,56,female,location_a,Seek he civil than they any.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+30,21,1,30,44,male,location_a,Call language material science process through resource.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+8,23,0,8,36,male,location_b,Seek what risk particularly up today.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+29,20,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+17,8,0,17,39,male,location_b,Energy voice player develop myself present.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+11,20,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+14,21,0,14,57,male,location_b,About term up financial their especially newspaper computer eight.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+6,16,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+26,4,1,26,29,female,location_c,Goal rate never success their weight week bag.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+17,12,0,17,39,male,location_b,Energy voice player develop myself present.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+26,29,0,26,29,female,location_c,Goal rate never success their weight week bag.,29,128.4,yellow,category_2,Time may involve important artist fear.
+7,30,0,7,56,female,location_a,Seek he civil than they any.,30,716.11,red,category_3,Only night people bill fly year.
+9,8,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+4,15,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+13,26,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+6,17,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+2,5,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,5,149.51,red,category_2,None laugh national tell as over.
+29,9,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+22,8,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+30,4,0,30,44,male,location_a,Call language material science process through resource.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+28,26,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+2,25,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+12,23,0,12,41,female,location_b,Just side picture road available summer spend require your sound.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+12,18,1,12,41,female,location_b,Just side picture road available summer spend require your sound.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+29,10,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+16,19,1,16,20,male,location_c,Its must health collection attack similar see.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+27,31,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+6,18,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+9,27,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+16,2,1,16,20,male,location_c,Its must health collection attack similar see.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+23,31,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+23,21,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+18,12,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+21,15,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+18,4,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+23,15,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+7,16,0,7,56,female,location_a,Seek he civil than they any.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+18,4,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+25,23,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+12,16,1,12,41,female,location_b,Just side picture road available summer spend require your sound.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+23,2,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+10,13,1,10,28,male,location_b,Return young national not right citizen environmental food these.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+20,12,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+3,6,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,6,804.18,green,category_1,Point thought or occur term way.
+15,15,0,15,41,male,location_a,Age provide more price kitchen wall education.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+31,1,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+13,9,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+12,21,0,12,41,female,location_b,Just side picture road available summer spend require your sound.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+2,10,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+8,24,0,8,36,male,location_b,Seek what risk particularly up today.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+24,26,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+22,20,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+9,11,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+32,21,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+23,15,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+25,20,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+9,12,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+26,3,0,26,29,female,location_c,Goal rate never success their weight week bag.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+21,15,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+32,10,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+5,16,1,5,25,female,location_c,Talk fly art me white kitchen our.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+21,11,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+22,29,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,29,128.4,yellow,category_2,Time may involve important artist fear.
+17,14,0,17,39,male,location_b,Energy voice player develop myself present.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+5,23,1,5,25,female,location_c,Talk fly art me white kitchen our.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+7,1,0,7,56,female,location_a,Seek he civil than they any.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+25,29,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,29,128.4,yellow,category_2,Time may involve important artist fear.
+11,27,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+25,23,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+7,31,0,7,56,female,location_a,Seek he civil than they any.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+11,29,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,29,128.4,yellow,category_2,Time may involve important artist fear.
+3,19,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+21,17,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+5,22,0,5,25,female,location_c,Talk fly art me white kitchen our.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
diff --git a/tests/test_multi_model_and_mutil_data/data_for_muti_tabular_components/train.csv b/tests/test_multi_model_and_mutil_data/data_for_muti_tabular_components/train.csv
new file mode 100644
index 00000000..afe31211
--- /dev/null
+++ b/tests/test_multi_model_and_mutil_data/data_for_muti_tabular_components/train.csv
@@ -0,0 +1,801 @@
+user_id,item_id,purchased,id_x,age,gender,location,review,id_y,price,color,category,description
+8,2,1,8,36,male,location_b,Seek what risk particularly up today.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+16,15,0,16,20,male,location_c,Its must health collection attack similar see.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+7,6,1,7,56,female,location_a,Seek he civil than they any.,6,804.18,green,category_1,Point thought or occur term way.
+3,2,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+14,15,0,14,57,male,location_b,About term up financial their especially newspaper computer eight.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+13,27,0,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+29,18,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+11,28,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+18,10,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+22,7,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+25,7,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+23,17,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+30,8,1,30,44,male,location_a,Call language material science process through resource.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+6,19,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+13,5,0,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,5,149.51,red,category_2,None laugh national tell as over.
+15,19,0,15,41,male,location_a,Age provide more price kitchen wall education.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+15,7,1,15,41,male,location_a,Age provide more price kitchen wall education.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+18,30,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,30,716.11,red,category_3,Only night people bill fly year.
+11,24,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+14,18,0,14,57,male,location_b,About term up financial their especially newspaper computer eight.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+11,16,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+30,25,1,30,44,male,location_a,Call language material science process through resource.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+15,21,0,15,41,male,location_a,Age provide more price kitchen wall education.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+15,3,1,15,41,male,location_a,Age provide more price kitchen wall education.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+26,18,0,26,29,female,location_c,Goal rate never success their weight week bag.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+14,21,0,14,57,male,location_b,About term up financial their especially newspaper computer eight.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+32,26,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+10,17,0,10,28,male,location_b,Return young national not right citizen environmental food these.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+16,17,1,16,20,male,location_c,Its must health collection attack similar see.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+26,24,0,26,29,female,location_c,Goal rate never success their weight week bag.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+9,32,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+4,8,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+11,28,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+25,25,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+17,1,0,17,39,male,location_b,Energy voice player develop myself present.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+18,22,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+19,28,0,19,41,male,location_b,Study social leave record wish public artist sit.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+30,1,1,30,44,male,location_a,Call language material science process through resource.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+12,7,1,12,41,female,location_b,Just side picture road available summer spend require your sound.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+13,10,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+11,1,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+32,2,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+24,20,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+4,16,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+6,32,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+9,9,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+11,17,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+14,13,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+26,24,1,26,29,female,location_c,Goal rate never success their weight week bag.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+29,8,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+15,5,1,15,41,male,location_a,Age provide more price kitchen wall education.,5,149.51,red,category_2,None laugh national tell as over.
+2,15,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+1,5,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,5,149.51,red,category_2,None laugh national tell as over.
+15,5,0,15,41,male,location_a,Age provide more price kitchen wall education.,5,149.51,red,category_2,None laugh national tell as over.
+22,5,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,5,149.51,red,category_2,None laugh national tell as over.
+18,32,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+9,31,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+31,27,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+7,7,1,7,56,female,location_a,Seek he civil than they any.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+23,28,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+30,4,0,30,44,male,location_a,Call language material science process through resource.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+4,26,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+7,16,0,7,56,female,location_a,Seek he civil than they any.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+13,29,0,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,29,128.4,yellow,category_2,Time may involve important artist fear.
+28,12,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+30,16,0,30,44,male,location_a,Call language material science process through resource.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+29,7,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+1,6,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,6,804.18,green,category_1,Point thought or occur term way.
+11,27,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+31,14,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+4,11,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+1,25,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+30,19,1,30,44,male,location_a,Call language material science process through resource.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+32,10,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+19,14,0,19,41,male,location_b,Study social leave record wish public artist sit.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+4,21,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+4,31,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+4,6,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,6,804.18,green,category_1,Point thought or occur term way.
+5,5,0,5,25,female,location_c,Talk fly art me white kitchen our.,5,149.51,red,category_2,None laugh national tell as over.
+26,8,0,26,29,female,location_c,Goal rate never success their weight week bag.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+3,6,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,6,804.18,green,category_1,Point thought or occur term way.
+21,17,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+21,16,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+26,9,1,26,29,female,location_c,Goal rate never success their weight week bag.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+30,21,0,30,44,male,location_a,Call language material science process through resource.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+1,30,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,30,716.11,red,category_3,Only night people bill fly year.
+5,14,1,5,25,female,location_c,Talk fly art me white kitchen our.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+9,23,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+16,24,1,16,20,male,location_c,Its must health collection attack similar see.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+11,29,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,29,128.4,yellow,category_2,Time may involve important artist fear.
+1,20,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+9,17,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+7,10,1,7,56,female,location_a,Seek he civil than they any.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+19,14,1,19,41,male,location_b,Study social leave record wish public artist sit.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+14,17,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+17,4,0,17,39,male,location_b,Energy voice player develop myself present.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+28,18,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+1,22,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+17,11,1,17,39,male,location_b,Energy voice player develop myself present.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+28,1,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+5,10,0,5,25,female,location_c,Talk fly art me white kitchen our.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+24,10,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+9,3,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+24,3,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+14,16,0,14,57,male,location_b,About term up financial their especially newspaper computer eight.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+23,27,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+16,11,0,16,20,male,location_c,Its must health collection attack similar see.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+27,2,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+22,27,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+18,11,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+25,3,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+15,13,1,15,41,male,location_a,Age provide more price kitchen wall education.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+23,20,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+15,2,0,15,41,male,location_a,Age provide more price kitchen wall education.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+26,22,1,26,29,female,location_c,Goal rate never success their weight week bag.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+5,18,1,5,25,female,location_c,Talk fly art me white kitchen our.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+26,22,0,26,29,female,location_c,Goal rate never success their weight week bag.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+8,17,0,8,36,male,location_b,Seek what risk particularly up today.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+17,3,0,17,39,male,location_b,Energy voice player develop myself present.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+28,23,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+28,8,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+13,17,0,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+28,1,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+24,28,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+22,21,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+20,20,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+21,26,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+9,13,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+25,12,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+26,1,1,26,29,female,location_c,Goal rate never success their weight week bag.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+19,14,1,19,41,male,location_b,Study social leave record wish public artist sit.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+21,30,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,30,716.11,red,category_3,Only night people bill fly year.
+29,14,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+11,6,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,6,804.18,green,category_1,Point thought or occur term way.
+22,6,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,6,804.18,green,category_1,Point thought or occur term way.
+20,15,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+10,2,0,10,28,male,location_b,Return young national not right citizen environmental food these.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+16,31,0,16,20,male,location_c,Its must health collection attack similar see.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+30,27,0,30,44,male,location_a,Call language material science process through resource.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+25,32,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+16,10,0,16,20,male,location_c,Its must health collection attack similar see.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+7,28,0,7,56,female,location_a,Seek he civil than they any.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+12,30,0,12,41,female,location_b,Just side picture road available summer spend require your sound.,30,716.11,red,category_3,Only night people bill fly year.
+16,8,1,16,20,male,location_c,Its must health collection attack similar see.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+9,30,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,30,716.11,red,category_3,Only night people bill fly year.
+29,28,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+11,31,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+17,16,1,17,39,male,location_b,Energy voice player develop myself present.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+32,16,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+29,5,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,5,149.51,red,category_2,None laugh national tell as over.
+16,18,1,16,20,male,location_c,Its must health collection attack similar see.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+21,6,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,6,804.18,green,category_1,Point thought or occur term way.
+10,15,1,10,28,male,location_b,Return young national not right citizen environmental food these.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+10,14,0,10,28,male,location_b,Return young national not right citizen environmental food these.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+27,27,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+30,27,0,30,44,male,location_a,Call language material science process through resource.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+14,27,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+2,25,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+1,23,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+25,27,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+32,15,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+28,32,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+25,22,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+11,30,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,30,716.11,red,category_3,Only night people bill fly year.
+2,26,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+6,28,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+30,12,0,30,44,male,location_a,Call language material science process through resource.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+17,25,1,17,39,male,location_b,Energy voice player develop myself present.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+14,30,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,30,716.11,red,category_3,Only night people bill fly year.
+22,25,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+27,17,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+31,2,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+23,15,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+3,2,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+13,2,0,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+16,9,1,16,20,male,location_c,Its must health collection attack similar see.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+8,14,1,8,36,male,location_b,Seek what risk particularly up today.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+17,24,0,17,39,male,location_b,Energy voice player develop myself present.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+8,11,1,8,36,male,location_b,Seek what risk particularly up today.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+7,2,1,7,56,female,location_a,Seek he civil than they any.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+25,26,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+5,16,0,5,25,female,location_c,Talk fly art me white kitchen our.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+20,8,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+23,28,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+5,22,0,5,25,female,location_c,Talk fly art me white kitchen our.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+27,32,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+28,24,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+10,28,0,10,28,male,location_b,Return young national not right citizen environmental food these.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+18,31,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+28,18,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+16,6,1,16,20,male,location_c,Its must health collection attack similar see.,6,804.18,green,category_1,Point thought or occur term way.
+29,16,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+25,22,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+32,21,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+32,14,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+17,22,1,17,39,male,location_b,Energy voice player develop myself present.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+18,1,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+6,16,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+32,16,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+32,29,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,29,128.4,yellow,category_2,Time may involve important artist fear.
+6,19,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+26,16,1,26,29,female,location_c,Goal rate never success their weight week bag.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+24,31,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+28,22,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+30,18,1,30,44,male,location_a,Call language material science process through resource.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+17,15,0,17,39,male,location_b,Energy voice player develop myself present.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+13,21,0,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+12,13,0,12,41,female,location_b,Just side picture road available summer spend require your sound.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+31,18,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+7,13,1,7,56,female,location_a,Seek he civil than they any.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+15,24,0,15,41,male,location_a,Age provide more price kitchen wall education.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+20,1,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+18,3,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+19,9,1,19,41,male,location_b,Study social leave record wish public artist sit.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+7,1,1,7,56,female,location_a,Seek he civil than they any.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+31,31,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+22,12,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+17,31,0,17,39,male,location_b,Energy voice player develop myself present.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+5,26,1,5,25,female,location_c,Talk fly art me white kitchen our.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+5,4,0,5,25,female,location_c,Talk fly art me white kitchen our.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+10,20,0,10,28,male,location_b,Return young national not right citizen environmental food these.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+16,8,1,16,20,male,location_c,Its must health collection attack similar see.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+15,25,1,15,41,male,location_a,Age provide more price kitchen wall education.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+29,20,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+20,4,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+14,14,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+6,11,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+12,5,0,12,41,female,location_b,Just side picture road available summer spend require your sound.,5,149.51,red,category_2,None laugh national tell as over.
+1,27,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+31,19,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+15,19,1,15,41,male,location_a,Age provide more price kitchen wall education.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+30,5,0,30,44,male,location_a,Call language material science process through resource.,5,149.51,red,category_2,None laugh national tell as over.
+17,13,1,17,39,male,location_b,Energy voice player develop myself present.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+8,15,0,8,36,male,location_b,Seek what risk particularly up today.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+18,10,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+4,11,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+19,29,0,19,41,male,location_b,Study social leave record wish public artist sit.,29,128.4,yellow,category_2,Time may involve important artist fear.
+30,20,1,30,44,male,location_a,Call language material science process through resource.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+18,32,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+6,3,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+21,17,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+6,15,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+18,3,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+31,29,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,29,128.4,yellow,category_2,Time may involve important artist fear.
+12,28,1,12,41,female,location_b,Just side picture road available summer spend require your sound.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+6,31,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+27,22,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+7,11,1,7,56,female,location_a,Seek he civil than they any.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+27,32,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+26,3,1,26,29,female,location_c,Goal rate never success their weight week bag.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+6,21,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+21,8,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+1,30,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,30,716.11,red,category_3,Only night people bill fly year.
+4,13,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+32,29,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,29,128.4,yellow,category_2,Time may involve important artist fear.
+14,18,0,14,57,male,location_b,About term up financial their especially newspaper computer eight.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+19,29,1,19,41,male,location_b,Study social leave record wish public artist sit.,29,128.4,yellow,category_2,Time may involve important artist fear.
+8,2,0,8,36,male,location_b,Seek what risk particularly up today.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+11,20,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+27,6,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,6,804.18,green,category_1,Point thought or occur term way.
+8,30,0,8,36,male,location_b,Seek what risk particularly up today.,30,716.11,red,category_3,Only night people bill fly year.
+10,32,1,10,28,male,location_b,Return young national not right citizen environmental food these.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+18,27,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+31,16,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+10,25,0,10,28,male,location_b,Return young national not right citizen environmental food these.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+9,5,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,5,149.51,red,category_2,None laugh national tell as over.
+27,22,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+1,19,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+32,10,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+31,23,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+25,30,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,30,716.11,red,category_3,Only night people bill fly year.
+13,16,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+15,27,0,15,41,male,location_a,Age provide more price kitchen wall education.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+21,31,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+25,10,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+3,9,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+7,29,0,7,56,female,location_a,Seek he civil than they any.,29,128.4,yellow,category_2,Time may involve important artist fear.
+30,1,0,30,44,male,location_a,Call language material science process through resource.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+27,10,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+31,17,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+26,6,1,26,29,female,location_c,Goal rate never success their weight week bag.,6,804.18,green,category_1,Point thought or occur term way.
+25,21,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+3,5,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,5,149.51,red,category_2,None laugh national tell as over.
+19,15,0,19,41,male,location_b,Study social leave record wish public artist sit.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+28,7,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+29,11,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+2,3,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+4,19,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+24,28,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+16,27,0,16,20,male,location_c,Its must health collection attack similar see.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+11,12,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+25,16,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+10,15,1,10,28,male,location_b,Return young national not right citizen environmental food these.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+17,30,1,17,39,male,location_b,Energy voice player develop myself present.,30,716.11,red,category_3,Only night people bill fly year.
+1,30,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,30,716.11,red,category_3,Only night people bill fly year.
+11,5,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,5,149.51,red,category_2,None laugh national tell as over.
+23,20,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+17,30,1,17,39,male,location_b,Energy voice player develop myself present.,30,716.11,red,category_3,Only night people bill fly year.
+13,25,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+7,16,1,7,56,female,location_a,Seek he civil than they any.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+23,19,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+2,26,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+1,4,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+19,15,1,19,41,male,location_b,Study social leave record wish public artist sit.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+15,13,1,15,41,male,location_a,Age provide more price kitchen wall education.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+9,7,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+20,29,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,29,128.4,yellow,category_2,Time may involve important artist fear.
+24,9,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+19,21,1,19,41,male,location_b,Study social leave record wish public artist sit.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+12,13,0,12,41,female,location_b,Just side picture road available summer spend require your sound.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+24,18,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+17,31,1,17,39,male,location_b,Energy voice player develop myself present.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+22,8,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+5,10,0,5,25,female,location_c,Talk fly art me white kitchen our.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+26,24,0,26,29,female,location_c,Goal rate never success their weight week bag.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+26,1,1,26,29,female,location_c,Goal rate never success their weight week bag.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+8,30,1,8,36,male,location_b,Seek what risk particularly up today.,30,716.11,red,category_3,Only night people bill fly year.
+17,25,1,17,39,male,location_b,Energy voice player develop myself present.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+7,15,1,7,56,female,location_a,Seek he civil than they any.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+2,21,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+5,30,1,5,25,female,location_c,Talk fly art me white kitchen our.,30,716.11,red,category_3,Only night people bill fly year.
+27,8,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+3,3,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+32,8,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+16,9,1,16,20,male,location_c,Its must health collection attack similar see.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+30,24,1,30,44,male,location_a,Call language material science process through resource.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+10,27,0,10,28,male,location_b,Return young national not right citizen environmental food these.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+32,27,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+3,24,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+29,29,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,29,128.4,yellow,category_2,Time may involve important artist fear.
+24,7,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+23,4,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+18,13,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+30,6,0,30,44,male,location_a,Call language material science process through resource.,6,804.18,green,category_1,Point thought or occur term way.
+2,4,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+16,9,0,16,20,male,location_c,Its must health collection attack similar see.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+5,14,0,5,25,female,location_c,Talk fly art me white kitchen our.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+15,22,0,15,41,male,location_a,Age provide more price kitchen wall education.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+1,18,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+9,17,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+8,2,0,8,36,male,location_b,Seek what risk particularly up today.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+1,23,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+21,2,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+17,4,0,17,39,male,location_b,Energy voice player develop myself present.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+27,8,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+31,29,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,29,128.4,yellow,category_2,Time may involve important artist fear.
+7,29,0,7,56,female,location_a,Seek he civil than they any.,29,128.4,yellow,category_2,Time may involve important artist fear.
+3,20,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+2,4,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+26,28,0,26,29,female,location_c,Goal rate never success their weight week bag.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+32,29,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,29,128.4,yellow,category_2,Time may involve important artist fear.
+6,21,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+5,9,1,5,25,female,location_c,Talk fly art me white kitchen our.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+21,25,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+30,28,0,30,44,male,location_a,Call language material science process through resource.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+8,14,1,8,36,male,location_b,Seek what risk particularly up today.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+29,15,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+22,30,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,30,716.11,red,category_3,Only night people bill fly year.
+27,7,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+28,21,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+24,10,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+5,6,0,5,25,female,location_c,Talk fly art me white kitchen our.,6,804.18,green,category_1,Point thought or occur term way.
+6,28,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+24,9,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+22,8,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+23,28,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+19,20,1,19,41,male,location_b,Study social leave record wish public artist sit.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+7,14,0,7,56,female,location_a,Seek he civil than they any.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+31,15,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+23,24,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+18,15,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+2,18,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+12,18,1,12,41,female,location_b,Just side picture road available summer spend require your sound.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+22,23,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+12,10,1,12,41,female,location_b,Just side picture road available summer spend require your sound.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+5,10,0,5,25,female,location_c,Talk fly art me white kitchen our.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+6,14,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+27,30,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,30,716.11,red,category_3,Only night people bill fly year.
+11,24,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+21,6,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,6,804.18,green,category_1,Point thought or occur term way.
+10,11,0,10,28,male,location_b,Return young national not right citizen environmental food these.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+6,18,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+28,32,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+27,18,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+8,23,1,8,36,male,location_b,Seek what risk particularly up today.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+8,19,1,8,36,male,location_b,Seek what risk particularly up today.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+20,3,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+26,4,0,26,29,female,location_c,Goal rate never success their weight week bag.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+14,20,0,14,57,male,location_b,About term up financial their especially newspaper computer eight.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+9,17,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+21,8,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+32,28,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+9,25,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+23,5,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,5,149.51,red,category_2,None laugh national tell as over.
+3,28,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+30,5,1,30,44,male,location_a,Call language material science process through resource.,5,149.51,red,category_2,None laugh national tell as over.
+28,26,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+19,8,1,19,41,male,location_b,Study social leave record wish public artist sit.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+2,21,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+30,24,0,30,44,male,location_a,Call language material science process through resource.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+28,7,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+28,26,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+26,20,1,26,29,female,location_c,Goal rate never success their weight week bag.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+15,22,0,15,41,male,location_a,Age provide more price kitchen wall education.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+5,8,0,5,25,female,location_c,Talk fly art me white kitchen our.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+23,23,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+16,7,0,16,20,male,location_c,Its must health collection attack similar see.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+17,13,0,17,39,male,location_b,Energy voice player develop myself present.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+10,5,0,10,28,male,location_b,Return young national not right citizen environmental food these.,5,149.51,red,category_2,None laugh national tell as over.
+32,30,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,30,716.11,red,category_3,Only night people bill fly year.
+21,21,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+29,5,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,5,149.51,red,category_2,None laugh national tell as over.
+29,20,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+4,23,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+20,30,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,30,716.11,red,category_3,Only night people bill fly year.
+3,4,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+19,5,0,19,41,male,location_b,Study social leave record wish public artist sit.,5,149.51,red,category_2,None laugh national tell as over.
+25,30,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,30,716.11,red,category_3,Only night people bill fly year.
+29,13,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+31,10,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+29,7,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+6,12,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+16,29,1,16,20,male,location_c,Its must health collection attack similar see.,29,128.4,yellow,category_2,Time may involve important artist fear.
+11,24,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+19,25,1,19,41,male,location_b,Study social leave record wish public artist sit.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+22,4,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+5,22,0,5,25,female,location_c,Talk fly art me white kitchen our.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+25,19,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+10,22,0,10,28,male,location_b,Return young national not right citizen environmental food these.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+10,23,1,10,28,male,location_b,Return young national not right citizen environmental food these.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+26,9,0,26,29,female,location_c,Goal rate never success their weight week bag.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+20,25,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+9,6,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,6,804.18,green,category_1,Point thought or occur term way.
+24,2,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+20,12,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+22,32,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+15,9,0,15,41,male,location_a,Age provide more price kitchen wall education.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+5,19,0,5,25,female,location_c,Talk fly art me white kitchen our.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+3,22,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+25,10,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+12,11,1,12,41,female,location_b,Just side picture road available summer spend require your sound.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+3,27,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+16,29,1,16,20,male,location_c,Its must health collection attack similar see.,29,128.4,yellow,category_2,Time may involve important artist fear.
+29,15,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+20,31,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+24,29,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,29,128.4,yellow,category_2,Time may involve important artist fear.
+19,25,1,19,41,male,location_b,Study social leave record wish public artist sit.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+11,13,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+17,4,1,17,39,male,location_b,Energy voice player develop myself present.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+24,7,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+19,6,0,19,41,male,location_b,Study social leave record wish public artist sit.,6,804.18,green,category_1,Point thought or occur term way.
+18,29,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,29,128.4,yellow,category_2,Time may involve important artist fear.
+28,6,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,6,804.18,green,category_1,Point thought or occur term way.
+29,23,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+27,4,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+24,16,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+6,24,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+2,21,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+22,10,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+3,19,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+9,31,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+1,6,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,6,804.18,green,category_1,Point thought or occur term way.
+17,14,0,17,39,male,location_b,Energy voice player develop myself present.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+28,8,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+16,20,0,16,20,male,location_c,Its must health collection attack similar see.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+4,16,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+30,5,0,30,44,male,location_a,Call language material science process through resource.,5,149.51,red,category_2,None laugh national tell as over.
+32,2,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+10,19,1,10,28,male,location_b,Return young national not right citizen environmental food these.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+1,23,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+27,12,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+24,5,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,5,149.51,red,category_2,None laugh national tell as over.
+25,31,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+25,24,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+2,23,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+23,16,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+22,9,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+23,22,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+30,31,0,30,44,male,location_a,Call language material science process through resource.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+9,5,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,5,149.51,red,category_2,None laugh national tell as over.
+3,19,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+3,13,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+21,20,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+31,17,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+13,19,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+4,22,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+8,24,1,8,36,male,location_b,Seek what risk particularly up today.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+26,29,1,26,29,female,location_c,Goal rate never success their weight week bag.,29,128.4,yellow,category_2,Time may involve important artist fear.
+22,12,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+32,24,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+6,28,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+28,12,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+21,7,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+21,19,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+29,28,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+29,23,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+3,23,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+18,4,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+26,24,0,26,29,female,location_c,Goal rate never success their weight week bag.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+2,10,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+3,9,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+16,24,1,16,20,male,location_c,Its must health collection attack similar see.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+25,3,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+29,24,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+29,5,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,5,149.51,red,category_2,None laugh national tell as over.
+24,26,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+18,16,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+2,12,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+25,8,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+17,29,0,17,39,male,location_b,Energy voice player develop myself present.,29,128.4,yellow,category_2,Time may involve important artist fear.
+19,32,0,19,41,male,location_b,Study social leave record wish public artist sit.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+8,9,0,8,36,male,location_b,Seek what risk particularly up today.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+29,12,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+6,21,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+5,19,1,5,25,female,location_c,Talk fly art me white kitchen our.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+11,12,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+15,8,0,15,41,male,location_a,Age provide more price kitchen wall education.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+9,16,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+2,24,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+30,9,0,30,44,male,location_a,Call language material science process through resource.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+5,20,1,5,25,female,location_c,Talk fly art me white kitchen our.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+31,27,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+5,9,1,5,25,female,location_c,Talk fly art me white kitchen our.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+5,29,1,5,25,female,location_c,Talk fly art me white kitchen our.,29,128.4,yellow,category_2,Time may involve important artist fear.
+23,9,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+9,28,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+8,10,1,8,36,male,location_b,Seek what risk particularly up today.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+11,11,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+15,23,1,15,41,male,location_a,Age provide more price kitchen wall education.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+6,17,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+18,9,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+6,11,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+11,22,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+2,6,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,6,804.18,green,category_1,Point thought or occur term way.
+17,14,1,17,39,male,location_b,Energy voice player develop myself present.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+2,32,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+20,31,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+26,20,1,26,29,female,location_c,Goal rate never success their weight week bag.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+5,4,0,5,25,female,location_c,Talk fly art me white kitchen our.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+29,27,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+30,14,1,30,44,male,location_a,Call language material science process through resource.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+10,21,1,10,28,male,location_b,Return young national not right citizen environmental food these.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+23,26,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+24,7,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+16,30,0,16,20,male,location_c,Its must health collection attack similar see.,30,716.11,red,category_3,Only night people bill fly year.
+18,29,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,29,128.4,yellow,category_2,Time may involve important artist fear.
+10,7,0,10,28,male,location_b,Return young national not right citizen environmental food these.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+19,25,1,19,41,male,location_b,Study social leave record wish public artist sit.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+16,11,1,16,20,male,location_c,Its must health collection attack similar see.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+21,13,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+32,30,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,30,716.11,red,category_3,Only night people bill fly year.
+20,32,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+6,26,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+16,14,1,16,20,male,location_c,Its must health collection attack similar see.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+4,4,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+32,31,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+1,7,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+9,17,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+26,24,0,26,29,female,location_c,Goal rate never success their weight week bag.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+26,4,0,26,29,female,location_c,Goal rate never success their weight week bag.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+24,19,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+25,29,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,29,128.4,yellow,category_2,Time may involve important artist fear.
+8,9,0,8,36,male,location_b,Seek what risk particularly up today.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+26,24,1,26,29,female,location_c,Goal rate never success their weight week bag.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+24,10,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+26,3,0,26,29,female,location_c,Goal rate never success their weight week bag.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+3,9,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+31,30,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,30,716.11,red,category_3,Only night people bill fly year.
+9,21,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+11,26,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+22,32,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+31,2,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+22,8,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+20,2,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+17,15,0,17,39,male,location_b,Energy voice player develop myself present.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+31,11,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+10,16,0,10,28,male,location_b,Return young national not right citizen environmental food these.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+8,13,0,8,36,male,location_b,Seek what risk particularly up today.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+29,21,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+10,27,0,10,28,male,location_b,Return young national not right citizen environmental food these.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+27,31,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+9,14,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+31,25,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+12,30,1,12,41,female,location_b,Just side picture road available summer spend require your sound.,30,716.11,red,category_3,Only night people bill fly year.
+23,17,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+13,16,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+20,15,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+19,14,1,19,41,male,location_b,Study social leave record wish public artist sit.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+21,31,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+18,19,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+25,26,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+10,19,0,10,28,male,location_b,Return young national not right citizen environmental food these.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+19,6,1,19,41,male,location_b,Study social leave record wish public artist sit.,6,804.18,green,category_1,Point thought or occur term way.
+29,17,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+14,13,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+18,9,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+16,16,0,16,20,male,location_c,Its must health collection attack similar see.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+15,15,0,15,41,male,location_a,Age provide more price kitchen wall education.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+7,27,1,7,56,female,location_a,Seek he civil than they any.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+31,7,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+1,11,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+31,31,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+19,21,1,19,41,male,location_b,Study social leave record wish public artist sit.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+4,6,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,6,804.18,green,category_1,Point thought or occur term way.
+3,12,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+12,3,1,12,41,female,location_b,Just side picture road available summer spend require your sound.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+32,12,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+3,1,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+7,22,1,7,56,female,location_a,Seek he civil than they any.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+30,32,0,30,44,male,location_a,Call language material science process through resource.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+30,18,0,30,44,male,location_a,Call language material science process through resource.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+8,22,0,8,36,male,location_b,Seek what risk particularly up today.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+30,17,0,30,44,male,location_a,Call language material science process through resource.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+1,22,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+13,12,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+28,21,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+26,7,0,26,29,female,location_c,Goal rate never success their weight week bag.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+9,31,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+16,1,1,16,20,male,location_c,Its must health collection attack similar see.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+25,16,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+18,22,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+1,17,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+16,4,0,16,20,male,location_c,Its must health collection attack similar see.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+30,20,0,30,44,male,location_a,Call language material science process through resource.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+26,20,0,26,29,female,location_c,Goal rate never success their weight week bag.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+19,24,0,19,41,male,location_b,Study social leave record wish public artist sit.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+15,9,1,15,41,male,location_a,Age provide more price kitchen wall education.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+10,30,1,10,28,male,location_b,Return young national not right citizen environmental food these.,30,716.11,red,category_3,Only night people bill fly year.
+27,31,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+16,6,1,16,20,male,location_c,Its must health collection attack similar see.,6,804.18,green,category_1,Point thought or occur term way.
+24,5,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,5,149.51,red,category_2,None laugh national tell as over.
+4,13,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+11,21,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+8,14,1,8,36,male,location_b,Seek what risk particularly up today.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+6,29,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,29,128.4,yellow,category_2,Time may involve important artist fear.
+30,9,1,30,44,male,location_a,Call language material science process through resource.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+30,4,1,30,44,male,location_a,Call language material science process through resource.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+20,2,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+17,1,0,17,39,male,location_b,Energy voice player develop myself present.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+5,3,1,5,25,female,location_c,Talk fly art me white kitchen our.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+23,5,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,5,149.51,red,category_2,None laugh national tell as over.
+5,31,0,5,25,female,location_c,Talk fly art me white kitchen our.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+19,27,0,19,41,male,location_b,Study social leave record wish public artist sit.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+9,27,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+25,29,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,29,128.4,yellow,category_2,Time may involve important artist fear.
+25,6,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,6,804.18,green,category_1,Point thought or occur term way.
+23,8,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+26,9,0,26,29,female,location_c,Goal rate never success their weight week bag.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+1,2,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+30,28,1,30,44,male,location_a,Call language material science process through resource.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+15,16,1,15,41,male,location_a,Age provide more price kitchen wall education.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+23,10,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+13,8,0,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+27,2,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+14,5,0,14,57,male,location_b,About term up financial their especially newspaper computer eight.,5,149.51,red,category_2,None laugh national tell as over.
+3,29,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,29,128.4,yellow,category_2,Time may involve important artist fear.
+16,3,1,16,20,male,location_c,Its must health collection attack similar see.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+29,15,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+4,9,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+15,21,1,15,41,male,location_a,Age provide more price kitchen wall education.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+16,20,0,16,20,male,location_c,Its must health collection attack similar see.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+15,27,1,15,41,male,location_a,Age provide more price kitchen wall education.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+18,4,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+28,32,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+23,25,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+27,27,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+20,25,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+31,16,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+20,10,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+4,27,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+9,25,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+17,14,1,17,39,male,location_b,Energy voice player develop myself present.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+6,29,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,29,128.4,yellow,category_2,Time may involve important artist fear.
+6,13,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+18,25,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+5,13,0,5,25,female,location_c,Talk fly art me white kitchen our.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+31,14,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+20,1,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+13,8,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+16,14,1,16,20,male,location_c,Its must health collection attack similar see.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+16,21,1,16,20,male,location_c,Its must health collection attack similar see.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+25,30,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,30,716.11,red,category_3,Only night people bill fly year.
+20,17,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+32,30,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,30,716.11,red,category_3,Only night people bill fly year.
+31,21,0,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+24,21,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+3,15,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+2,17,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+19,10,0,19,41,male,location_b,Study social leave record wish public artist sit.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+22,15,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
+16,32,1,16,20,male,location_c,Its must health collection attack similar see.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+17,32,1,17,39,male,location_b,Energy voice player develop myself present.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+30,11,1,30,44,male,location_a,Call language material science process through resource.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+11,9,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+12,4,0,12,41,female,location_b,Just side picture road available summer spend require your sound.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+5,4,0,5,25,female,location_c,Talk fly art me white kitchen our.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+27,9,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+5,10,0,5,25,female,location_c,Talk fly art me white kitchen our.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+14,30,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,30,716.11,red,category_3,Only night people bill fly year.
+4,31,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+2,1,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+1,2,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+19,2,1,19,41,male,location_b,Study social leave record wish public artist sit.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+12,7,0,12,41,female,location_b,Just side picture road available summer spend require your sound.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+10,16,0,10,28,male,location_b,Return young national not right citizen environmental food these.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+17,23,1,17,39,male,location_b,Energy voice player develop myself present.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+26,6,1,26,29,female,location_c,Goal rate never success their weight week bag.,6,804.18,green,category_1,Point thought or occur term way.
+26,30,0,26,29,female,location_c,Goal rate never success their weight week bag.,30,716.11,red,category_3,Only night people bill fly year.
+15,20,0,15,41,male,location_a,Age provide more price kitchen wall education.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+3,6,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,6,804.18,green,category_1,Point thought or occur term way.
+25,25,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+4,1,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+6,32,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+22,7,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+15,14,1,15,41,male,location_a,Age provide more price kitchen wall education.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+18,3,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+18,3,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+21,2,0,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+10,26,0,10,28,male,location_b,Return young national not right citizen environmental food these.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+20,11,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+25,22,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+9,6,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,6,804.18,green,category_1,Point thought or occur term way.
+4,7,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+15,5,1,15,41,male,location_a,Age provide more price kitchen wall education.,5,149.51,red,category_2,None laugh national tell as over.
+26,21,0,26,29,female,location_c,Goal rate never success their weight week bag.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+18,29,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,29,128.4,yellow,category_2,Time may involve important artist fear.
+15,23,1,15,41,male,location_a,Age provide more price kitchen wall education.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+28,12,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+6,19,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+5,6,1,5,25,female,location_c,Talk fly art me white kitchen our.,6,804.18,green,category_1,Point thought or occur term way.
+10,25,0,10,28,male,location_b,Return young national not right citizen environmental food these.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+25,21,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+7,6,0,7,56,female,location_a,Seek he civil than they any.,6,804.18,green,category_1,Point thought or occur term way.
+20,29,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,29,128.4,yellow,category_2,Time may involve important artist fear.
+18,7,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+6,12,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+29,27,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+2,6,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,6,804.18,green,category_1,Point thought or occur term way.
+6,21,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+1,19,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+25,6,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,6,804.18,green,category_1,Point thought or occur term way.
+11,25,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+9,18,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+18,32,0,18,19,male,location_b,Task five concern building concern next mouth benefit.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+5,11,1,5,25,female,location_c,Talk fly art me white kitchen our.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+18,27,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+31,5,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,5,149.51,red,category_2,None laugh national tell as over.
+17,16,1,17,39,male,location_b,Energy voice player develop myself present.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+13,30,0,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,30,716.11,red,category_3,Only night people bill fly year.
+9,20,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+26,22,1,26,29,female,location_c,Goal rate never success their weight week bag.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+22,29,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,29,128.4,yellow,category_2,Time may involve important artist fear.
+28,9,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+21,13,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+21,12,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+21,19,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+16,21,1,16,20,male,location_c,Its must health collection attack similar see.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+18,26,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+8,13,0,8,36,male,location_b,Seek what risk particularly up today.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+2,30,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,30,716.11,red,category_3,Only night people bill fly year.
+29,19,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+27,32,0,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+20,16,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+10,28,1,10,28,male,location_b,Return young national not right citizen environmental food these.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+5,29,1,5,25,female,location_c,Talk fly art me white kitchen our.,29,128.4,yellow,category_2,Time may involve important artist fear.
+26,27,0,26,29,female,location_c,Goal rate never success their weight week bag.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+24,6,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,6,804.18,green,category_1,Point thought or occur term way.
+16,23,0,16,20,male,location_c,Its must health collection attack similar see.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+3,26,1,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+28,7,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+7,17,0,7,56,female,location_a,Seek he civil than they any.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+15,29,1,15,41,male,location_a,Age provide more price kitchen wall education.,29,128.4,yellow,category_2,Time may involve important artist fear.
+26,30,0,26,29,female,location_c,Goal rate never success their weight week bag.,30,716.11,red,category_3,Only night people bill fly year.
+23,2,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+7,19,1,7,56,female,location_a,Seek he civil than they any.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+6,8,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+23,20,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+30,14,1,30,44,male,location_a,Call language material science process through resource.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+23,31,0,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+29,29,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,29,128.4,yellow,category_2,Time may involve important artist fear.
+5,20,0,5,25,female,location_c,Talk fly art me white kitchen our.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+8,2,1,8,36,male,location_b,Seek what risk particularly up today.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+7,11,0,7,56,female,location_a,Seek he civil than they any.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+12,11,1,12,41,female,location_b,Just side picture road available summer spend require your sound.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+28,30,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,30,716.11,red,category_3,Only night people bill fly year.
+26,12,0,26,29,female,location_c,Goal rate never success their weight week bag.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+28,26,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+13,29,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,29,128.4,yellow,category_2,Time may involve important artist fear.
+25,1,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+14,18,0,14,57,male,location_b,About term up financial their especially newspaper computer eight.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+7,12,1,7,56,female,location_a,Seek he civil than they any.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+21,13,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,13,709.79,green,category_3,Allow case what protect charge woman field however great relationship parent card.
+8,17,1,8,36,male,location_b,Seek what risk particularly up today.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+21,25,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+8,23,1,8,36,male,location_b,Seek what risk particularly up today.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+30,12,1,30,44,male,location_a,Call language material science process through resource.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+6,9,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+8,16,1,8,36,male,location_b,Seek what risk particularly up today.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+8,12,1,8,36,male,location_b,Seek what risk particularly up today.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+10,25,1,10,28,male,location_b,Return young national not right citizen environmental food these.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
diff --git a/tests/test_multi_model_and_mutil_data/data_for_muti_tabular_components/val.csv b/tests/test_multi_model_and_mutil_data/data_for_muti_tabular_components/val.csv
new file mode 100644
index 00000000..40c4d7ce
--- /dev/null
+++ b/tests/test_multi_model_and_mutil_data/data_for_muti_tabular_components/val.csv
@@ -0,0 +1,101 @@
+user_id,item_id,purchased,id_x,age,gender,location,review,id_y,price,color,category,description
+9,25,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+12,32,0,12,41,female,location_b,Just side picture road available summer spend require your sound.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+11,32,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+9,12,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+29,4,1,29,42,female,location_a,Open soon same fall also table agree dinner community.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+1,32,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+13,25,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+27,32,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+29,32,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+6,2,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,2,363.19,blue,category_3,Officer American business real police generation budget listen through political garden strategy.
+19,3,1,19,41,male,location_b,Study social leave record wish public artist sit.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+15,19,0,15,41,male,location_a,Age provide more price kitchen wall education.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+30,32,0,30,44,male,location_a,Call language material science process through resource.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+8,18,1,8,36,male,location_b,Seek what risk particularly up today.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+3,29,0,3,32,male,location_a,Reach in color economic drug other maintain almost five one important administration list.,29,128.4,yellow,category_2,Time may involve important artist fear.
+28,7,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+20,3,1,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,3,288.13,green,category_2,Employee behavior study painting benefit nor attack five like the that each guy.
+17,23,0,17,39,male,location_b,Energy voice player develop myself present.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+29,8,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+14,23,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+11,9,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+14,4,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+18,11,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+19,19,1,19,41,male,location_b,Study social leave record wish public artist sit.,19,864.47,yellow,category_1,Plan lose reach response ahead recognize involve service single method.
+7,9,1,7,56,female,location_a,Seek he civil than they any.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+4,18,0,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+9,16,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+21,16,1,21,47,female,location_b,House federal degree upon should spend hand meeting service compare at section great.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+32,10,1,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+27,25,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+5,6,1,5,25,female,location_c,Talk fly art me white kitchen our.,6,804.18,green,category_1,Point thought or occur term way.
+18,24,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+32,21,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+6,30,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,30,716.11,red,category_3,Only night people bill fly year.
+5,26,0,5,25,female,location_c,Talk fly art me white kitchen our.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+28,14,1,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+18,21,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+22,9,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+22,4,0,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+7,30,0,7,56,female,location_a,Seek he civil than they any.,30,716.11,red,category_3,Only night people bill fly year.
+8,11,1,8,36,male,location_b,Seek what risk particularly up today.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+30,1,1,30,44,male,location_a,Call language material science process through resource.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+13,9,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+11,18,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+24,17,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,17,364.88,yellow,category_3,Man nearly role claim quickly address until process.
+24,11,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+26,20,0,26,29,female,location_c,Goal rate never success their weight week bag.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+28,6,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,6,804.18,green,category_1,Point thought or occur term way.
+1,14,1,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+5,9,0,5,25,female,location_c,Talk fly art me white kitchen our.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+29,1,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+22,8,1,22,55,female,location_b,Mean while my receive power professor education TV camera culture true ball ready.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+9,31,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+15,25,0,15,41,male,location_a,Age provide more price kitchen wall education.,25,732.31,red,category_1,Need education place power commercial commercial business community his inside senior social.
+7,7,1,7,56,female,location_a,Seek he civil than they any.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+24,20,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+25,9,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+5,1,0,5,25,female,location_c,Talk fly art me white kitchen our.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+29,23,0,29,42,female,location_a,Open soon same fall also table agree dinner community.,23,317.87,green,category_2,Discussion throw apply his economic true with officer different.
+10,27,1,10,28,male,location_b,Return young national not right citizen environmental food these.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+14,6,0,14,57,male,location_b,About term up financial their especially newspaper computer eight.,6,804.18,green,category_1,Point thought or occur term way.
+9,8,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+23,28,1,23,19,male,location_c,While entire newspaper suggest too under senior too film similar in.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+9,16,1,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+6,16,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+2,20,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+13,26,1,13,53,male,location_b,Particularly road into actually style spring story floor that listen simple hit.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+4,16,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+25,8,0,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,8,987.02,blue,category_1,Accept final people us onto treat party a many.
+32,5,0,32,45,female,location_c,Collection project specific yourself film serve big if national resource.,5,149.51,red,category_2,None laugh national tell as over.
+24,21,0,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,21,337.59,blue,category_1,Effect feeling bar day build magazine wish money also top number.
+25,27,1,25,50,female,location_b,More former while say to behind yourself technology sometimes particular thing.,27,888.34,green,category_3,Role really window former executive life house factor player anyone reflect.
+12,31,0,12,41,female,location_b,Just side picture road available summer spend require your sound.,31,763.18,yellow,category_1,Size whom operation past place pass PM down choose affect road mean range.
+2,5,0,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,5,149.51,red,category_2,None laugh national tell as over.
+17,14,0,17,39,male,location_b,Energy voice player develop myself present.,14,731.72,yellow,category_2,American note not bag mean nation prevent.
+4,26,1,4,60,female,location_c,Like police keep place poor student beautiful message yeah scene.,26,641.18,green,category_1,Lead policy represent above us agree Mrs somebody within.
+27,28,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,28,477.49,red,category_3,Will rich western behavior gun dinner medical decide Congress stock machine computer.
+6,10,1,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+6,20,0,6,38,male,location_b,Story close policy whose kitchen time different probably both anything fine.,20,627.07,yellow,category_1,We pretty all provide Mrs opportunity mean know force item catch day left.
+14,10,1,14,57,male,location_b,About term up financial their especially newspaper computer eight.,10,206.73,yellow,category_2,Information rate simple sport as member six individual everyone quickly national listen school.
+26,5,1,26,29,female,location_c,Goal rate never success their weight week bag.,5,149.51,red,category_2,None laugh national tell as over.
+28,16,0,28,61,male,location_b,Fall scene expect good hot three little half statement little fish.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+16,6,1,16,20,male,location_c,Its must health collection attack similar see.,6,804.18,green,category_1,Point thought or occur term way.
+8,30,0,8,36,male,location_b,Seek what risk particularly up today.,30,716.11,red,category_3,Only night people bill fly year.
+20,12,0,20,61,male,location_b,Accept purpose environmental second especially perform spring else fund rather between.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+1,9,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,9,774.52,blue,category_1,Skill enough administration research Democrat energy contain ok effect six wear.
+11,32,1,11,28,female,location_b,Bed short memory none voice happen information standard factor.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+17,11,1,17,39,male,location_b,Energy voice player develop myself present.,11,15.47,red,category_2,Ahead unit state while grow happen success site region value thought center.
+9,18,0,9,40,female,location_b,Develop process drive movement clear already consumer impact write her type attack smile.,18,124.71,red,category_2,Go job step analysis leave debate everyone expert.
+2,22,1,2,46,female,location_c,During measure imagine along leave necessary eye all anything action number.,22,72.92,red,category_3,Remember lawyer student yet them structure each dream especially history.
+18,5,1,18,19,male,location_b,Task five concern building concern next mouth benefit.,5,149.51,red,category_2,None laugh national tell as over.
+5,32,1,5,25,female,location_c,Talk fly art me white kitchen our.,32,565.66,green,category_2,Out difficult cause pay nor clearly recent move beautiful usually capital western final.
+24,4,1,24,38,female,location_c,Without skin deep strong sense trouble interesting author.,4,547.27,green,category_2,Person us pick show door site data development crime tough make.
+11,24,0,11,28,female,location_b,Bed short memory none voice happen information standard factor.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+17,7,0,17,39,male,location_b,Energy voice player develop myself present.,7,83.81,green,category_2,Station hand week region reason government discover interest garden teach arrive far contain.
+15,1,0,15,41,male,location_a,Age provide more price kitchen wall education.,1,830.45,green,category_3,Simple listen less test interview success away reduce employee.
+1,16,0,1,56,female,location_c,Way him section night effort return conference career particularly word play call.,16,83.3,red,category_1,Report family raise color discussion national off establish.
+31,24,1,31,59,female,location_b,Speech whole understand attention sound simply why between actually ball ok family live.,24,331.93,green,category_1,Through finish including part receive other meet best social may.
+10,12,0,10,28,male,location_b,Return young national not right citizen environmental food these.,12,817.31,green,category_2,Prepare relate possible page keep at run fine side or day store.
+27,15,1,27,39,female,location_a,Air sort hit business fact management upon foot add consumer growth truth.,15,773.56,green,category_3,Long fact some decide economic author PM your agent.
diff --git a/tests/test_multi_model_and_mutil_data/generate_synthetic_data.py b/tests/test_multi_model_and_mutil_data/generate_synthetic_data.py
new file mode 100644
index 00000000..becdeae1
--- /dev/null
+++ b/tests/test_multi_model_and_mutil_data/generate_synthetic_data.py
@@ -0,0 +1,166 @@
+# sometimes I call this script generate_fake_data.py
+import os
+import random
+from typing import Tuple
+from pathlib import Path
+
+import cv2
+import numpy as np
+import pandas as pd
+from faker import Faker
+
+
+def generate_fake_data() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+
+    current_dir = os.path.dirname(os.path.realpath(__file__))
+
+    # Set seed for reproducibility
+    random.seed(42)
+
+    Faker.seed(42)
+
+    num_rows = 64 + 16 + 16
+
+    # Generate random categorical data
+    categories = ["category_A", "category_B", "category_C"]
+
+    cat_col = [random.choice(categories) for _ in range(num_rows)]
+
+    # Generate random numerical data
+    num_col = [np.random.rand() for _ in range(num_rows)]
+
+    # Generate random sentences
+    fake = Faker()
+    text_col1 = [fake.sentence() for _ in range(num_rows)]
+    text_col2 = [fake.sentence() for _ in range(num_rows)]
+
+    # Generate the image data
+    img_folder = "images"
+
+    img_path = "/".join([current_dir, "load_from_folder_test_data", img_folder])
+
+    if not os.path.exists(img_path):
+        os.makedirs(img_path)
+
+    for i in range(num_rows):
+        image = np.random.randint(0, 256, (16, 16, 3), dtype="uint8")
+        image_name = "image_set1_{}.png".format(i)
+        cv2.imwrite("/".join([img_path, image_name]), image)
+
+        image = np.random.randint(0, 256, (16, 16, 3), dtype="uint8")
+        image_name = "image_set2_{}.png".format(i)
+        cv2.imwrite("/".join([img_path, image_name]), image)
+
+    # Generate fake target values
+    target = [random.choice([0, 1]) for _ in range(num_rows)]
+
+    # Create DataFrame
+    data = {
+        "cat_col": cat_col,
+        "num_col": num_col,
+        "text_col1": text_col1,
+        "text_col2": text_col2,
+        "image_col1": ["image_set1_{}.png".format(i) for i in range(num_rows)],
+        "image_col2": ["image_set2_{}.png".format(i) for i in range(num_rows)],
+        "target": target,
+    }
+
+    df = pd.DataFrame(data)
+
+    save_dir = Path(current_dir) / "load_from_folder_test_data"
+
+    if not save_dir.exists():
+        save_dir.mkdir(parents=True)
+
+    train_df = df.iloc[:64]
+    val_df = df.iloc[64:80]
+    test_df = df.iloc[80:]
+
+    train_df.to_csv(save_dir / "train.csv", index=False)
+    val_df.to_csv(save_dir / "val.csv", index=False)
+    test_df.to_csv(save_dir / "test.csv", index=False)
+
+    print("Dataset and images created and saved successfully.")
+
+    return train_df, val_df, test_df
+
+
+def generate_fake_data_for_mutil_tabular_components() -> (
+    Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]
+):
+
+    current_dir = os.path.dirname(os.path.realpath(__file__))
+    save_dir = Path(current_dir) / "data_for_muti_tabular_components"
+
+    if not save_dir.exists():
+        save_dir.mkdir(parents=True)
+
+    fake = Faker()
+
+    random.seed(42)
+    np.random.seed(42)
+
+    # Create User Features DataFrame
+    user_ids = range(1, 33)
+    ages = np.random.randint(18, 65, size=32)
+    genders = np.random.choice(["male", "female"], size=32)
+    locations = np.random.choice(["location_a", "location_b", "location_c"], size=32)
+    reviews = [fake.sentence(nb_words=10) for _ in range(32)]
+
+    user_features = pd.DataFrame(
+        {
+            "id": user_ids,
+            "age": ages,
+            "gender": genders,
+            "location": locations,
+            "review": reviews,
+        }
+    )
+
+    # Create Item Features DataFrame
+    item_ids = range(1, 33)
+    prices = np.round(np.random.uniform(10, 1000, size=32), 2)
+    colors = np.random.choice(["red", "blue", "green", "yellow"], size=32)
+    categories = np.random.choice(["category_1", "category_2", "category_3"], size=32)
+    descriptions = [fake.sentence(nb_words=10) for _ in range(32)]
+
+    item_features = pd.DataFrame(
+        {
+            "id": item_ids,
+            "price": prices,
+            "color": colors,
+            "category": categories,
+            "description": descriptions,
+        }
+    )
+
+    # Create Interaction DataFrame
+    interaction_data = []
+    for _ in range(1000):  # maybe 1000 interactions is too much for a test
+        user_id = random.choice(user_ids)
+        item_id = random.choice(item_ids)
+        purchased = random.choice([0, 1])
+        interaction_data.append([user_id, item_id, purchased])
+
+    interactions = pd.DataFrame(
+        interaction_data, columns=["user_id", "item_id", "purchased"]
+    )
+
+    user_item_purchased_df = interactions.merge(
+        user_features, left_on="user_id", right_on="id"
+    ).merge(item_features, left_on="item_id", right_on="id")
+
+    train_df = user_item_purchased_df.iloc[:800]
+    val_df = user_item_purchased_df.iloc[800:900]
+    test_df = user_item_purchased_df.iloc[900:]
+
+    train_df.to_csv(save_dir / "train.csv", index=False)
+    val_df.to_csv(save_dir / "val.csv", index=False)
+    test_df.to_csv(save_dir / "test.csv", index=False)
+
+    return train_df, val_df, test_df
+
+
+if __name__ == "__main__":
+    # _, _, _ = generate_fake_data()
+    _, _, _ = generate_fake_data_for_mutil_tabular_components()
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_0.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_0.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_0.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_0.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_1.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_1.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_1.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_1.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_10.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_10.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_10.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_10.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_11.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_11.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_11.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_11.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_12.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_12.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_12.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_12.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_13.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_13.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_13.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_13.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_14.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_14.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_14.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_14.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_15.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_15.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_15.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_15.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_16.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_16.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_16.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_16.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_17.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_17.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_17.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_17.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_18.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_18.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_18.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_18.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_19.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_19.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_19.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_19.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_2.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_2.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_2.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_2.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_20.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_20.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_20.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_20.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_21.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_21.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_21.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_21.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_22.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_22.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_22.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_22.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_23.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_23.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_23.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_23.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_24.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_24.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_24.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_24.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_25.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_25.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_25.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_25.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_26.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_26.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_26.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_26.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_27.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_27.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_27.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_27.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_28.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_28.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_28.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_28.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_29.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_29.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_29.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_29.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_3.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_3.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_3.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_3.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_30.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_30.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_30.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_30.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_31.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_31.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_31.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_31.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_32.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_32.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_32.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_32.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_33.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_33.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_33.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_33.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_34.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_34.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_34.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_34.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_35.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_35.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_35.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_35.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_36.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_36.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_36.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_36.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_37.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_37.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_37.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_37.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_38.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_38.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_38.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_38.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_39.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_39.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_39.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_39.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_4.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_4.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_4.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_4.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_40.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_40.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_40.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_40.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_41.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_41.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_41.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_41.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_42.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_42.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_42.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_42.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_43.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_43.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_43.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_43.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_44.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_44.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_44.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_44.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_45.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_45.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_45.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_45.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_46.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_46.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_46.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_46.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_47.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_47.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_47.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_47.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_48.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_48.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_48.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_48.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_49.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_49.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_49.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_49.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_5.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_5.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_5.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_5.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_50.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_50.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_50.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_50.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_51.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_51.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_51.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_51.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_52.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_52.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_52.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_52.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_53.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_53.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_53.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_53.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_54.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_54.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_54.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_54.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_55.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_55.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_55.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_55.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_56.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_56.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_56.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_56.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_57.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_57.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_57.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_57.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_58.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_58.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_58.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_58.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_59.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_59.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_59.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_59.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_6.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_6.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_6.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_6.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_60.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_60.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_60.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_60.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_61.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_61.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_61.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_61.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_62.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_62.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_62.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_62.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_63.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_63.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_63.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_63.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_64.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_64.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_64.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_64.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_65.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_65.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_65.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_65.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_66.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_66.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_66.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_66.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_67.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_67.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_67.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_67.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_68.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_68.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_68.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_68.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_69.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_69.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_69.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_69.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_7.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_7.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_7.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_7.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_70.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_70.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_70.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_70.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_71.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_71.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_71.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_71.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_72.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_72.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_72.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_72.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_73.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_73.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_73.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_73.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_74.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_74.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_74.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_74.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_75.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_75.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_75.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_75.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_76.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_76.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_76.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_76.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_77.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_77.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_77.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_77.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_78.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_78.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_78.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_78.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_79.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_79.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_79.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_79.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_8.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_8.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_8.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_8.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_80.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_80.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_80.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_80.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_81.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_81.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_81.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_81.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_82.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_82.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_82.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_82.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_83.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_83.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_83.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_83.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_84.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_84.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_84.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_84.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_85.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_85.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_85.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_85.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_86.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_86.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_86.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_86.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_87.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_87.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_87.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_87.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_88.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_88.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_88.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_88.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_89.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_89.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_89.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_89.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_9.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_9.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_9.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_9.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_90.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_90.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_90.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_90.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_91.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_91.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_91.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_91.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_92.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_92.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_92.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_92.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_93.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_93.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_93.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_93.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_94.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_94.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_94.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_94.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_95.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_95.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set1_95.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set1_95.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_0.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_0.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_0.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_0.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_1.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_1.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_1.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_1.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_10.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_10.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_10.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_10.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_11.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_11.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_11.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_11.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_12.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_12.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_12.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_12.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_13.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_13.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_13.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_13.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_14.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_14.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_14.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_14.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_15.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_15.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_15.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_15.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_16.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_16.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_16.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_16.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_17.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_17.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_17.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_17.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_18.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_18.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_18.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_18.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_19.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_19.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_19.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_19.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_2.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_2.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_2.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_2.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_20.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_20.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_20.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_20.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_21.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_21.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_21.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_21.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_22.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_22.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_22.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_22.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_23.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_23.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_23.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_23.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_24.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_24.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_24.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_24.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_25.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_25.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_25.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_25.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_26.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_26.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_26.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_26.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_27.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_27.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_27.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_27.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_28.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_28.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_28.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_28.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_29.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_29.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_29.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_29.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_3.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_3.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_3.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_3.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_30.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_30.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_30.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_30.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_31.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_31.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_31.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_31.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_32.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_32.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_32.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_32.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_33.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_33.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_33.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_33.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_34.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_34.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_34.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_34.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_35.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_35.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_35.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_35.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_36.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_36.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_36.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_36.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_37.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_37.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_37.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_37.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_38.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_38.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_38.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_38.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_39.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_39.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_39.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_39.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_4.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_4.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_4.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_4.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_40.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_40.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_40.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_40.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_41.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_41.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_41.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_41.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_42.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_42.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_42.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_42.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_43.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_43.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_43.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_43.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_44.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_44.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_44.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_44.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_45.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_45.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_45.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_45.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_46.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_46.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_46.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_46.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_47.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_47.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_47.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_47.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_48.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_48.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_48.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_48.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_49.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_49.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_49.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_49.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_5.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_5.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_5.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_5.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_50.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_50.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_50.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_50.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_51.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_51.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_51.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_51.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_52.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_52.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_52.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_52.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_53.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_53.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_53.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_53.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_54.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_54.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_54.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_54.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_55.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_55.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_55.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_55.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_56.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_56.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_56.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_56.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_57.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_57.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_57.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_57.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_58.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_58.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_58.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_58.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_59.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_59.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_59.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_59.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_6.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_6.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_6.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_6.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_60.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_60.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_60.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_60.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_61.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_61.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_61.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_61.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_62.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_62.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_62.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_62.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_63.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_63.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_63.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_63.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_64.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_64.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_64.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_64.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_65.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_65.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_65.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_65.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_66.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_66.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_66.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_66.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_67.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_67.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_67.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_67.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_68.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_68.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_68.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_68.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_69.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_69.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_69.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_69.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_7.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_7.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_7.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_7.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_70.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_70.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_70.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_70.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_71.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_71.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_71.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_71.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_72.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_72.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_72.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_72.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_73.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_73.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_73.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_73.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_74.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_74.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_74.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_74.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_75.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_75.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_75.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_75.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_76.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_76.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_76.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_76.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_77.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_77.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_77.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_77.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_78.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_78.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_78.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_78.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_79.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_79.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_79.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_79.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_8.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_8.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_8.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_8.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_80.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_80.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_80.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_80.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_81.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_81.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_81.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_81.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_82.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_82.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_82.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_82.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_83.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_83.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_83.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_83.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_84.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_84.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_84.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_84.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_85.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_85.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_85.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_85.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_86.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_86.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_86.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_86.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_87.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_87.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_87.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_87.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_88.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_88.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_88.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_88.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_89.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_89.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_89.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_89.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_9.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_9.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_9.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_9.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_90.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_90.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_90.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_90.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_91.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_91.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_91.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_91.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_92.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_92.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_92.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_92.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_93.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_93.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_93.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_93.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_94.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_94.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_94.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_94.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_95.png b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_95.png
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/images/image_set2_95.png
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/images/image_set2_95.png
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/test.csv b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/test.csv
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/test.csv
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/test.csv
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/train.csv b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/train.csv
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/train.csv
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/train.csv
diff --git a/tests/test_multi_text_or_image_data/load_from_folder_test_data/val.csv b/tests/test_multi_model_and_mutil_data/load_from_folder_test_data/val.csv
similarity index 100%
rename from tests/test_multi_text_or_image_data/load_from_folder_test_data/val.csv
rename to tests/test_multi_model_and_mutil_data/load_from_folder_test_data/val.csv
diff --git a/tests/test_multi_model_and_mutil_data/test_multi_tab_and_text_components.py b/tests/test_multi_model_and_mutil_data/test_multi_tab_and_text_components.py
new file mode 100644
index 00000000..11c710e0
--- /dev/null
+++ b/tests/test_multi_model_and_mutil_data/test_multi_tab_and_text_components.py
@@ -0,0 +1,728 @@
+# NOTE FOR ME: Most of the tests here could go to
+# the 'test_multi_text_or_image_cols.py' script. However the functionality to
+# add multiple tabular model components came later and I prefer to write
+# separate tests for it. There is going to be a lot of code repetition.
+import os
+
+import numpy as np
+import torch
+import pandas as pd
+import pytest
+
+from pytorch_widedeep import Trainer
+from pytorch_widedeep.models import (
+    TabMlp,
+    TabNet,
+    BasicRNN,
+    WideDeep,
+    ModelFuser,
+)
+from pytorch_widedeep.metrics import F1Score, Accuracy
+from pytorch_widedeep.callbacks import LRHistory
+from pytorch_widedeep.initializers import XavierNormal, KaimingNormal
+from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor
+from pytorch_widedeep.models._base_wd_model_component import (
+    BaseWDModelComponent,
+)
+
+
+class CustomHead(BaseWDModelComponent):
+
+    def __init__(self, input_units: int, output_units: int):
+        super(CustomHead, self).__init__()
+        self.fc = torch.nn.Linear(input_units, output_units)
+
+    def forward(self, X: torch.Tensor) -> torch.Tensor:
+        return self.fc(X)
+
+    @property
+    def output_dim(self) -> int:
+        return self.fc.out_features
+
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+data_dir = "/".join([current_dir, "data_for_muti_tabular_components"])
+
+if not os.path.exists(data_dir):
+    raise FileNotFoundError("The data directory does not exist")
+
+train_df = pd.read_csv(data_dir + "/train.csv")
+valid_df = pd.read_csv(data_dir + "/val.csv")
+test_df = pd.read_csv(data_dir + "/test.csv")
+
+user_cols = ["age", "gender", "location"]
+item_cols = ["price", "color", "category"]
+
+tab_preprocessor_user = TabPreprocessor(
+    cat_embed_cols=["gender", "location"],
+    continuous_cols=["age"],
+    cols_to_scale=["age"],
+)
+X_tab_user_tr = tab_preprocessor_user.fit_transform(train_df)
+X_tab_user_val = tab_preprocessor_user.transform(valid_df)
+# in the real world, one would merge train and valid and refit the preprocessor
+X_tab_user_te = tab_preprocessor_user.transform(test_df)
+
+tab_preprocessor_item = TabPreprocessor(
+    cat_embed_cols=["color", "category"],
+    continuous_cols=["price"],
+    cols_to_scale=["price"],
+)
+X_tab_item_tr = tab_preprocessor_item.fit_transform(train_df)
+X_tab_item_val = tab_preprocessor_item.transform(valid_df)
+X_tab_item_te = tab_preprocessor_item.transform(test_df)
+
+text_cols = ["review", "description"]
+
+text_preprocessor_reviews = TextPreprocessor(
+    text_col="review", max_vocab=100, min_freq=2, maxlen=10, n_cpus=1, verbose=0
+)
+X_text_review_tr = text_preprocessor_reviews.fit_transform(train_df)
+X_text_review_val = text_preprocessor_reviews.transform(valid_df)
+X_text_review_te = text_preprocessor_reviews.transform(test_df)
+
+text_preprocessor_descriptions = TextPreprocessor(
+    text_col="description", max_vocab=100, min_freq=2, maxlen=10, n_cpus=1, verbose=0
+)
+X_text_description_tr = text_preprocessor_descriptions.fit_transform(train_df)
+X_text_description_val = text_preprocessor_descriptions.transform(valid_df)
+X_text_description_te = text_preprocessor_descriptions.transform(test_df)
+
+tab_mlp_user = TabMlp(
+    column_idx=tab_preprocessor_user.column_idx,
+    cat_embed_input=tab_preprocessor_user.cat_embed_input,
+    continuous_cols=tab_preprocessor_user.continuous_cols,
+    mlp_hidden_dims=[16, 4],
+)
+
+tab_mlp_item = TabMlp(
+    column_idx=tab_preprocessor_item.column_idx,
+    cat_embed_input=tab_preprocessor_item.cat_embed_input,
+    continuous_cols=tab_preprocessor_item.continuous_cols,
+    mlp_hidden_dims=[16, 4],
+)
+
+
+rnn_reviews = BasicRNN(
+    vocab_size=len(text_preprocessor_reviews.vocab.itos),
+    embed_dim=16,
+    hidden_dim=16,
+    n_layers=1,
+    bidirectional=False,
+    head_hidden_dims=[16, 8],
+)
+
+rnn_descriptions = BasicRNN(
+    vocab_size=len(text_preprocessor_descriptions.vocab.itos),
+    embed_dim=16,
+    hidden_dim=16,
+    n_layers=1,
+    bidirectional=False,
+    head_hidden_dims=[16, 4],  # just to make the head_hidden_dims different
+)
+
+global_model = WideDeep(
+    deeptabular=[tab_mlp_user, tab_mlp_item],
+    deeptext=[rnn_reviews, rnn_descriptions],
+    pred_dim=1,
+)
+
+
+@pytest.mark.parametrize(
+    "X_tab, X_text, X_train, X_val, val_split, target",
+    [
+        (
+            [X_tab_user_tr, X_tab_item_tr],
+            [X_text_review_tr, X_text_description_tr],
+            None,
+            None,
+            None,
+            train_df["purchased"].values,
+        ),
+        (
+            [X_tab_user_tr, X_tab_item_tr],
+            [X_text_review_tr, X_text_description_tr],
+            None,
+            None,
+            0.2,
+            train_df["purchased"].values,
+        ),
+        (
+            None,
+            None,
+            {
+                "X_tab": [X_tab_user_tr, X_tab_item_tr],
+                "X_text": [X_text_review_tr, X_text_description_tr],
+                "target": train_df["purchased"].values,
+            },
+            {
+                "X_tab": [X_tab_user_val, X_tab_item_val],
+                "X_text": [X_text_review_val, X_text_description_val],
+                "target": valid_df["purchased"].values,
+            },
+            None,
+            None,
+        ),
+        (
+            None,
+            None,
+            {
+                "X_tab": [X_tab_user_tr, X_tab_item_tr],
+                "X_text": [X_text_review_tr, X_text_description_tr],
+                "target": train_df["purchased"].values,
+            },
+            None,
+            0.2,
+            None,
+        ),
+    ],
+)
+def test_multi_text_and_tab_col_input_options(
+    X_tab, X_text, X_train, X_val, val_split, target
+):
+
+    trainer = Trainer(
+        global_model,
+        objective="binary",
+    )
+
+    trainer.fit(
+        X_tab=X_tab,
+        X_text=X_text,
+        X_train=X_train,
+        X_val=X_val,
+        val_split=val_split,
+        target=target,
+        n_epochs=1,
+        batch_size=32,
+    )
+
+    assert trainer.history["train_loss"] is not None
+
+
+def test_multiple_setups_for_multi_text_or_tab_cols():
+
+    model = WideDeep(
+        deeptabular=[tab_mlp_user, tab_mlp_item],
+        deeptext=[rnn_reviews, rnn_descriptions],
+        pred_dim=1,
+    )
+
+    tab_opt_user = torch.optim.Adam(model.deeptabular[0].parameters(), lr=0.01)
+    tab_opt_item = torch.optim.Adam(model.deeptabular[1].parameters(), lr=0.01)
+
+    text_opt_reviews = torch.optim.Adam(model.deeptext[0].parameters(), lr=0.01)
+    text_opt2_descriptions = torch.optim.Adam(model.deeptext[1].parameters(), lr=0.01)
+
+    tab_scheduler_user = torch.optim.lr_scheduler.StepLR(tab_opt_user, step_size=2)
+    tab_scheduler_item = torch.optim.lr_scheduler.StepLR(tab_opt_item, step_size=3)
+
+    text_scheduler_reviews = torch.optim.lr_scheduler.StepLR(
+        text_opt_reviews, step_size=2
+    )
+    text_scheduler_descriptions = torch.optim.lr_scheduler.StepLR(
+        text_opt2_descriptions, step_size=3
+    )
+
+    optimizers = {
+        "deeptabular": [tab_opt_user, tab_opt_item],
+        "deeptext": [text_opt_reviews, text_opt2_descriptions],
+    }
+    schedulers = {
+        "deeptabular": [tab_scheduler_user, tab_scheduler_item],
+        "deeptext": [text_scheduler_reviews, text_scheduler_descriptions],
+    }
+    initializers = {
+        "deeptabular": [XavierNormal, KaimingNormal],
+        "deepimage": [XavierNormal, KaimingNormal],
+    }
+
+    n_epochs = 6
+    trainer = Trainer(
+        model,
+        objective="binary",
+        optimizers=optimizers,
+        lr_schedulers=schedulers,
+        initializers=initializers,
+        metrics=[Accuracy(), F1Score(average=True)],
+        callbacks=[LRHistory(n_epochs=n_epochs)],
+    )
+
+    X_train = {
+        "X_tab": [X_tab_user_tr, X_tab_item_tr],
+        "X_text": [X_text_review_tr, X_text_description_tr],
+        "target": train_df["purchased"].values,
+    }
+    X_val = {
+        "X_tab": [X_tab_user_val, X_tab_item_val],
+        "X_text": [X_text_review_val, X_text_description_val],
+        "target": valid_df["purchased"].values,
+    }
+    trainer.fit(
+        X_train=X_train,
+        X_val=X_val,
+        n_epochs=n_epochs,
+        batch_size=4,
+        verbose=0,
+    )
+
+    assert len(trainer.history["train_loss"]) == n_epochs
+
+    deeptabular_keys = sorted(
+        [k for k in trainer.lr_history.keys() if "deeptabular" in k]
+    )
+    deeptext_keys = sorted([k for k in trainer.lr_history.keys() if "deeptext" in k])
+
+    for k, sz in zip(
+        deeptabular_keys, [tab_scheduler_user.step_size, tab_scheduler_item.step_size]
+    ):
+        n_lr_decreases = n_epochs // sz - 1 if n_epochs % sz == 0 else n_epochs // sz
+        lr_decrease_factor = 10**n_lr_decreases
+        assert len(trainer.lr_history[k]) == n_epochs
+        assert np.allclose(
+            trainer.lr_history[k][0] / trainer.lr_history[k][-1], lr_decrease_factor
+        )
+
+    for k, sz in zip(
+        deeptext_keys, [tab_scheduler_user.step_size, tab_scheduler_item.step_size]
+    ):
+        n_lr_decreases = n_epochs // sz - 1 if n_epochs % sz == 0 else n_epochs // sz
+        lr_decrease_factor = 10**n_lr_decreases
+        assert len(trainer.lr_history[k]) == n_epochs
+        assert np.allclose(
+            trainer.lr_history[k][0] / trainer.lr_history[k][-1], lr_decrease_factor
+        )
+
+
+def test_finetune_all_for_multi_text_or_tab_cols():
+
+    model = WideDeep(
+        deeptabular=[tab_mlp_user, tab_mlp_item],
+        deeptext=[rnn_reviews, rnn_descriptions],
+        pred_dim=1,
+    )
+
+    n_epochs = 5
+    trainer = Trainer(
+        model,
+        objective="binary",
+    )
+
+    X_train = {
+        "X_tab": [X_tab_user_tr, X_tab_item_tr],
+        "X_text": [X_text_review_tr, X_text_description_tr],
+        "target": train_df["purchased"].values,
+    }
+    X_val = {
+        "X_tab": [X_tab_user_val, X_tab_item_val],
+        "X_text": [X_text_review_val, X_text_description_val],
+        "target": valid_df["purchased"].values,
+    }
+    trainer.fit(
+        X_train=X_train,
+        X_val=X_val,
+        n_epochs=n_epochs,
+        batch_size=4,
+        finetune=True,
+        finetune_epochs=2,
+        verbose=0,
+    )
+
+    # weak assertion, but anyway...
+    assert len(trainer.history["train_loss"]) == n_epochs
+
+
+@pytest.mark.parametrize("routine", ["felbo", "howard"])
+def test_finetune_gradual_for_multi_text_or_tab_cols(routine):
+
+    model = WideDeep(
+        deeptabular=[tab_mlp_user, tab_mlp_item],
+        deeptext=[rnn_reviews, rnn_descriptions],
+        pred_dim=1,
+    )
+
+    deeptabular_users_layers = [
+        model.deeptabular[0][0].encoder.mlp[1],
+        model.deeptabular[0][0].encoder.mlp[0],
+    ]
+
+    deeptabular_items_layers = [
+        model.deeptabular[1][0].encoder.mlp[1],
+        model.deeptabular[1][0].encoder.mlp[0],
+    ]
+
+    deeptext_reviews_layers = [
+        model.deeptext[0][0].rnn_mlp.mlp[1],
+        model.deeptext[0][0].rnn_mlp.mlp[0],
+    ]
+    deeptext_descriptions_layers = [
+        model.deeptext[1][0].rnn_mlp.mlp[1],
+        model.deeptext[1][0].rnn_mlp.mlp[0],
+    ]
+
+    n_epochs = 5
+    trainer = Trainer(
+        model,
+        objective="binary",
+    )
+
+    X_train = {
+        "X_tab": [X_tab_user_tr, X_tab_item_tr],
+        "X_text": [X_text_review_tr, X_text_description_tr],
+        "target": train_df["purchased"].values,
+    }
+    X_val = {
+        "X_tab": [X_tab_user_val, X_tab_item_val],
+        "X_text": [X_text_review_val, X_text_description_val],
+        "target": valid_df["purchased"].values,
+    }
+    trainer.fit(
+        X_train=X_train,
+        X_val=X_val,
+        n_epochs=n_epochs,
+        batch_size=4,
+        finetune=True,
+        finetune_epochs=2,
+        routine=routine,  # add alias as finetune_routine
+        deeptabular_gradual=True,
+        deeptabular_layers=[deeptabular_users_layers, deeptabular_items_layers],
+        deeptabular_max_lr=0.01,
+        deeptext_gradual=True,
+        deeptext_layers=[deeptext_reviews_layers, deeptext_descriptions_layers],
+        deepteext_max_lr=0.01,
+        deepimage_gradual=True,
+        deepimage_max_lr=0.01,
+        verbose=0,
+    )
+
+    # weak assertion, but anyway...
+    assert len(trainer.history["train_loss"]) == n_epochs
+
+
+@pytest.mark.parametrize(
+    "fusion_method",
+    [
+        "concatenate",
+        "mean",
+        "max",
+        "sum",
+        "mult",
+        "dot",
+        "head",
+        ["max", "dot"],
+        ["concatenate", "mean"],
+        ["concatenate", "max", "mean"],
+        ["concatenate", "max", "mean", "mult"],
+    ],
+)
+def test_tab_model_fusion_methods(fusion_method):
+
+    tab_mlp_user = TabMlp(
+        column_idx=tab_preprocessor_user.column_idx,
+        cat_embed_input=tab_preprocessor_user.cat_embed_input,
+        continuous_cols=tab_preprocessor_user.continuous_cols,
+        mlp_hidden_dims=[16, 4],
+    )
+
+    tab_mlp_item = TabMlp(
+        column_idx=tab_preprocessor_item.column_idx,
+        cat_embed_input=tab_preprocessor_item.cat_embed_input,
+        continuous_cols=tab_preprocessor_item.continuous_cols,
+        mlp_hidden_dims=[16, 4],
+    )
+
+    if isinstance(fusion_method, list) and "dot" in fusion_method:
+        with pytest.raises(ValueError):
+            models_fuser = ModelFuser(
+                models=[tab_mlp_user, tab_mlp_item],
+                fusion_method=fusion_method,
+            )
+        return
+    else:
+        models_fuser = ModelFuser(
+            models=[tab_mlp_user, tab_mlp_item],
+            fusion_method=fusion_method,
+            projection_method="max",
+            head_hidden_dims=[32, 8] if "head" in fusion_method else None,
+        )
+
+    X_tab_user_tr_tnsr = torch.from_numpy(X_tab_user_tr)[:16]
+    X_tab_item_tr_tnsr = torch.from_numpy(X_tab_item_tr)[:16]
+
+    out = models_fuser([X_tab_user_tr_tnsr, X_tab_item_tr_tnsr])
+
+    if fusion_method == "dot":
+        assert out.shape[1] == 1 == models_fuser.output_dim
+    elif fusion_method == "concatenate":
+        assert (
+            out.shape[1]
+            == tab_mlp_user.output_dim + tab_mlp_item.output_dim
+            == models_fuser.output_dim
+        )
+    elif any(
+        [
+            fusion_method == "mean",
+            fusion_method == "max",
+            fusion_method == "sum",
+            fusion_method == "mult",
+        ]
+    ):
+        assert (
+            out.shape[1]
+            == max(tab_mlp_user.output_dim, tab_mlp_item.output_dim)
+            == models_fuser.output_dim
+        )
+    elif fusion_method == "head":
+        assert (
+            out.shape[1] == models_fuser.head_hidden_dims[-1] == models_fuser.output_dim
+        )
+    elif fusion_method == ["concatenate", "mean"]:
+        assert (
+            out.shape[1]
+            == tab_mlp_user.output_dim
+            + tab_mlp_item.output_dim
+            + max(tab_mlp_user.output_dim, tab_mlp_item.output_dim)
+            == models_fuser.output_dim
+        )
+    elif fusion_method == ["concatenate", "max", "mean"]:
+        assert (
+            out.shape[1]
+            == tab_mlp_user.output_dim
+            + tab_mlp_item.output_dim
+            + max(tab_mlp_user.output_dim, tab_mlp_item.output_dim) * 2
+            == models_fuser.output_dim
+        )
+    else:
+        # ["concatenate", "max", "mean", "mult"]
+        assert (
+            out.shape[1]
+            == tab_mlp_user.output_dim
+            + tab_mlp_item.output_dim
+            + max(tab_mlp_user.output_dim, tab_mlp_item.output_dim) * 3
+            == models_fuser.output_dim
+        )
+
+
+def test_model_fusion_custom_head():
+
+    tab_mlp_user = TabMlp(
+        column_idx=tab_preprocessor_user.column_idx,
+        cat_embed_input=tab_preprocessor_user.cat_embed_input,
+        continuous_cols=tab_preprocessor_user.continuous_cols,
+        mlp_hidden_dims=[16, 4],
+    )
+
+    tab_mlp_item = TabMlp(
+        column_idx=tab_preprocessor_item.column_idx,
+        cat_embed_input=tab_preprocessor_item.cat_embed_input,
+        continuous_cols=tab_preprocessor_item.continuous_cols,
+        mlp_hidden_dims=[16, 4],
+    )
+
+    custom_head = CustomHead(tab_mlp_user.output_dim + tab_mlp_item.output_dim, 8)
+
+    models_fuser = ModelFuser(
+        models=[tab_mlp_user, tab_mlp_item],
+        fusion_method="head",
+        custom_head=custom_head,
+        projection_method="max",
+    )
+
+    X_tab_user_tr_tnsr = torch.from_numpy(X_tab_user_tr)[:16]
+    X_tab_item_tr_tnsr = torch.from_numpy(X_tab_item_tr)[:16]
+    out = models_fuser([X_tab_user_tr_tnsr, X_tab_item_tr_tnsr])
+
+    assert out.shape[1] == custom_head.output_dim == models_fuser.output_dim
+
+
+@pytest.mark.parametrize(
+    "projection_method",
+    ["min", "max", "mean"],
+)
+def test_model_fusion_projection_methods(projection_method):
+
+    tab_mlp_user = TabMlp(
+        column_idx=tab_preprocessor_user.column_idx,
+        cat_embed_input=tab_preprocessor_user.cat_embed_input,
+        continuous_cols=tab_preprocessor_user.continuous_cols,
+        mlp_hidden_dims=[16, 4],
+    )
+
+    tab_mlp_item = TabMlp(
+        column_idx=tab_preprocessor_item.column_idx,
+        cat_embed_input=tab_preprocessor_item.cat_embed_input,
+        continuous_cols=tab_preprocessor_item.continuous_cols,
+        mlp_hidden_dims=[16, 4],
+    )
+
+    models_fuser = ModelFuser(
+        models=[tab_mlp_user, tab_mlp_item],
+        fusion_method="mean",
+        projection_method=projection_method,
+    )
+
+    X_tab_user_tr_tnsr = torch.from_numpy(X_tab_user_tr)[:16]
+    X_tab_item_tr_tnsr = torch.from_numpy(X_tab_item_tr)[:16]
+    out = models_fuser([X_tab_user_tr_tnsr, X_tab_item_tr_tnsr])
+
+    if projection_method == "min":
+        proj_dim = min(tab_mlp_user.output_dim, tab_mlp_item.output_dim)
+    elif projection_method == "max":
+        proj_dim = max(tab_mlp_user.output_dim, tab_mlp_item.output_dim)
+    else:
+        proj_dim = int((tab_mlp_user.output_dim + tab_mlp_item.output_dim) / 2)
+
+    assert out.shape[1] == proj_dim == models_fuser.output_dim
+
+
+@pytest.mark.parametrize("head_type", [None, "via_params", "custom"])
+def test_full_process_with_fusion(head_type):
+
+    fused_tab_model = ModelFuser(
+        models=[tab_mlp_user, tab_mlp_item],
+        fusion_method="mean",
+        projection_method="max",
+    )
+
+    fused_text_model = ModelFuser(
+        models=[rnn_reviews, rnn_descriptions],
+        fusion_method="mean",
+        projection_method="min",
+    )
+
+    if head_type == "via_params":
+        head_hidden_dims = [fused_tab_model.output_dim + fused_text_model.output_dim, 8]
+        custom_head = None
+    elif head_type == "custom":
+        head_hidden_dims = None
+        custom_head = CustomHead(
+            fused_tab_model.output_dim + fused_text_model.output_dim, 8
+        )
+    else:
+        head_hidden_dims = None
+        custom_head = None
+
+    model = WideDeep(
+        deeptabular=fused_tab_model,
+        deeptext=fused_text_model,
+        pred_dim=1,
+        head_hidden_dims=head_hidden_dims,
+        deephead=custom_head,
+    )
+
+    n_epochs = 2
+    trainer = Trainer(
+        model,
+        objective="binary",
+        verbose=0,
+    )
+
+    X_train = {
+        "X_tab": [X_tab_user_tr, X_tab_item_tr],
+        "X_text": [X_text_review_tr, X_text_description_tr],
+        "target": train_df["purchased"].values,
+    }
+    X_val = {
+        "X_tab": [X_tab_user_val, X_tab_item_val],
+        "X_text": [X_text_review_val, X_text_description_val],
+        "target": valid_df["purchased"].values,
+    }
+    trainer.fit(
+        X_train=X_train,
+        X_val=X_val,
+        n_epochs=n_epochs,
+        batch_size=4,
+    )
+
+    # weak assertion, but anyway...
+    assert len(trainer.history["train_loss"]) == n_epochs
+
+
+@pytest.mark.parametrize("head_type", [None, "via_params", "custom"])
+def test_full_process_without_fusion(head_type):
+
+    # the 4 models to be combined are tab_mlp_user, tab_mlp_item, rnn_reviews,
+    # rnn_descriptions
+    if head_type == "via_params":
+        head_hidden_dims = [
+            tab_mlp_user.output_dim
+            + tab_mlp_item.output_dim
+            + rnn_reviews.output_dim
+            + rnn_descriptions.output_dim,
+            8,
+        ]
+        custom_head = None
+    elif head_type == "custom":
+        head_hidden_dims = None
+        custom_head = CustomHead(
+            tab_mlp_user.output_dim
+            + tab_mlp_item.output_dim
+            + rnn_reviews.output_dim
+            + rnn_descriptions.output_dim,
+            8,
+        )
+    else:
+        head_hidden_dims = None
+        custom_head = None
+
+    model = WideDeep(
+        deeptabular=[tab_mlp_user, tab_mlp_item],
+        deeptext=[rnn_reviews, rnn_descriptions],
+        pred_dim=1,
+        head_hidden_dims=head_hidden_dims,
+        deephead=custom_head,
+    )
+
+    n_epochs = 2
+    trainer = Trainer(
+        model,
+        objective="binary",
+        verbose=0,
+    )
+
+    X_train = {
+        "X_tab": [X_tab_user_tr, X_tab_item_tr],
+        "X_text": [X_text_review_tr, X_text_description_tr],
+        "target": train_df["purchased"].values,
+    }
+    X_val = {
+        "X_tab": [X_tab_user_val, X_tab_item_val],
+        "X_text": [X_text_review_val, X_text_description_val],
+        "target": valid_df["purchased"].values,
+    }
+    trainer.fit(
+        X_train=X_train,
+        X_val=X_val,
+        n_epochs=n_epochs,
+        batch_size=4,
+    )
+
+    # weak assertion, but anyway...
+    assert len(trainer.history["train_loss"]) == n_epochs
+
+
+@pytest.mark.parametrize("fuse_models", [True, False])
+def test_catch_tabnet_error(fuse_models):
+
+    tabnet_user = TabNet(
+        column_idx=tab_preprocessor_user.column_idx,
+        cat_embed_input=tab_preprocessor_user.cat_embed_input,
+        continuous_cols=tab_preprocessor_user.continuous_cols,
+    )
+
+    tab_mlp_item = TabMlp(
+        column_idx=tab_preprocessor_item.column_idx,
+        cat_embed_input=tab_preprocessor_item.cat_embed_input,
+        continuous_cols=tab_preprocessor_item.continuous_cols,
+    )
+
+    if fuse_models:
+        with pytest.raises(ValueError):
+            fused_model = ModelFuser(  # noqa: F841
+                models=[tabnet_user, tab_mlp_item],
+                fusion_method="mean",
+                projection_method="max",
+            )
+    else:
+        with pytest.raises(ValueError):
+            model = WideDeep(deeptabular=[tabnet_user, tab_mlp_item])  # noqa: F841
diff --git a/tests/test_multi_text_or_image_data/test_multi_text_or_image_cols.py b/tests/test_multi_model_and_mutil_data/test_multi_text_or_image_cols.py
similarity index 90%
rename from tests/test_multi_text_or_image_data/test_multi_text_or_image_cols.py
rename to tests/test_multi_model_and_mutil_data/test_multi_text_or_image_cols.py
index ab1d5936..395f9f60 100644
--- a/tests/test_multi_text_or_image_data/test_multi_text_or_image_cols.py
+++ b/tests/test_multi_model_and_mutil_data/test_multi_text_or_image_cols.py
@@ -694,7 +694,8 @@ def test_model_fusion_projection_methods(projection_method):
     assert out.shape[1] == proj_dim == models_fuser.output_dim
 
 
-def test_model_fusion_full_process():
+@pytest.mark.parametrize("head_type", [None, "via_params", "custom"])
+def test_full_process_with_fusion(head_type):
 
     fused_text_model = ModelFuser(
         models=[rnn_1, rnn_2],
@@ -708,10 +709,94 @@ def test_model_fusion_full_process():
         projection_method="max",
     )
 
+    if head_type == "via_params":
+        head_hidden_dims = [
+            fused_text_model.output_dim,
+            fused_image_model.output_dim + tab_mlp.output_dim,
+            8,
+        ]
+        custom_head = None
+    elif head_type == "custom":
+        custom_head = CustomHead(
+            fused_text_model.output_dim
+            + fused_image_model.output_dim
+            + tab_mlp.output_dim,
+            8,
+        )
+        head_hidden_dims = None
+    else:
+        custom_head = None
+        head_hidden_dims = None
+
     model = WideDeep(
         deeptabular=tab_mlp,
         deeptext=fused_text_model,
         deepimage=fused_image_model,
+        head_hidden_dims=head_hidden_dims,
+        deephead=custom_head,
+        pred_dim=1,
+    )
+
+    n_epochs = 2
+    trainer = Trainer(
+        model,
+        objective="binary",
+        verbose=0,
+    )
+
+    X_train = {
+        "X_tab": X_tab_tr,
+        "X_text": [X_text_tr_1, X_text_tr_2],
+        "X_img": [X_img_tr_1, X_img_tr_2],
+        "target": train_df["target"].values,
+    }
+    X_val = {
+        "X_tab": X_tab_val,
+        "X_text": [X_text_val_1, X_text_val_2],
+        "X_img": [X_img_val_1, X_img_val_2],
+        "target": valid_df["target"].values,
+    }
+    trainer.fit(
+        X_train=X_train,
+        X_val=X_val,
+        n_epochs=n_epochs,
+        batch_size=4,
+    )
+
+    # weak assertion, but anyway...
+    assert len(trainer.history["train_loss"]) == n_epochs
+
+
+@pytest.mark.parametrize("head_type", [None, "via_params", "custom"])
+def test_full_process_without_fusion(head_type):
+
+    if head_type == "via_params":
+        head_hidden_dims = [
+            rnn_1.output_dim + rnn_2.output_dim,
+            vision_1.output_dim + vision_2.output_dim + tab_mlp.output_dim,
+            8,
+        ]
+        custom_head = None
+    elif head_type == "custom":
+        custom_head = CustomHead(
+            rnn_1.output_dim
+            + rnn_2.output_dim
+            + vision_1.output_dim
+            + vision_2.output_dim
+            + tab_mlp.output_dim,
+            8,
+        )
+        head_hidden_dims = None
+    else:
+        custom_head = None
+        head_hidden_dims = None
+
+    model = WideDeep(
+        deeptabular=tab_mlp,
+        deeptext=[rnn_1, rnn_2],
+        deepimage=[vision_1, vision_2],
+        head_hidden_dims=head_hidden_dims,
+        deephead=custom_head,
         pred_dim=1,
     )
 
@@ -719,6 +804,7 @@ def test_model_fusion_full_process():
     trainer = Trainer(
         model,
         objective="binary",
+        verbose=0,
     )
 
     X_train = {
@@ -738,7 +824,6 @@ def test_model_fusion_full_process():
         X_val=X_val,
         n_epochs=n_epochs,
         batch_size=4,
-        verbose=1,
     )
 
     # weak assertion, but anyway...
diff --git a/tests/test_multi_text_or_image_data/test_multi_text_or_image_cols_from_folder.py b/tests/test_multi_model_and_mutil_data/test_multi_text_or_image_cols_from_folder.py
similarity index 100%
rename from tests/test_multi_text_or_image_data/test_multi_text_or_image_cols_from_folder.py
rename to tests/test_multi_model_and_mutil_data/test_multi_text_or_image_cols_from_folder.py
diff --git a/tests/test_multi_text_or_image_data/generate_synthetic_data.py b/tests/test_multi_text_or_image_data/generate_synthetic_data.py
deleted file mode 100644
index fe3da1be..00000000
--- a/tests/test_multi_text_or_image_data/generate_synthetic_data.py
+++ /dev/null
@@ -1,89 +0,0 @@
-# sometimes I call this script generate_fake_data.py
-import os
-import random
-from typing import Tuple
-from pathlib import Path
-
-import cv2
-import numpy as np
-import pandas as pd
-from faker import Faker
-
-
-def generate_fake_data() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
-
-    current_dir = os.path.dirname(os.path.realpath(__file__))
-
-    # Set seed for reproducibility
-    random.seed(42)
-
-    Faker.seed(42)
-
-    num_rows = 64 + 16 + 16
-
-    # Generate random categorical data
-    categories = ["category_A", "category_B", "category_C"]
-
-    cat_col = [random.choice(categories) for _ in range(num_rows)]
-
-    # Generate random numerical data
-    num_col = [np.random.rand() for _ in range(num_rows)]
-
-    # Generate random sentences
-    fake = Faker()
-    text_col1 = [fake.sentence() for _ in range(num_rows)]
-    text_col2 = [fake.sentence() for _ in range(num_rows)]
-
-    # Generate the image data
-    img_folder = "images"
-
-    img_path = "/".join([current_dir, "load_from_folder_test_data", img_folder])
-
-    if not os.path.exists(img_path):
-        os.makedirs(img_path)
-
-    for i in range(num_rows):
-        image = np.random.randint(0, 256, (16, 16, 3), dtype="uint8")
-        image_name = "image_set1_{}.png".format(i)
-        cv2.imwrite("/".join([img_path, image_name]), image)
-
-        image = np.random.randint(0, 256, (16, 16, 3), dtype="uint8")
-        image_name = "image_set2_{}.png".format(i)
-        cv2.imwrite("/".join([img_path, image_name]), image)
-
-    # Generate fake target values
-    target = [random.choice([0, 1]) for _ in range(num_rows)]
-
-    # Create DataFrame
-    data = {
-        "cat_col": cat_col,
-        "num_col": num_col,
-        "text_col1": text_col1,
-        "text_col2": text_col2,
-        "image_col1": ["image_set1_{}.png".format(i) for i in range(num_rows)],
-        "image_col2": ["image_set2_{}.png".format(i) for i in range(num_rows)],
-        "target": target,
-    }
-
-    df = pd.DataFrame(data)
-
-    save_dir = Path(current_dir) / "load_from_folder_test_data"
-
-    if not save_dir.exists():
-        save_dir.mkdir(parents=True)
-
-    train_df = df.iloc[:64]
-    val_df = df.iloc[64:80]
-    test_df = df.iloc[80:]
-
-    train_df.to_csv(save_dir / "train.csv", index=False)
-    val_df.to_csv(save_dir / "val.csv", index=False)
-    test_df.to_csv(save_dir / "test.csv", index=False)
-
-    print("Dataset and images created and saved successfully.")
-
-    return train_df, val_df, test_df
-
-
-if __name__ == "__main__":
-    _, _, _ = generate_fake_data()