UNECE_Fasttext_Step2_HyperparameterTune_Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The following European Classification of Individual Consumption according to Purpose (ECOICOP) data used for this example is open-source, provided by Statistics Poland\n",
    "\n",
    "#### Link: https://github.com/UNECE/ML_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.gaussian_process as gp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import norm\n",
    "from scipy.optimize import minimize\n",
    "import fasttext\n",
    "import os\n",
    "import pickle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define folder locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_directory = \"C:\\\\Users\\\\Justin Evans\\\\Documents\\\\Python\\\\UNECE\\\\Poland_FastText\\\\\"\n",
    "model_directory = \"C:\\\\Users\\\\Justin Evans\\\\Documents\\\\Python\\\\UNECE\\\\Poland_FastText\\\\\"\n",
    "\n",
    "model_name = \"model.bin\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define hyperparameter tuning values, quality metric to optimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyperparameter_metric = 'accuracy' # accuracy, weighted-F1, weighted precision, weighted recall\n",
    "\n",
    "tune_vars = [[\"epochs\", 3, 10], [\"learning_rate\", 0.1, 1], [\"dimensions\", 100, 100],\n",
    "            [\"min_char_grams\", 1, 4], [\"max_char_grams\", 1, 6],\n",
    "            [\"minimum_word_count\", 0, 8], [\"word_ngrams\", 1, 5]]\n",
    "\n",
    "n_iters = \"10\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Default model parameters\n",
    "\n",
    "Quality metric options (hyperparameter_metric): accuracy, weighted_f1, weighted_precision, weighted_recall\n",
    "\n",
    "Iterations during tuning (n_iters): number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create an arguement dictionary which will be picked up in the training model (Step 3) \n",
    "\n",
    "# set up arguements\n",
    "def default_arguements():\n",
    "    # create dictionary\n",
    "    arg_dict = dict(data_directory=\"\",\n",
    "                    model_directory=\"\",\n",
    "                    model_name=\"model_1\",\n",
    "                    model_description=\"default text\",\n",
    "                    epochs=\"20\",\n",
    "                    learning_rate=\"0.7\",\n",
    "                    dimensions=\"60\",\n",
    "                    minimum_word_count=\"6\",\n",
    "                    word_ngrams=\"6\",\n",
    "                    min_char_grams=\"4\",\n",
    "                    max_char_grams=\"5\",\n",
    "\n",
    "                    hyperparameter_metric=\"accuracy\",\n",
    "                    n_iters=\"5\"\n",
    "                    )\n",
    "\n",
    "    # save and load\n",
    "    with open(\"args.txt\", \"wb\") as file:\n",
    "        pickle.dump(arg_dict, file)\n",
    "\n",
    "def update_arguements(updates):\n",
    "    with open(\"args.txt\", \"rb\") as file:\n",
    "        arg_dict = pickle.load(file)\n",
    "\n",
    "    arg_dict.update(updates)\n",
    "\n",
    "    # save and load\n",
    "    with open(\"args.txt\", \"wb\") as file:\n",
    "        pickle.dump(arg_dict, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# update arguements from above - usually done beforehand in a config file\n",
    "default_arguements() # generate default arguement\n",
    "update_arguements({\"data_directory\": data_directory, \"model_directory\": model_directory, \"n_iters\":n_iters})\n",
    "\n",
    "# load the arguements file\n",
    "with open(\"args.txt\", \"rb\") as file:\n",
    "    args = pickle.load(file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions required to hyperparameter tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_train_test(path, data):\n",
    "    # raw_data = pd.read_csv(path + data, sep=\" \", header=None)\n",
    "    with open(path + data) as f:\n",
    "        raw_data = f.readlines()\n",
    "    raw_data = [x.strip() for x in raw_data]\n",
    "\n",
    "    train, test = train_test_split(raw_data, test_size=0.3, stratify=None)\n",
    "\n",
    "    np.savetxt(args['data_directory'] + \"temp_train.txt\", train, fmt=\"%s\")\n",
    "    np.savetxt(args['data_directory'] + \"temp_test.txt\", test, fmt=\"%s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# code referenced from github.com/thuijskens/bayesian-optimization, and YanPeng Gao\n",
    "\n",
    "def get_tuning_params(bounds, hyper_params):\n",
    "    discrete = [\"min_char_grams\", \"max_char_grams\", \"word_ngrams\", \"epochs\", \"minimum_word_count\"]\n",
    "    params = []\n",
    "    for index, item in enumerate(bounds):\n",
    "        if hyper_params[index] in discrete:\n",
    "            if hyper_params[index] == \"min_char_grams\" and \"max_char_grams\" in hyper_params and hyper_params.index(\n",
    "                    \"min_char_grams\") > hyper_params.index(\"max_char_grams\"):\n",
    "                params.append(\n",
    "                    np.random.randint(low=item[0], high=min(bounds[hyper_params.index(\"max_char_grams\")][0], item[1]) + 1))\n",
    "            elif hyper_params[index] == \"max_char_grams\" and \"min_char_grams\" in hyper_params and hyper_params.index(\n",
    "                    \"max_char_grams\") > hyper_params.index(\"min_char_grams\"):\n",
    "                params.append(\n",
    "                    np.random.randint(low=max(item[0], bounds[hyper_params.index(\"min_char_grams\")][1]), high=item[1] + 1))\n",
    "            else:\n",
    "                params.append(np.random.randint(low=item[0], high=item[1] + 1))\n",
    "        else:\n",
    "            params.append(np.random.uniform(low=item[0], high=item[1]))\n",
    "\n",
    "    return np.array(params)\n",
    "\n",
    "\n",
    "def expected_improvement(x, gaussian_process, evaluated_loss, greater_is_better=False, n_params=1):\n",
    "    x_to_predict = x.reshape(-1, n_params)\n",
    "\n",
    "    mu, sigma = gaussian_process.predict(x_to_predict, return_std=True)\n",
    "\n",
    "    if greater_is_better:\n",
    "        loss_optimum = np.max(evaluated_loss)\n",
    "    else:\n",
    "        loss_optimum = np.min(evaluated_loss)\n",
    "\n",
    "    scaling_factor = (-1) ** (not greater_is_better)\n",
    "\n",
    "    # In case sigma equals zero\n",
    "    with np.errstate(divide='ignore'):\n",
    "        Z = scaling_factor * (mu - loss_optimum) / sigma\n",
    "        expected_improvement = scaling_factor * (mu - loss_optimum) * norm.cdf(Z) + sigma * norm.pdf(Z)\n",
    "        expected_improvement[sigma == 0.0] == 0.0\n",
    "\n",
    "    return -1 * expected_improvement\n",
    "\n",
    "\n",
    "def sample_next_hyperparameter(acquisition_func, gaussian_process, evaluated_loss, bounds, hyper_params,\n",
    "                               greater_is_better=False, n_restarts=25):\n",
    "    best_x = None\n",
    "    best_acquisition_value = 1\n",
    "    n_params = bounds.shape[0]\n",
    "\n",
    "    for i in range(n_restarts):\n",
    "        starting_point = get_tuning_params(bounds=bounds, hyper_params=hyper_params)\n",
    "        res = minimize(fun=acquisition_func,\n",
    "                       x0=starting_point.reshape(1, -1),\n",
    "                       bounds=bounds,\n",
    "                       method='L-BFGS-B',\n",
    "                       args=(gaussian_process, evaluated_loss, greater_is_better, n_params))\n",
    "        if res.fun < best_acquisition_value:\n",
    "            best_acquisition_value = res.fun\n",
    "            best_x = res.x\n",
    "\n",
    "    return best_x\n",
    "\n",
    "\n",
    "def bayesian_optimisation(n_iters, sample_loss, tuning_vars, n_pre_samples=5, alpha=1e-5, epsilon=1e-7, saved_df=None):\n",
    "    # tuning vars list of lists\n",
    "    # bounds, np array (matrix)\n",
    "    # hyper_params list of params\n",
    "    temp_bounds_list = []\n",
    "    hyper_params = []\n",
    "    for index, item in enumerate(tuning_vars):\n",
    "        hyper_params.append(item[0])\n",
    "        temp_bounds_list.append(np.array([item[1], item[2]]))\n",
    "\n",
    "    bounds = np.array(temp_bounds_list)\n",
    "\n",
    "    x_list = []\n",
    "    y_list = []\n",
    "\n",
    "    n_params = bounds.shape[0]\n",
    "\n",
    "    if saved_df is None:\n",
    "        for i in range(n_pre_samples):\n",
    "            params = get_tuning_params(bounds=bounds, hyper_params=hyper_params)\n",
    "            x_list.append(params)\n",
    "            y_list.append(sample_loss(params, hyper_params))\n",
    "    else:\n",
    "        numpy_matr = saved_df.values\n",
    "\n",
    "        for i in range(numpy_matr.shape[0]):\n",
    "            saved_row = numpy_matr[i][:-1].copy()\n",
    "            saved_y = numpy_matr[i][-1].copy()\n",
    "            x_list.append(saved_row)\n",
    "            y_list.append(float(saved_y))\n",
    "\n",
    "    xp = np.array(x_list)\n",
    "    yp = np.array(y_list)\n",
    "    matr = np.column_stack((xp, yp))\n",
    "    col_names = hyper_params.copy()\n",
    "    col_names.append(args[\"hyperparameter_metric\"])\n",
    "    df = pd.DataFrame(matr, columns=col_names)\n",
    "    df.to_csv(args['model_directory'] + args['model_name'] + \"_hyperparametertune_results.csv\")\n",
    "\n",
    "    kernel = gp.kernels.Matern()\n",
    "    model = gp.GaussianProcessRegressor(kernel=kernel,\n",
    "                                        alpha=alpha,\n",
    "                                        n_restarts_optimizer=10,\n",
    "                                        normalize_y=True)\n",
    "\n",
    "    for n in range(n_iters):\n",
    "\n",
    "        model.fit(xp, yp)\n",
    "\n",
    "        # Sample next hyperparameter\n",
    "        next_sample = sample_next_hyperparameter(acquisition_func=expected_improvement, gaussian_process=model,\n",
    "                                                 evaluated_loss=yp, greater_is_better=True, bounds=bounds,\n",
    "                                                 hyper_params=hyper_params, n_restarts=100)\n",
    "\n",
    "        # Duplicates will break the GP. In case of a duplicate, we will randomly sample a next query point.\n",
    "        if np.any(np.abs(next_sample - xp) <= epsilon):\n",
    "            next_sample = get_tuning_params(bounds=bounds, hyper_params=hyper_params)\n",
    "\n",
    "        # Sample loss for new set of parameters\n",
    "        cv_score = sample_loss(next_sample, hyper_params, n + 1)\n",
    "\n",
    "        # Update lists\n",
    "        x_list.append(next_sample)\n",
    "        y_list.append(cv_score)\n",
    "\n",
    "        # Update xp and yp\n",
    "        xp = np.array(x_list)\n",
    "        yp = np.array(y_list)\n",
    "\n",
    "        matr = np.column_stack((xp, yp))  # TODO fix bad practice here\n",
    "        col_names = hyper_params.copy()\n",
    "        col_names.append(args[\"hyperparameter_metric\"])\n",
    "        df = pd.DataFrame(matr, columns=col_names)\n",
    "        df.to_csv(args['model_directory'] + args['model_name'] + \"_hyperparametertune_results.csv\")\n",
    "\n",
    "    return xp, yp\n",
    "\n",
    "\n",
    "def sample_loss(points, hyperparams, n=0):\n",
    "    train_data = os.path.join(os.getenv(\"DATADIR\", \"\"), args['data_directory'] + \"temp_train.txt\")\n",
    "    #test_data = os.path.join(os.getenv(\"DATADIR\", \"\"), args['data_directory'] + \"temp_test.txt\")\n",
    "\n",
    "    # default values\n",
    "    minimum_word_count = args['minimum_word_count']\n",
    "    word_ngrams = args['word_ngrams']\n",
    "    min_char_grams = args['min_char_grams']\n",
    "    max_char_grams = args['max_char_grams']\n",
    "    learning_rate = args['learning_rate']\n",
    "    dimensions = args['dimensions']\n",
    "    epoch = args['epochs']\n",
    "\n",
    "    for i in range(len(hyperparams)):\n",
    "        key = hyperparams[i]\n",
    "        value = points[i]\n",
    "        if key == \"minimum_word_count\":\n",
    "            minimum_word_count = int(round(value))\n",
    "        elif key == \"word_ngrams\":\n",
    "            word_ngrams = int(round(value))\n",
    "        elif key == \"min_char_grams\":\n",
    "            min_char_grams = int(round(value))\n",
    "        elif key == \"max_char_grams\":\n",
    "            max_char_grams = int(round(value))\n",
    "        elif key == \"learning_rate\":\n",
    "            learning_rate = round(value, 2)\n",
    "        elif key == \"dimensions\":\n",
    "            dimensions = int(round(value))\n",
    "        elif key == \"epochs\":\n",
    "            epochs = int(round(value))\n",
    "        else:\n",
    "            raise Exception(\"Invalid Hyperparameter: \", key)\n",
    "\n",
    "    model = fasttext.train_supervised(input=train_data, minCount=minimum_word_count,\n",
    "                                      wordNgrams=word_ngrams, minn=min_char_grams, maxn=max_char_grams,\n",
    "                                      lr=learning_rate, dim=dimensions, epoch=epochs)\n",
    "\n",
    "    print(\"Iteration\", n, \"complete\")\n",
    "\n",
    "    # evaluate the model that was built\n",
    "    with open(args['data_directory'] + \"temp_test.txt\") as file:\n",
    "        df = file.readlines()\n",
    "\n",
    "    labels = []\n",
    "    pred_labels = []\n",
    "\n",
    "    for line in df:\n",
    "        label = line.split(' ', 1)[0].replace('__label__', '')\n",
    "        labels.append(label)\n",
    "        text = line.split(' ', 1)[1].strip()\n",
    "        pred = model.predict(text)\n",
    "        pred_label = pred[0][0].replace(\"__label__\",\"\")\n",
    "        pred_labels.append(pred_label)\n",
    "\n",
    "    if args[\"hyperparameter_metric\"] == \"accuracy\":\n",
    "        accuracy = accuracy_score(labels, pred_labels)\n",
    "        print(\"Accuracy: \", round(accuracy,4)*100, \"%\")\n",
    "        return accuracy\n",
    "\n",
    "    elif args[\"hyperparameter_metric\"] == \"weighted_f1\":\n",
    "        weighted_f1 = f1_score(labels, pred_labels, average=\"weighted\")\n",
    "        print(\"Weighted F1: \", round(weighted_f1, 4) * 100, \"%\")\n",
    "        return weighted_f1\n",
    "\n",
    "    elif args[\"hyperparameter_metric\"] == \"weighted_precision\":\n",
    "        weighted_precision = precision_score(labels, pred_labels, average=\"weighted\")\n",
    "        print(\"Weighted Precision: \", round(weighted_precision, 4) * 100, \"%\")\n",
    "        return weighted_precision\n",
    "\n",
    "    elif args[\"hyperparameter_metric\"] == \"weighted_recall\":\n",
    "        weighted_recall = recall_score(labels, pred_labels, average=\"weighted\")\n",
    "        print(\"Weighted Recall: \", round(weighted_recall, 4) * 100, \"%\")\n",
    "        return weighted_recall\n",
    "\n",
    "    else:\n",
    "        print(\"hyperparameter metric options: accuracy, weighted_f1, weighted_precision, weighted_recall\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_model_args():\n",
    "    df = pd.read_csv(args['model_directory'] + args['model_name'] + \"_hyperparametertune_results.csv\")\n",
    "    df_sort = df.sort_values(by=[args['hyperparameter_metric']], ascending=False)\n",
    "\n",
    "    df_sort.head(20)\n",
    "    \n",
    "    # exceptions were introduced in case a tuning var wasn't included \n",
    "    try:\n",
    "        update_arguements({\"learning_rate\": df_sort.learning_rate.iloc[0]})\n",
    "    except Exception:\n",
    "        print('learning_rate: save error')\n",
    "\n",
    "    try:\n",
    "        update_arguements({\"epochs\": df_sort.epochs.iloc[0]})\n",
    "    except Exception:\n",
    "        print('epochs: save error')\n",
    "\n",
    "    try:\n",
    "        update_arguements({\"dimensions\": df_sort.dimensions.iloc[0]})\n",
    "    except Exception:\n",
    "        print('dimensions: save error')\n",
    "\n",
    "    try:\n",
    "        update_arguements({\"minimum_word_count\": df_sort.minimum_word_count.iloc[0]})\n",
    "    except Exception:\n",
    "        print('minimum_word_count: save error')\n",
    "\n",
    "    try:\n",
    "        update_arguements({\"word_ngrams\": df_sort.word_ngrams.iloc[0]})\n",
    "    except Exception:\n",
    "        print('word_ngrams: save error')\n",
    "\n",
    "    try:\n",
    "        update_arguements({\"min_char_grams\": df_sort.min_char_grams.iloc[0]})\n",
    "    except Exception:\n",
    "        print('min_char_grams: save error')\n",
    "\n",
    "    try:\n",
    "        update_arguements({\"max_char_grams\": df_sort.max_char_grams.iloc[0]})\n",
    "    except Exception:\n",
    "        print('max_char_grams: save error')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 0 complete\n",
      "Accuracy:  56.699999999999996 %\n",
      "Iteration 0 complete\n",
      "Accuracy:  82.75 %\n",
      "Iteration 0 complete\n",
      "Accuracy:  82.02000000000001 %\n",
      "Iteration 0 complete\n",
      "Accuracy:  83.02000000000001 %\n",
      "Iteration 0 complete\n",
      "Accuracy:  67.45 %\n",
      "Iteration 1 complete\n",
      "Accuracy:  65.5 %\n",
      "Iteration 2 complete\n",
      "Accuracy:  81.43 %\n",
      "Iteration 3 complete\n",
      "Accuracy:  57.29 %\n",
      "Iteration 4 complete\n",
      "Accuracy:  85.65 %\n",
      "Iteration 5 complete\n",
      "Accuracy:  80.87 %\n",
      "Iteration 6 complete\n",
      "Accuracy:  86.28 %\n",
      "Iteration 7 complete\n",
      "Accuracy:  83.28 %\n",
      "Iteration 8 complete\n",
      "Accuracy:  85.6 %\n",
      "Iteration 9 complete\n",
      "Accuracy:  81.77 %\n",
      "Iteration 10 complete\n",
      "Accuracy:  82.99 %\n"
     ]
    }
   ],
   "source": [
    "update_arguements({\"hyperparameter_metric\": args['hyperparameter_metric']}) # decide which evaluatin metric during tuning\n",
    "create_train_test(args['data_directory'], \"train.txt\") # define the training dataset\n",
    "xp, yp = bayesian_optimisation(n_iters=int(args['n_iters']), sample_loss=sample_loss, tuning_vars=tune_vars) # run the tuning\n",
    "update_model_args() # update the args dictionary, taking the best tuning result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hyperparameter tuning done! A log is saved and the best parameters have been added to the arguement dictionary\n"
     ]
    }
   ],
   "source": [
    "print(\"Hyperparameter tuning done! A log is saved and the best parameters have been added to the arguement dictionary\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lastly, lets remove our train/test temp files\n",
    "os.remove(args['data_directory'] + \"temp_train.txt\")\n",
    "os.remove(args['data_directory'] + \"temp_test.txt\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}