demo-remote.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part I. Usage *mlpanel* from notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation\n",
    "\n",
    "In *UI*:\n",
    "\n",
    "1. create new project \"InNotebook\";\n",
    "2. run project;\n",
    "3. copy tracking server URI.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implement ML workflow functions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-16T21:17:31.460557Z",
     "start_time": "2019-06-16T21:17:29.395297Z"
    }
   },
   "outputs": [],
   "source": [
    "import joblib\n",
    "from IPython.display import display, HTML\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "import mlflow\n",
    "from mlflow import log_artifact, log_metric, log_param, log_params\n",
    "import mlflow.sklearn\n",
    "from mlflow.tracking import MlflowClient\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import confusion_matrix, f1_score, make_scorer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import train_test_split\n",
    "from typing import Dict, Text, Tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(csv: Text) -> pd.DataFrame:\n",
    "    \n",
    "    dataset = pd.read_csv(csv)\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Features engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_features(dataset: pd.DataFrame) -> pd.DataFrame:\n",
    "    \n",
    "    dataset['sepal_length_to_sepal_width'] = dataset['sepal_length'] / dataset['sepal_width']\n",
    "    dataset['petal_length_to_petal_width'] = dataset['petal_length'] / dataset['petal_width']\n",
    "\n",
    "    dataset = dataset[[\n",
    "        'sepal_length', 'sepal_width', 'petal_length', 'petal_width',\n",
    "        'sepal_length_to_sepal_width', 'petal_length_to_petal_width',\n",
    "        'species'\n",
    "    ]]\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Translate target values to labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def target_values_to_labels(dataset: pd.DataFrame, target_column: Text) -> pd.DataFrame:\n",
    "    \n",
    "    dataset[target_column] = LabelEncoder().fit_transform(dataset[target_column])\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_train_test(dataset: pd.DataFrame,\n",
    "                     test_size: float,\n",
    "                     random_state: int = 42) -> Tuple[pd.DataFrame, pd.DataFrame]:\n",
    "    \n",
    "    train_dataset, test_dataset = train_test_split(dataset, test_size=test_size, random_state=random_state)\n",
    "    \n",
    "    return train_dataset, test_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PARAM_GRIDS = {\n",
    "    \n",
    "    'LogisticRegression': {\n",
    "                'C': [0.001, 0.01],\n",
    "                'max_iter': [100, 200, 300],\n",
    "                'solver': ['lbfgs'],\n",
    "                'multi_class': ['multinomial']\n",
    "    },\n",
    "    \n",
    "    'SVC': {\n",
    "        'C': [0.1, 1.0],\n",
    "        'kernel': ['rbf', 'linear'],\n",
    "        'gamma': ['scale'],\n",
    "        'degree': [3, 5]\n",
    "    },\n",
    "    \n",
    "}\n",
    "\n",
    "ESTIMATORS = {\n",
    "        'LogisticRegression': LogisticRegression,\n",
    "        'SVC': SVC\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(estimator, param_grid: Dict, cv: int, dataset: pd.DataFrame, target_column: Text):\n",
    "    \n",
    "    X_train = dataset.drop(target_column, axis=1)\n",
    "    y_train = dataset[target_column]\n",
    "    f1_scorer = make_scorer(f1_score, average='weighted')\n",
    "    clf = GridSearchCV(estimator=estimator,\n",
    "                       param_grid=param_grid,\n",
    "                       cv=cv,\n",
    "                       verbose=1,\n",
    "                       scoring=f1_scorer,\n",
    "                       iid=True)\n",
    "\n",
    "    clf.fit(X_train, y_train)\n",
    "    \n",
    "    return clf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-16T21:21:55.875303Z",
     "start_time": "2019-06-16T21:21:55.864724Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm,\n",
    "                          target_names,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=None,\n",
    "                          normalize=True):\n",
    "    \"\"\"\n",
    "    given a sklearn confusion matrix (cm), make a nice plot\n",
    "\n",
    "    Arguments\n",
    "    ---------\n",
    "    cm:           confusion matrix from sklearn.metrics.confusion_matrix\n",
    "\n",
    "    target_names: given classification classes such as [0, 1, 2]\n",
    "                  the class names, for example: ['high', 'medium', 'low']\n",
    "\n",
    "    title:        the text to display at the top of the matrix\n",
    "\n",
    "    cmap:         the gradient of the values displayed from matplotlib.pyplot.cm\n",
    "                  see http://matplotlib.org/examples/color/colormaps_reference.html\n",
    "                  plt.get_cmap('jet') or plt.cm.Blues\n",
    "\n",
    "    normalize:    If False, plot the raw numbers\n",
    "                  If True, plot the proportions\n",
    "\n",
    "    Usage\n",
    "    -----\n",
    "    plot_confusion_matrix(cm           = cm,                  # confusion matrix created by\n",
    "                                                              # sklearn.metrics.confusion_matrix\n",
    "                          normalize    = True,                # show proportions\n",
    "                          target_names = y_labels_vals,       # list of names of the classes\n",
    "                          title        = best_estimator_name) # title of graph\n",
    "\n",
    "    Citiation\n",
    "    ---------\n",
    "    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    accuracy = np.trace(cm) / float(np.sum(cm))\n",
    "    misclass = 1 - accuracy\n",
    "\n",
    "    if cmap is None:\n",
    "        cmap = plt.get_cmap('Blues')\n",
    "\n",
    "    plt.figure(figsize=(8, 6))\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "\n",
    "    if target_names is not None:\n",
    "        tick_marks = np.arange(len(target_names))\n",
    "        plt.xticks(tick_marks, target_names, rotation=45)\n",
    "        plt.yticks(tick_marks, target_names)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "    thresh = cm.max() / 1.5 if normalize else cm.max() / 2\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        if normalize:\n",
    "            plt.text(j, i, \"{:0.4f}\".format(cm[i, j]),\n",
    "                     horizontalalignment=\"center\",\n",
    "                     color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "        else:\n",
    "            plt.text(j, i, \"{:,}\".format(cm[i, j]),\n",
    "                     horizontalalignment=\"center\",\n",
    "                     color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label\\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(clf, dataset: pd.DataFrame, target_column: Text) -> Tuple[np.ndarray, float]:\n",
    "    \n",
    "    Xtest = dataset.drop(target_column, axis=1)\n",
    "    Ytest = dataset[target_column]\n",
    "    y_pred = clf.predict(Xtest)\n",
    "\n",
    "    cm = confusion_matrix(y_pred, Ytest)\n",
    "    f1 = f1_score(y_true=Ytest, y_pred=y_pred, average='macro')\n",
    "    \n",
    "    return cm, f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_dataframe(df: pd.DataFrame, caption='DF', head_only=True):\n",
    "    \n",
    "    print(f'{caption}: ')\n",
    "    \n",
    "    print('\\tshape:', df.shape)\n",
    "    print_df = df\n",
    "    \n",
    "    if head_only:\n",
    "        print_df = df.head()\n",
    "    \n",
    "    print('\\tdataframe:')\n",
    "    display(HTML(print_df.to_html()))\n",
    "    \n",
    "    print('\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_experiment(data_folder: Text,\n",
    "                   dataset_csv: Text, \n",
    "                   target_column: Text, \n",
    "                   test_size, \n",
    "                   estimator_name: Text,\n",
    "                   cv: int,\n",
    "                   experiment_name: Text = None,\n",
    "                   tracking_uri: Text = None,\n",
    "                  ):\n",
    "    \n",
    "    if tracking_uri:\n",
    "        print(tracking_uri)\n",
    "        mlflow.set_tracking_uri(tracking_uri)\n",
    "    \n",
    "    mlflow.set_experiment(experiment_name or 'Default')\n",
    "    \n",
    "    with mlflow.start_run():\n",
    "        \n",
    "        log_param('target_column', target_column)\n",
    "        log_param('test_size', test_size)\n",
    "        log_param('estimator_name', estimator_name)\n",
    "        log_param('cv', cv)\n",
    "\n",
    "        \n",
    "        dataset = load_dataset(dataset_csv)\n",
    "        print_dataframe(dataset, 'raw dataset')\n",
    "        log_artifact(dataset_csv)\n",
    "\n",
    "        target_names = dataset[target_column].unique().tolist()\n",
    "        print(f'target names: {target_names}\\n\\n')\n",
    "        log_param('taget_names', target_names)\n",
    "\n",
    "        dataset = create_features(dataset)\n",
    "        dataset = target_values_to_labels(dataset, target_column)\n",
    "        print_dataframe(dataset, 'processed dataset')\n",
    "        processed_csv = os.path.join(data_folder, 'processed_iris.csv')\n",
    "        dataset.to_csv(processed_csv, index=False)\n",
    "        log_artifact(processed_csv)\n",
    "\n",
    "        trainset, testset = split_train_test(dataset, test_size)\n",
    "        print_dataframe(trainset, 'trainset')\n",
    "        print_dataframe(testset, 'testset')\n",
    "        train_csv = os.path.join(data_folder, 'train_iris.csv')\n",
    "        test_csv = os.path.join(data_folder, 'test_iris.csv')\n",
    "        trainset.to_csv(train_csv, index=False)\n",
    "        testset.to_csv(test_csv, index=False)\n",
    "        log_artifact(train_csv)\n",
    "        log_artifact(test_csv)\n",
    "        \n",
    "\n",
    "        estimator = ESTIMATORS[estimator_name]()\n",
    "        param_grid = PARAM_GRIDS[estimator_name]\n",
    "        log_param('param_grid', param_grid)\n",
    "\n",
    "        clf = train(estimator, param_grid, cv, trainset, target_column)\n",
    "        print(f'best estimator: {clf.best_estimator_}\\n\\n')\n",
    "        log_params(clf.best_estimator_.get_params())\n",
    "        mlflow.sklearn.log_model(clf, 'model')\n",
    "        \n",
    "        cm, f1 = evaluate(clf, testset, target_column)\n",
    "        print(f'f1 score: {f1}\\n\\n')\n",
    "        \n",
    "        log_param('cm', str(cm.tolist()))\n",
    "        log_metric('f1', f1)\n",
    "        plot_confusion_matrix(cm, target_names, normalize=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_experiment(data_folder='data', \n",
    "               dataset_csv='data/iris.csv', \n",
    "               target_column='species', \n",
    "               test_size=0.2, \n",
    "               estimator_name='LogisticRegression', \n",
    "               cv=5, \n",
    "               experiment_name='IrisLogregNotebook',\n",
    "               tracking_uri='http://35.227.124.46:5002'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_experiment(data_folder='data', \n",
    "               dataset_csv='data/iris.csv', \n",
    "               target_column='species', \n",
    "               test_size=0.5, \n",
    "               estimator_name='LogisticRegression', \n",
    "               cv=5, \n",
    "               experiment_name='IrisLogregNotebook',\n",
    "               tracking_uri='http://35.227.124.46:5002'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_experiment(data_folder='data', \n",
    "               dataset_csv='data/iris.csv', \n",
    "               target_column='species', \n",
    "               test_size=0.2, \n",
    "               estimator_name='SVC', \n",
    "               cv=5, \n",
    "               experiment_name='IrisSVCNotebook',\n",
    "               tracking_uri='http://35.227.124.46:5002'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_experiment(data_folder='data', \n",
    "               dataset_csv='data/iris.csv', \n",
    "               target_column='species', \n",
    "               test_size=0.1, \n",
    "               estimator_name='SVC', \n",
    "               cv=10, \n",
    "               experiment_name='IrisSVCNotebook',\n",
    "               tracking_uri='http://35.227.124.46:5002'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_experiment(data_folder='data', \n",
    "               dataset_csv='data/iris.csv', \n",
    "               target_column='species', \n",
    "               test_size=0.5, \n",
    "               estimator_name='SVC', \n",
    "               cv=10, \n",
    "               experiment_name='IrisSVCNotebook',\n",
    "               tracking_uri='http://35.227.124.46:5002'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part II. Usage mlpanel from code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation\n",
    "\n",
    "As in previous part in *UI*:\n",
    "\n",
    "1. create new project \"IrisProject\";\n",
    "2. run project;\n",
    "3. copy tracking server URI."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assume we have project.\n",
    "\n",
    "Go to folder IrisProject/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd IrisProject/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To manage workflow we use config - *config/pipeline_config.yml*:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(open('config/pipeline_config.yml').read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Edit config and make run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!MLFLOW_TRACKING_URI=http://35.227.124.46:5003 python run.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Repeate editing config and make runs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part III. Working with mlpanel\n",
    "\n",
    "\n",
    "## Register models and create deployments\n",
    "\n",
    "Now we have 2 projects - **InNotebook** and **IrisProject**.  \n",
    "\n",
    "***Task***: [enter ui](http://35.227.124.46:3000), register some models and create deployments.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"prediction\":\"[2, 2]\"}"
     ]
    }
   ],
   "source": [
    "!curl -X POST http://35.227.124.46:8080/deployments/3/predict -d \"data=[[1,2,3,4], [100,200,300,400]]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}