visualization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dkt.load_data import ASSISTment2009\n",
    "from dkt import model as DKTModel\n",
    "from utils import BasicDKT, GaussianInputNoiseDKT\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import time\n",
    "\n",
    "\"\"\"\n",
    "Assignable variables:\n",
    "num_runs: int\n",
    "num_epochs: int\n",
    "keep_prob: float\n",
    "is_early_stopping: boolean\n",
    "early_stopping: int\n",
    "batch_size: int\n",
    "hidden_layer_structure: tuple\n",
    "data_dir: str\n",
    "train_file_name: str\n",
    "test_file_name: str\n",
    "ckpt_save_dir: str\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "DATA_DIR = './data/'\n",
    "train_file = 'skill_id_train.csv'\n",
    "test_file = 'skill_id_test.csv'\n",
    "train_path = os.path.join(DATA_DIR, train_file)\n",
    "test_path = os.path.join(DATA_DIR, test_file)\n",
    "\n",
    "network_config = {\n",
    "    'batch_size': 32,\n",
    "    'hidden_layer_structure': (200,),\n",
    "    'rnn_cell': tf.contrib.rnn.LSTMCell,\n",
    "    'learning_rate': 0.01,\n",
    "    'keep_prob': 0.5,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading ./data/skill_id_train.csv\n",
      "10119 lines was read\n",
      "max_num_problems_answered: 1219\n",
      "num_problems: 124\n",
      "The number of students is 3137\n",
      "Finish reading data.\n",
      "Reading ./data/skill_id_test.csv\n",
      "2532 lines was read\n",
      "max_num_problems_answered: 1114\n",
      "num_problems: 124\n",
      "The number of students is 784\n",
      "Finish reading data.\n"
     ]
    }
   ],
   "source": [
    "data = ASSISTment2009(train_path, test_path, batch_size=32)\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "\n",
    "model_name = 'original_gaussian'\n",
    "save_dir = './checkpoints/' + model_name + '/'\n",
    "# initialize model\n",
    "dkt = BasicDKT(sess=sess,\n",
    "               data=data,\n",
    "               network_config=network_config,\n",
    "               num_epochs=1000,\n",
    "               num_runs=5,\n",
    "               save_dir=save_dir)\n",
    "\n",
    "# run optimization of the created model\n",
    "dkt.model.build_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./checkpoints/original_gaussian/model\n"
     ]
    }
   ],
   "source": [
    "dkt.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.88901371909681326, 0.38593635775826202)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dkt.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Obtain the hidden layer output\n",
    "As the hidden layer size is large, the visualization is a bit convoluted to be understanded even if we visualize it.\n",
    "In order to better visualize the hidden layer result. A rough idea is to extract all the student hidden layer output, and then perform PCA over those vector. Afterwards, check the proportion of variance and its eigen value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dkt.trainest_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_pca_model(data, n_components):\n",
    "    pca = PCA(n_components=50)\n",
    "    pca.fit(data)\n",
    "    pca_dim = len(pca.explained_variance_ratio_)\n",
    "    print(pca.explained_variance_ratio_)\n",
    "    print(\"PoV:\", sum(pca.explained_variance_ratio_))\n",
    "    return pca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "print(\"Loading the saved variable to the current session.\")\n",
    "saver.restore(sess=sess, save_path=save_path)\n",
    "\n",
    "auc_test, loss_test = evaluate(sess, is_train=False)\n",
    "print (\"auc_test: {0:.5}, loss_test: {0:.5}\".format(auc_test, loss_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hl1_outputs = []\n",
    "for student in students_train:\n",
    "    # student basic information\n",
    "    num_question_answered = student[0]\n",
    "    \n",
    "    # it is the hidden layer output sequence of the student. (in shape [max_num_steps, hl1_size])\n",
    "    hl1 = get_student_hidden_layer(sess, student=student, layer_num=1)\n",
    "    hl1 = hl1[:num_question_answered]\n",
    "    \n",
    "    hl1_outputs += [hl1_output for hl1_output in hl1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save the pca\n",
    "import pickle\n",
    "with open('data/original_1hl_hl1_newfile_outputs.pkl', 'wb') as f:\n",
    "    pickle.dump(hl1_outputs, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization\n",
    "In the following, the student output and hidden layer will be visualized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "print(\"Loading the saved variable to the current session.\")\n",
    "saver.restore(sess=sess, save_path=save_path)\n",
    "\n",
    "auc_test, loss_test = evaluate(sess, is_train=False)\n",
    "print (\"auc_test: {0:.5}, loss_test: {0:.5}\".format(auc_test, loss_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.trainable_variables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "%matplotlib inline\n",
    "#http://bokeh.pydata.org/en/0.10.0/docs/gallery/cat_heatmap_chart.html\n",
    "\n",
    "def plot_heatmap(data, x_labels, y_labels, second_x_labels=None, fig_size_inches=[15, 5], cmap=plt.cm.Blues):\n",
    "#     plt.figure(figsize=(40,100))\n",
    "\n",
    "    fig, ax = plt.subplots()\n",
    "    heatmap = ax.pcolor(data, cmap=cmap)\n",
    "    \n",
    "    # Format\n",
    "    fig = plt.gcf()\n",
    "    \n",
    "    # turn off the frame\n",
    "    ax.set_frame_on(False)\n",
    "    \n",
    "    # put the major ticks at the middle of each cell\n",
    "    ax.set_xticks(np.arange(len(x_labels)) + 0.5, minor=False)\n",
    "    ax.set_yticks(np.arange(len(y_labels)) + 0.5, minor=False)\n",
    "    \n",
    "    # want a more natural, table-like display\n",
    "    ax.invert_yaxis()\n",
    "    ax.xaxis.tick_top()\n",
    "    \n",
    "    # set the label\n",
    "    ax.set_xticklabels(x_labels, minor=False)\n",
    "    ax.set_yticklabels(y_labels, minor=False)\n",
    "    ax.set_xlabel(\"the skill id answered at the time step\")\n",
    "    ax.set_ylabel(\"the skill id of the output layer\")\n",
    "\n",
    "    fig.set_size_inches(fig_size_inches[0], fig_size_inches[1])\n",
    "    \n",
    "    # second axis label\n",
    "    if second_x_labels != None:\n",
    "        ax2 = ax.twiny()\n",
    "        ax2.set_xticks(np.arange(len(second_x_labels)) + 0.5, minor=False)\n",
    "        ax2.set_xticklabels(second_x_labels)\n",
    "        ax2.set_xlabel(\"Correct Label\")\n",
    "        ax2.xaxis.tick_top()\n",
    "\n",
    "    # Turn off all the ticks\n",
    "    ax = plt.gca()\n",
    "    \n",
    "    \n",
    "#     fig.colorbar(heatmap, fraction=0.02, pad=0.04)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = []\n",
    "for i in range(len(students_test)):\n",
    "    student = students_test[i]\n",
    "    num_question_answered = student[0]\n",
    "    question_ids_answered = np.sort(np.array([int(qid) for qid in set(student[1]) if qid != -1]))\n",
    "    num_distict_question = len(question_ids_answered)\n",
    "    \n",
    "    if 50 >= num_question_answered >= 30 and 10 >= num_distict_question >= 5:\n",
    "        targets.append(i)\n",
    "    \n",
    "print(targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# selecting one student to visualize\n",
    "# bad example: 598\n",
    "# good example: 30, 738\n",
    "sid = 126\n",
    "student = students_test[sid]\n",
    "num_question_answered = student[0]\n",
    "question_ids_answered = np.sort(np.array([int(qid) for qid in set(student[1]) if qid != -1]))\n",
    "\n",
    "question_seq = student[1][:num_question_answered]\n",
    "correct_seq = student[2][:num_question_answered]\n",
    "\n",
    "print(num_question_answered)\n",
    "print(question_seq)\n",
    "print(correct_seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# modify the student-126 to assume that he answer the question-82 correctly for 10 times\n",
    "a= student[0] + 10\n",
    "b = student[1][:40] + ['45']*10 + [0]*(1218-40-10+1)\n",
    "c = student[2][:40] + ['1']*10 + [-1]*(1218-40-10+1)\n",
    "student = (a, b, c)\n",
    "num_question_answered = student[0]\n",
    "question_ids_answered = np.sort(np.array([int(qid) for qid in set(student[1]) if qid != -1]))\n",
    "\n",
    "question_seq = student[1][:num_question_answered]\n",
    "correct_seq = student[2][:num_question_answered]\n",
    "\n",
    "print(num_question_answered)\n",
    "print(question_seq)\n",
    "print(correct_seq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualizing the Output Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "output_layer = get_student_output_layer(sess, student)\n",
    "\n",
    "output_layer = output_layer[:num_question_answered, question_ids_answered]\n",
    "output_layer = np.transpose(output_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_heatmap(output_layer, x_labels=question_seq, y_labels=question_ids_answered, second_x_labels=correct_seq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualizing the Hidden Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the pca\n",
    "import pickle\n",
    "with open('data/original_1hl_hl1_newfile_outputs.pkl', 'rb') as f:\n",
    "    hl1_outputs = pickle.load(f)\n",
    "    pca = get_pca_model(hl1_outputs, n_components=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hl1 = get_student_hidden_layer(sess, student=student, layer_num=1)\n",
    "hl1 = hl1[:num_question_answered]\n",
    "hl1_orginal = np.transpose(hl1)\n",
    "print(hl1_orginal.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hl1_pca = pca.transform(hl1)\n",
    "hl1_pac = np.transpose(hl1_pca)\n",
    "print(hl1_pac.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_labels=[\"{}({})\".format(question_seq[i], correct_seq[i]) for i in range(num_question_answered)]\n",
    "print(x_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# red: negative, white: zero, blue: positive\n",
    "plot_heatmap(hl1_orginal, \n",
    "             x_labels=x_labels, \n",
    "             y_labels=range(hl1_orginal.shape[0]),\n",
    "#              second_x_labels=correct_seq, \n",
    "             fig_size_inches=[20, 15],\n",
    "            cmap='RdBu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# red: negative, white: zero, blue: positive\n",
    "plot_heatmap(hl1_pac, \n",
    "             x_labels=x_labels, \n",
    "             y_labels=range(hl1_pac.shape[0]),\n",
    "#              second_x_labels=correct_seq, \n",
    "             fig_size_inches=[15, 15],\n",
    "            cmap='RdBu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}