predict_by_hero_selection_with_neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### About\n",
    "\n",
    "This is the notebook for building PredictionByHero model using neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load and randomise data\n",
    "dataset = pd.read_csv('heroSelect.csv', index_col = 0)\n",
    "dataset = dataset.take(np.random.permutation(len(dataset)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Baseline\n",
    "\n",
    "The below codes tell us percentages of matches are won by team 1. Herem team 1 won less matches than team 2. And if we blindly choose team 2 to be the winner for all matches, this tells us the \"accuracy\" for this non-sense method and serves as the baseline. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset (173365, 273)\n",
      "train: (140425, 273) validation: (17337, 273) test: (15603, 273)\n"
     ]
    }
   ],
   "source": [
    "print('dataset', dataset.shape)\n",
    "dataset, validation = train_test_split(dataset, test_size = 0.1)\n",
    "train, test = train_test_split(dataset, test_size = 0.1)\n",
    "print('train:', train.shape, 'validation:', validation.shape, 'test:', test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79743\n",
      "0.5400282640671416\n"
     ]
    }
   ],
   "source": [
    "t1win = 0\n",
    "for idx, x in dataset['team1Win'].iteritems():\n",
    "    if(x==1.0):\n",
    "        t1win+=1\n",
    "print(t1win)\n",
    "print((173365-t1win)/173365)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow parts:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#input/output placeholders\n",
    "x_team1    = tf.placeholder(\"float\", shape=[None, 136], name='x_team1')\n",
    "x_team2 = tf.placeholder(\"float\", shape=[None, 136], name='x_team2')\n",
    "y_true = tf.placeholder(\"float\", shape=[None, 2], name='y_true')\n",
    "\n",
    "#we'll use dropout layers for regularisation which need a keep probability\n",
    "keep_prob1 = tf.placeholder(\"float\", name='keep_prob1')\n",
    "keep_prob2 = tf.placeholder(\"float\", name='keep_prob2')\n",
    "\n",
    "#there doesn't seem to be any other way to differenciate train and validation summaries for TensorBoard\n",
    "loss_name     = tf.placeholder(\"string\", name='loss_name')\n",
    "accuracy_name = tf.placeholder(\"string\", name='accuracy_name')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Weight init for fully connected layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fc_weight_bias(in_size, out_size):\n",
    "    initial_weight = tf.truncated_normal([in_size, out_size], stddev=0.2, mean=0.0)\n",
    "    initial_bias = tf.constant(0.1, shape=[out_size])\n",
    "    return tf.Variable(initial_weight), tf.Variable(initial_bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Our model structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"hero_layers_1\") as scope:\n",
    "    W_hero1, b_hero1 = fc_weight_bias(136,100)      \n",
    "    #note that team1 layer and team2 layer use the same weights and biases\n",
    "    team1_layer1 = tf.nn.relu(tf.matmul(x_team1, W_hero1) + b_hero1)\n",
    "    team2_layer1 = tf.nn.relu(tf.matmul(x_team2, W_hero1) + b_hero1)\n",
    "\n",
    "#second hero layer\n",
    "with tf.name_scope(\"hero_layers_2\") as scope:    \n",
    "    W_hero2, b_hero2 = fc_weight_bias(100,100)    \n",
    "    #again, team1 and team2 use the same weights and biases\n",
    "    team1_layer2 = tf.nn.relu(tf.matmul(team1_layer1, W_hero2) + b_hero2)\n",
    "    team2_layer2 = tf.nn.relu(tf.matmul(team2_layer1, W_hero2) + b_hero2)\n",
    "\n",
    "#now concatenate the team1 and team2 team outputs\n",
    "with tf.name_scope(\"hero_layers_concat\") as scope:\n",
    "    team1_team2_concat = tf.concat([team1_layer2, team2_layer2], 1)\n",
    "    team1_team2_drop = tf.nn.dropout(team1_team2_concat, keep_prob1)\n",
    "    h_drop1 = tf.nn.dropout(team1_team2_drop, keep_prob1)\n",
    "\n",
    "with tf.name_scope(\"hidden_layer_1\") as scope:\n",
    "    W_hidden1, b_hidden1 = fc_weight_bias(200,130)    \n",
    "    h_hidden1 = tf.nn.relu(tf.matmul(h_drop1, W_hidden1) + b_hidden1)\n",
    "    h_drop2 = tf.nn.dropout(h_hidden1, keep_prob2)\n",
    "\n",
    "with tf.name_scope(\"hidden_layer_2\") as scope:\n",
    "    W_hidden2, b_hidden2 = fc_weight_bias(130,70)    \n",
    "    h_hidden2 = tf.nn.relu(tf.matmul(h_drop2, W_hidden2) + b_hidden2)\n",
    "\n",
    "with tf.name_scope(\"hidden_layer_3\") as scope:\n",
    "    W_hidden3, b_hidden3 = fc_weight_bias(70,25)    \n",
    "    h_hidden3 = tf.nn.relu(tf.matmul(h_hidden2, W_hidden3) + b_hidden3)\n",
    "\n",
    "with tf.name_scope(\"output_layer\") as scope:\n",
    "    W_hidden4, b_hidden4 = fc_weight_bias(25,2)    \n",
    "    y_pred = tf.nn.softmax(tf.matmul(h_hidden3, W_hidden4) + b_hidden4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-9-4712cb8c56f1>:6: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with tf.name_scope(\"loss_calculations\") as scope:\n",
    "    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred)\n",
    "    loss = tf.reduce_mean(cross_entropy)\n",
    "\n",
    "with tf.name_scope(\"trainer\") as scope:\n",
    "    train_step    = tf.train.AdamOptimizer(0.0001).minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"accuracy_calculations\") as scope:\n",
    "    correct  = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DataFeed Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_feed(dataset, kp1=1.0, kp2=1.0, loss_str='loss', accuracy_str='accuracy'):\n",
    "    team1_data, team2_data = dataset.iloc[:,1:137], dataset.iloc[:,137:273]\n",
    "    winners = pd.get_dummies(dataset['team1Win'])\n",
    "    return {\n",
    "        x_team1: team1_data,\n",
    "        x_team2: team2_data,\n",
    "        y_true: winners,\n",
    "        loss_name: loss_str,\n",
    "        accuracy_name: accuracy_str,\n",
    "        keep_prob1: kp1,\n",
    "        keep_prob2: kp2\n",
    "    }  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_feed      = get_data_feed(train,      loss_str = 'loss_train',      accuracy_str = 'accuracy_train')\n",
    "validation_feed = get_data_feed(validation, loss_str = 'loss_validation', accuracy_str = 'accuracy_validation')\n",
    "test_feed       = get_data_feed(test,       loss_str = 'loss_test',       accuracy_str = 'accuracy_test')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Batch Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batches(dataset, batch_size=1700): #1700 is about 1% of the entire training sets\n",
    "    #randomise before every epoch\n",
    "    dataset = dataset.take(np.random.permutation(len(dataset)))\n",
    "    \n",
    "    i = 0\n",
    "    while i < len(dataset):\n",
    "        yield dataset[i : i + batch_size]\n",
    "        i = i + batch_size "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0, loss: 0.700056, train: 0.50929, validation: 0.51566\n",
      "epoch 1, loss: 0.695224, train: 0.509553, validation: 0.515487\n",
      "epoch 2, loss: 0.69364, train: 0.509945, validation: 0.5107\n",
      "epoch 3, loss: 0.693158, train: 0.510465, validation: 0.513641\n",
      "epoch 4, loss: 0.692967, train: 0.510899, validation: 0.514968\n",
      "epoch 5, loss: 0.692908, train: 0.511013, validation: 0.512892\n",
      "epoch 6, loss: 0.692845, train: 0.510935, validation: 0.513584\n",
      "epoch 7, loss: 0.692827, train: 0.511155, validation: 0.513353\n",
      "epoch 8, loss: 0.69281, train: 0.510742, validation: 0.51641\n",
      "epoch 9, loss: 0.692803, train: 0.511348, validation: 0.516006\n",
      "epoch 10, loss: 0.692802, train: 0.51107, validation: 0.515487\n",
      "epoch 11, loss: 0.692781, train: 0.511504, validation: 0.515891\n",
      "epoch 12, loss: 0.692792, train: 0.511027, validation: 0.513757\n",
      "epoch 13, loss: 0.692781, train: 0.511476, validation: 0.515602\n",
      "epoch 14, loss: 0.692786, train: 0.511398, validation: 0.516122\n",
      "epoch 15, loss: 0.692787, train: 0.511426, validation: 0.516295\n",
      "epoch 16, loss: 0.692775, train: 0.511191, validation: 0.515949\n",
      "epoch 17, loss: 0.692776, train: 0.510849, validation: 0.515026\n",
      "epoch 18, loss: 0.69281, train: 0.511832, validation: 0.514795\n",
      "epoch 19, loss: 0.692787, train: 0.511725, validation: 0.515891\n",
      "epoch 20, loss: 0.692762, train: 0.511625, validation: 0.516641\n",
      "epoch 21, loss: 0.692769, train: 0.511718, validation: 0.517391\n",
      "epoch 22, loss: 0.692782, train: 0.511953, validation: 0.516064\n",
      "epoch 23, loss: 0.692777, train: 0.51191, validation: 0.51716\n",
      "epoch 24, loss: 0.692785, train: 0.51196, validation: 0.515776\n",
      "epoch 25, loss: 0.692788, train: 0.51196, validation: 0.516814\n",
      "epoch 26, loss: 0.692768, train: 0.51186, validation: 0.517564\n",
      "epoch 27, loss: 0.692749, train: 0.512288, validation: 0.517044\n",
      "epoch 28, loss: 0.692782, train: 0.512416, validation: 0.517621\n",
      "epoch 29, loss: 0.692752, train: 0.512017, validation: 0.517679\n"
     ]
    }
   ],
   "source": [
    "for i in range(30):    \n",
    "    for mini_batch in get_batches(train):\n",
    "        mini_batch_feed = get_data_feed(mini_batch, 0.5, 0.5)   \n",
    "        train_step.run(feed_dict = mini_batch_feed)\n",
    "    \n",
    "    #log every epoch\n",
    "    train_loss          = loss.eval(feed_dict = train_feed)\n",
    "    validation_loss     = loss.eval(feed_dict = validation_feed)\n",
    "\n",
    "    train_accuracy      = accuracy.eval(feed_dict = train_feed)\n",
    "    validation_accuracy = accuracy.eval(feed_dict = validation_feed)\n",
    "\n",
    "    print(\"epoch %d, loss: %g, train: %g, validation: %g\"% (i, train_loss, train_accuracy, validation_accuracy)) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5134269"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy.eval(feed_dict=test_feed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusion\n",
    "As we can see, neural network can only achieve about 51% accuracy and could not passed the baseline percentage (54%)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}