toxic_comments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# w266: Final Project\n",
    "### Christopher Danicic, Robert Deng, Chandan Gope\n",
    "https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load libraries\n",
    "import os, sys, re, json, time\n",
    "import itertools, collections\n",
    "from importlib import reload\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('yourselfgo', 199), ('wrong', 198), ('world', 197), ('work', 196), ('wo', 195), ('wikipedia', 194), ('wiki', 193), ('way', 192), ('warning', 191), ('want', 190), ('wanker', 189), ('ve', 188), ('vandalism', 187), ('users', 186), ('user', 185), ('useless', 184), ('use', 183), ('ur', 182), ('trying', 181), ('try', 180), ('time', 179), ('think', 178), ('things', 177), ('thing', 176), ('thanks', 175), ('tell', 174), ('talk', 173), ('super', 172), ('sucks', 171), ('suck', 170), ('stupid', 169), ('stop', 168), ('small', 167), ('site', 166), ('shut', 165), ('shit', 164), ('sexual', 163), ('sexsex', 162), ('sex', 161), ('say', 160), ('said', 159), ('rules', 158), ('romney', 157), ('right', 156), ('retarded', 155), ('reason', 154), ('really', 153), ('real', 152), ('read', 151), ('rape', 150), ('racist', 149), ('pussy', 148), ('pro', 147), ('post', 146), ('point', 145), ('place', 144), ('piece', 143), ('personal', 142), ('person', 141), ('people', 140), ('penis', 139), ('pathetic', 138), ('pages', 137), ('page', 136), ('oxymoron', 135), ('org', 134), ('old', 133), ('oh', 132), ('offfuck', 131), ('num', 130), ('noobs', 129), ('nigger', 128), ('niggas', 127), ('new', 126), ('need', 125), ('nazi', 124), ('mothjer', 123), ('mother', 122), ('moron', 121), ('mitt', 120), ('mean', 119), ('maybe', 118), ('marcolfuck', 117), ('man', 116), ('making', 115), ('make', 114), ('love', 113), ('loser', 112), ('look', 111), ('lol', 110), ('ll', 109), ('little', 108), ('like', 107), ('life', 106), ('lick', 105), ('let', 104), ('leave', 103), ('know', 102), ('kill', 101), ('just', 100), ('jew', 99), ('ip', 98), ('information', 97), ('im', 96), ('idiot', 95), ('huge', 94), ('hope', 93), ('homo', 92), ('hitler', 91), ('history', 90), ('hi', 89), ('hey', 88), ('hell', 87), ('heil', 86), ('hate', 85), ('ha', 84), ('guy', 83), ('got', 82), ('good', 81), ('going', 80), ('god', 79), ('gay', 78), ('fucksex', 77), ('fucking', 76), ('fucker', 75), ('fuck', 74), ('freedom', 73), ('fool', 72), ('fggt', 71), ('fat', 70), ('fart', 69), ('fan', 68), ('faggots', 67), ('faggot', 66), ('fag', 65), ('fact', 64), ('en', 63), ('edits', 62), ('editing', 61), ('edit', 60), ('eat', 59), ('dumb', 58), ('dont', 57), ('doing', 56), ('dog', 55), ('does', 54), ('die', 53), ('did', 52), ('dickhead', 51), ('dick', 50), ('deleting', 49), ('delete', 48), ('day', 47), ('damn', 46), ('cunt', 45), ('criminalwar', 44), ('crap', 43), ('comment', 42), ('come', 41), ('cocksucking', 40), ('cocksucker', 39), ('cock', 38), ('china', 37), ('chester', 36), ('change', 35), ('care', 34), ('ca', 33), ('buttsecks', 32), ('bush', 31), ('bunksteve', 30), ('bum', 29), ('bullshit', 28), ('bot', 27), ('bollocks', 26), ('blocked', 25), ('block', 24), ('bitches', 23), ('bitch', 22), ('big', 21), ('better', 20), ('bastered', 19), ('bastard', 18), ('bark', 17), ('banned', 16), ('ban', 15), ('balls', 14), ('bad', 13), ('attack', 12), ('asshole', 11), ('assad', 10), ('ass', 9), ('articles', 8), ('article', 7), ('anal', 6), ('aids', 5), ('admins', 4), ('administrator', 3), ('admin', 2), ('actually', 1), ('account', 0)])\n"
     ]
    }
   ],
   "source": [
    "positive_examples = list(open('./data/toxic_yes_train.txt', \"r\").readlines())\n",
    "vectorizer = TfidfVectorizer(stop_words='english', max_features=200, ngram_range=(1,1), analyzer='word')\n",
    "vectorizer.fit(positive_examples)\n",
    "top_words_dict = vectorizer.vocabulary_\n",
    "top_words_dict = OrderedDict(sorted(top_words_dict.items(), key=lambda t: t[1], reverse=True))\n",
    "print(top_words_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_comment(in_comment, top_words_dict, max_length, neighborhood=2):\n",
    "    in_comment = re.sub(r\"[^A-Za-z0-9]\", \" \", in_comment)\n",
    "    in_comment = re.sub(\" \\d+\", \" NUM\", in_comment)\n",
    "    in_comment = re.sub(r\"\\s{2,}\", \" \", in_comment)\n",
    "    in_words = in_comment.split()\n",
    "    print(in_words)\n",
    "    if len(in_words) <= max_length:\n",
    "        return in_comment.strip().lower()\n",
    "    else:\n",
    "        triggers = []\n",
    "        for i,k in enumerate(top_words_dict.keys()):\n",
    "            if k in in_words:\n",
    "                index = in_words.index(k)\n",
    "                min_index = max(0, index-neighborhood)\n",
    "                max_index = min(len(in_words), index+neighborhood)\n",
    "                phrase = [w for w in in_words[min_index:max_index+1]]\n",
    "                triggers.append((min_index, max_index, phrase))\n",
    "        sorted_triggers = sorted(triggers, key=lambda t: t[0] , reverse=False)\n",
    "        triggers_count = 0\n",
    "        for c in sorted_triggers:\n",
    "            triggers_count += len(c[2])\n",
    "        nonTriggerRoom = max_length - triggers_count\n",
    "        out_comment = []\n",
    "        prev_end = 0\n",
    "        for c in sorted_triggers:\n",
    "            start = c[0]\n",
    "            end = c[1]\n",
    "            if prev_end < start and nonTriggerRoom > 0:\n",
    "                end_tmp = min(start, prev_end+nonTriggerRoom)\n",
    "                out_comment.extend(in_words[prev_end : end_tmp])\n",
    "                nonTriggerRoom -= end_tmp - prev_end\n",
    "            start = max(start, prev_end)\n",
    "            out_comment.extend(in_words[start:end])\n",
    "            prev_end = end\n",
    "            \n",
    "        out_comment = out_comment[:max_length]\n",
    "    \n",
    "    return ' '.join(out_comment).strip().lower()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I', 'love', 'WALKING', 'and', 'running', 'and', 'Hey', 'ass', 'there', 'and', 'yoga', 'and', 'books', 'and', 'sleeping', 'how', 'are', 'ya', 'are', 'you', 'suck', 'today', 'I', 'hate', 'you', 'today', 'is', 'Sunday']\n",
      "I love WALKING and running and Hey ass there and yoga are you suck today I hate you\n"
     ]
    }
   ],
   "source": [
    "in_comment = \"    I love WALKING, and running, and .... Hey ass   ^&%$% !!!    there and yoga, and books, and sleeping how are ya, are you suck today. I hate you, today is Sunday\"\n",
    "max_length = 22\n",
    "out_comment = preprocess_comment(in_comment, top_words_dict, max_length, neighborhood=2)\n",
    "print(out_comment)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded train.csv: Num rows = 159571, Num cols = 8\n",
      "Loaded test.csv: Num rows = 153164, Num cols = 2\n",
      "Train data column names: ['id' 'comment_text' 'toxic' 'severe_toxic' 'obscene' 'threat' 'insult'\n",
      " 'identity_hate']\n",
      "Test data column names: ['id' 'comment_text']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000997932d777bf</td>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000103f0d9cfb60f</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000113f07ec002fd</td>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0001b41b1c6bb37e</td>\n",
       "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0001d958c54c6e35</td>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>00025465d4725e87</td>\n",
       "      <td>\"\\n\\nCongratulations from me as well, use the ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0002bcb3da6cb337</td>\n",
       "      <td>COCKSUCKER BEFORE YOU PISS AROUND ON MY WORK</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>00031b1e95af7921</td>\n",
       "      <td>Your vandalism to the Matt Shirvington article...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>00037261f536c51d</td>\n",
       "      <td>Sorry if the word 'nonsense' was offensive to ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>00040093b2687caa</td>\n",
       "      <td>alignment on this subject and which are contra...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0005300084f90edc</td>\n",
       "      <td>\"\\nFair use rationale for Image:Wonju.jpg\\n\\nT...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>00054a5e18b50dd4</td>\n",
       "      <td>bbq \\n\\nbe a man and lets discuss it-maybe ove...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0005c987bdfc9d4b</td>\n",
       "      <td>Hey... what is it..\\n@ | talk .\\nWhat is it......</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0006f16e4e9f292e</td>\n",
       "      <td>Before you start throwing accusations and warn...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>00070ef96486d6f9</td>\n",
       "      <td>Oh, and the girl above started her arguments w...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>00078f8ce7eb276d</td>\n",
       "      <td>\"\\n\\nJuelz Santanas Age\\n\\nIn 2002, Juelz Sant...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0007e25b2121310b</td>\n",
       "      <td>Bye! \\n\\nDon't look, come or think of comming ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>000897889268bc93</td>\n",
       "      <td>REDIRECT Talk:Voydan Pop Georgiev- Chernodrinski</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0009801bd85e5806</td>\n",
       "      <td>The Mitsurugi point made no sense - why not ar...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0009eaea3325de8c</td>\n",
       "      <td>Don't mean to bother you \\n\\nI see that you're...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id                                       comment_text  \\\n",
       "0   0000997932d777bf  Explanation\\nWhy the edits made under my usern...   \n",
       "1   000103f0d9cfb60f  D'aww! He matches this background colour I'm s...   \n",
       "2   000113f07ec002fd  Hey man, I'm really not trying to edit war. It...   \n",
       "3   0001b41b1c6bb37e  \"\\nMore\\nI can't make any real suggestions on ...   \n",
       "4   0001d958c54c6e35  You, sir, are my hero. Any chance you remember...   \n",
       "5   00025465d4725e87  \"\\n\\nCongratulations from me as well, use the ...   \n",
       "6   0002bcb3da6cb337       COCKSUCKER BEFORE YOU PISS AROUND ON MY WORK   \n",
       "7   00031b1e95af7921  Your vandalism to the Matt Shirvington article...   \n",
       "8   00037261f536c51d  Sorry if the word 'nonsense' was offensive to ...   \n",
       "9   00040093b2687caa  alignment on this subject and which are contra...   \n",
       "10  0005300084f90edc  \"\\nFair use rationale for Image:Wonju.jpg\\n\\nT...   \n",
       "11  00054a5e18b50dd4  bbq \\n\\nbe a man and lets discuss it-maybe ove...   \n",
       "12  0005c987bdfc9d4b  Hey... what is it..\\n@ | talk .\\nWhat is it......   \n",
       "13  0006f16e4e9f292e  Before you start throwing accusations and warn...   \n",
       "14  00070ef96486d6f9  Oh, and the girl above started her arguments w...   \n",
       "15  00078f8ce7eb276d  \"\\n\\nJuelz Santanas Age\\n\\nIn 2002, Juelz Sant...   \n",
       "16  0007e25b2121310b  Bye! \\n\\nDon't look, come or think of comming ...   \n",
       "17  000897889268bc93   REDIRECT Talk:Voydan Pop Georgiev- Chernodrinski   \n",
       "18  0009801bd85e5806  The Mitsurugi point made no sense - why not ar...   \n",
       "19  0009eaea3325de8c  Don't mean to bother you \\n\\nI see that you're...   \n",
       "\n",
       "    toxic  severe_toxic  obscene  threat  insult  identity_hate  \n",
       "0       0             0        0       0       0              0  \n",
       "1       0             0        0       0       0              0  \n",
       "2       0             0        0       0       0              0  \n",
       "3       0             0        0       0       0              0  \n",
       "4       0             0        0       0       0              0  \n",
       "5       0             0        0       0       0              0  \n",
       "6       1             1        1       0       1              0  \n",
       "7       0             0        0       0       0              0  \n",
       "8       0             0        0       0       0              0  \n",
       "9       0             0        0       0       0              0  \n",
       "10      0             0        0       0       0              0  \n",
       "11      0             0        0       0       0              0  \n",
       "12      1             0        0       0       0              0  \n",
       "13      0             0        0       0       0              0  \n",
       "14      0             0        0       0       0              0  \n",
       "15      0             0        0       0       0              0  \n",
       "16      1             0        0       0       0              0  \n",
       "17      0             0        0       0       0              0  \n",
       "18      0             0        0       0       0              0  \n",
       "19      0             0        0       0       0              0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Load data\n",
    "import data_utils; reload(data_utils)\n",
    "\n",
    "data_loader = data_utils.DataLoader(path=\"./data\")\n",
    "train, test = data_loader.readTrainTest(verbose=True)\n",
    "\n",
    "#Print some train data\n",
    "train.head(n=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic data exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"bf318918-83fe-48c7-b5d8-45180c28b72c\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id !== undefined) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var element_id = msg.content.text.trim();\n",
       "            Bokeh.index[element_id].model.document.clear();\n",
       "            delete Bokeh.index[element_id];\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[0].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[0].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[0]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"bf318918-83fe-48c7-b5d8-45180c28b72c\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "    }\n",
       "    finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.info(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(js_urls, callback) {\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = js_urls.length;\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var s = document.createElement('script');\n",
       "      s.src = url;\n",
       "      s.async = false;\n",
       "      s.onreadystatechange = s.onload = function() {\n",
       "        root._bokeh_is_loading--;\n",
       "        if (root._bokeh_is_loading === 0) {\n",
       "          console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "          run_callbacks()\n",
       "        }\n",
       "      };\n",
       "      s.onerror = function() {\n",
       "        console.warn(\"failed to load library \" + url);\n",
       "      };\n",
       "      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "  };var element = document.getElementById(\"bf318918-83fe-48c7-b5d8-45180c28b72c\");\n",
       "  if (element == null) {\n",
       "    console.log(\"Bokeh: ERROR: autoload.js configured with elementid 'bf318918-83fe-48c7-b5d8-45180c28b72c' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.12.14.min.js\"];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    function(Bokeh) {\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.css\");\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((root.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i].call(root, root.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"bf318918-83fe-48c7-b5d8-45180c28b72c\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(js_urls, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"bf318918-83fe-48c7-b5d8-45180c28b72c\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n    }\n    finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.info(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(js_urls, callback) {\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = js_urls.length;\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var s = document.createElement('script');\n      s.src = url;\n      s.async = false;\n      s.onreadystatechange = s.onload = function() {\n        root._bokeh_is_loading--;\n        if (root._bokeh_is_loading === 0) {\n          console.log(\"Bokeh: all BokehJS libraries loaded\");\n          run_callbacks()\n        }\n      };\n      s.onerror = function() {\n        console.warn(\"failed to load library \" + url);\n      };\n      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.getElementsByTagName(\"head\")[0].appendChild(s);\n    }\n  };var element = document.getElementById(\"bf318918-83fe-48c7-b5d8-45180c28b72c\");\n  if (element == null) {\n    console.log(\"Bokeh: ERROR: autoload.js configured with elementid 'bf318918-83fe-48c7-b5d8-45180c28b72c' but no matching script tag was found. \")\n    return false;\n  }\n\n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.12.14.min.js\"];\n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    \n    function(Bokeh) {\n      \n    },\n    function(Bokeh) {\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.14.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.14.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.14.min.css\");\n    }\n  ];\n\n  function run_inline_js() {\n    \n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"bf318918-83fe-48c7-b5d8-45180c28b72c\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(js_urls, function() {\n      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Bokeh for plotting.\n",
    "import bokeh.plotting as bp\n",
    "from bokeh.models import HoverTool\n",
    "bp.output_notebook()\n",
    "\n",
    "# Helper code for plotting histograms\n",
    "def plot_length_histogram(lengths, x_range=[0,100], bins=40, normed=True):\n",
    "    hist, bin_edges = np.histogram(a=lengths, bins=bins, normed=normed, range=x_range)\n",
    "    bin_centers = (bin_edges[1:] + bin_edges[:-1])/2\n",
    "    bin_widths =  (bin_edges[1:] - bin_edges[:-1])\n",
    "\n",
    "    hover = HoverTool(tooltips=[(\"bucket\", \"@x\"), (\"count\", \"@top\")], mode=\"vline\")\n",
    "    fig = bp.figure(plot_width=800, plot_height=400, tools=[hover])\n",
    "    fig.vbar(x=bin_centers, width=bin_widths, top=hist, hover_fill_color=\"firebrick\")\n",
    "    fig.y_range.start = 0\n",
    "    fig.x_range.start = 0\n",
    "    fig.xaxis.axis_label = \"Number of tokens\"\n",
    "    fig.yaxis.axis_label = \"Frequency\"\n",
    "    bp.show(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max length = 1411\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<div class=\"bk-root\">\n",
       "    <div class=\"bk-plotdiv\" id=\"12c92110-a8df-4173-95fc-9fc161ef40f7\"></div>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"d78f0aeb-b589-4220-9bfb-51d890af09d1\":{\"roots\":{\"references\":[{\"attributes\":{\"callback\":null,\"start\":0},\"id\":\"1eaa8e0e-2c1d-439f-8f25-03519369b17b\",\"type\":\"DataRange1d\"},{\"attributes\":{\"source\":{\"id\":\"f10aa059-be36-423c-ac04-63fec9c06f8d\",\"type\":\"ColumnDataSource\"}},\"id\":\"b83011b8-53ba-4c04-b024-5c00e3175291\",\"type\":\"CDSView\"},{\"attributes\":{\"axis_label\":\"Frequency\",\"formatter\":{\"id\":\"e01f31e5-ff7b-40ac-81d3-ce68cfdfc57a\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"9a26f8d8-ab11-492a-8014-0fc49f352102\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"f96262d4-d8c2-4f95-9425-387d88d8f8e2\",\"type\":\"BasicTicker\"}},\"id\":\"3581c9d0-cda4-4de5-89c3-b6ec8c3d84bf\",\"type\":\"LinearAxis\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"top\",\"width\",\"x\"],\"data\":{\"top\":{\"__ndarray__\":\"OVlHGh8YiT+yEvhgzayRP8QRdHzPU5E/e+X1xnEIij+Q7Ix3GU+IP6PcRSTQ5YA/kumfxPDfgD8ZJbHdqRF4P4NXZ787g3Y/0nGCUobcbz8cNIVENRRwPy2upSEQ2WY/tw4H4oWAZj9/cmyaPophPzxGyYRWBmE//rPDyd3vWz/sQnv9vGpcP6z9bvR6BFQ/MBbMTSDlVT9dSpshrDJPPzpWWmf2qFE/EXWGSsk+Sz8LtcrUSMlNPwFEJ1DT4Eo/J8asUap+ST9Jl0pEjKlCP+TwPMJyJ0I/+UOCCChEPj/SH+le5zVCP/By0+xH7zo/5WE7/zxzOD/F/3wKUbc3P4ZM88bYHjw/uu7kHEY7NT8dVQnNNJY2P3FZuReObjA/mowmKFp/ND+tbm0xRVAqP5pdMPyONys/3JCM41weJT8=\",\"dtype\":\"float64\",\"shape\":[40]},\"width\":{\"__ndarray__\":\"AAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkAAAAAAAAAeQAAAAAAAAB5AAAAAAAAAHkA=\",\"dtype\":\"float64\",\"shape\":[40]},\"x\":{\"__ndarray__\":\"AAAAAAAADkAAAAAAAIAmQAAAAAAAwDJAAAAAAABAOkAAAAAAAOBAQAAAAAAAoERAAAAAAABgSEAAAAAAACBMQAAAAAAA4E9AAAAAAADQUUAAAAAAALBTQAAAAAAAkFVAAAAAAABwV0AAAAAAAFBZQAAAAAAAMFtAAAAAAAAQXUAAAAAAAPBeQAAAAAAAaGBAAAAAAABYYUAAAAAAAEhiQAAAAAAAOGNAAAAAAAAoZEAAAAAAABhlQAAAAAAACGZAAAAAAAD4ZkAAAAAAAOhnQAAAAAAA2GhAAAAAAADIaUAAAAAAALhqQAAAAAAAqGtAAAAAAACYbEAAAAAAAIhtQAAAAAAAeG5AAAAAAABob0AAAAAAACxwQAAAAAAApHBAAAAAAAAccUAAAAAAAJRxQAAAAAAADHJAAAAAAACEckA=\",\"dtype\":\"float64\",\"shape\":[40]}}},\"id\":\"f10aa059-be36-423c-ac04-63fec9c06f8d\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null,\"mode\":\"vline\",\"tooltips\":[[\"bucket\",\"@x\"],[\"count\",\"@top\"]]},\"id\":\"819e2fdd-a427-4f84-b5cf-62859715b416\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"10939275-c51b-4b19-96a8-d15aaa5c8eb6\",\"type\":\"BasicTicker\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"9a26f8d8-ab11-492a-8014-0fc49f352102\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"f96262d4-d8c2-4f95-9425-387d88d8f8e2\",\"type\":\"BasicTicker\"}},\"id\":\"605d5479-b4ea-4048-b4bc-443434924cd7\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"e01f31e5-ff7b-40ac-81d3-ce68cfdfc57a\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"axis_label\":\"Number of tokens\",\"formatter\":{\"id\":\"8f0aba0b-f790-46af-9068-4acf7dabaf99\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"9a26f8d8-ab11-492a-8014-0fc49f352102\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"10939275-c51b-4b19-96a8-d15aaa5c8eb6\",\"type\":\"BasicTicker\"}},\"id\":\"e8364510-4c1d-45fd-b37b-74b292994dfa\",\"type\":\"LinearAxis\"},{\"attributes\":{\"fill_color\":{\"value\":\"firebrick\"},\"line_color\":{\"value\":\"#1f77b4\"},\"top\":{\"field\":\"top\"},\"width\":{\"field\":\"width\"},\"x\":{\"field\":\"x\"}},\"id\":\"5aaa3613-82ee-42dd-b95b-0a9dff0c0f44\",\"type\":\"VBar\"},{\"attributes\":{},\"id\":\"8f0aba0b-f790-46af-9068-4acf7dabaf99\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null,\"start\":0},\"id\":\"09a38421-f442-4156-9fa2-8808c75b2142\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"c45e6751-8c49-4e69-aba2-89eac9d09bf0\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"57117cf6-b81e-43a3-941e-b4bbcbdbca5d\",\"type\":\"LinearScale\"},{\"attributes\":{\"data_source\":{\"id\":\"f10aa059-be36-423c-ac04-63fec9c06f8d\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"54b1dec6-7c1f-4ab6-ae7f-00e686993a43\",\"type\":\"VBar\"},\"hover_glyph\":{\"id\":\"5aaa3613-82ee-42dd-b95b-0a9dff0c0f44\",\"type\":\"VBar\"},\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"c064d1e1-542a-4cf3-a10c-8b4ec7fead79\",\"type\":\"VBar\"},\"selection_glyph\":null,\"view\":{\"id\":\"b83011b8-53ba-4c04-b024-5c00e3175291\",\"type\":\"CDSView\"}},\"id\":\"51b8ae77-9066-408c-85f5-dff171d4cc01\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"819e2fdd-a427-4f84-b5cf-62859715b416\",\"type\":\"HoverTool\"}]},\"id\":\"e620cf71-2bc8-482f-8cce-b28cb6c5376c\",\"type\":\"Toolbar\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"#1f77b4\"},\"top\":{\"field\":\"top\"},\"width\":{\"field\":\"width\"},\"x\":{\"field\":\"x\"}},\"id\":\"54b1dec6-7c1f-4ab6-ae7f-00e686993a43\",\"type\":\"VBar\"},{\"attributes\":{\"below\":[{\"id\":\"e8364510-4c1d-45fd-b37b-74b292994dfa\",\"type\":\"LinearAxis\"}],\"left\":[{\"id\":\"3581c9d0-cda4-4de5-89c3-b6ec8c3d84bf\",\"type\":\"LinearAxis\"}],\"plot_height\":400,\"plot_width\":800,\"renderers\":[{\"id\":\"e8364510-4c1d-45fd-b37b-74b292994dfa\",\"type\":\"LinearAxis\"},{\"id\":\"3baa98e1-9bf6-4e85-b2fa-e192380c7dc7\",\"type\":\"Grid\"},{\"id\":\"3581c9d0-cda4-4de5-89c3-b6ec8c3d84bf\",\"type\":\"LinearAxis\"},{\"id\":\"605d5479-b4ea-4048-b4bc-443434924cd7\",\"type\":\"Grid\"},{\"id\":\"51b8ae77-9066-408c-85f5-dff171d4cc01\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"657c2839-d2fc-4317-8fb4-3d1843a2496a\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"e620cf71-2bc8-482f-8cce-b28cb6c5376c\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"1eaa8e0e-2c1d-439f-8f25-03519369b17b\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"c45e6751-8c49-4e69-aba2-89eac9d09bf0\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"09a38421-f442-4156-9fa2-8808c75b2142\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"57117cf6-b81e-43a3-941e-b4bbcbdbca5d\",\"type\":\"LinearScale\"}},\"id\":\"9a26f8d8-ab11-492a-8014-0fc49f352102\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"657c2839-d2fc-4317-8fb4-3d1843a2496a\",\"type\":\"Title\"},{\"attributes\":{\"plot\":{\"id\":\"9a26f8d8-ab11-492a-8014-0fc49f352102\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"10939275-c51b-4b19-96a8-d15aaa5c8eb6\",\"type\":\"BasicTicker\"}},\"id\":\"3baa98e1-9bf6-4e85-b2fa-e192380c7dc7\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"f96262d4-d8c2-4f95-9425-387d88d8f8e2\",\"type\":\"BasicTicker\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"top\":{\"field\":\"top\"},\"width\":{\"field\":\"width\"},\"x\":{\"field\":\"x\"}},\"id\":\"c064d1e1-542a-4cf3-a10c-8b4ec7fead79\",\"type\":\"VBar\"}],\"root_ids\":[\"9a26f8d8-ab11-492a-8014-0fc49f352102\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.14\"}};\n",
       "  var render_items = [{\"docid\":\"d78f0aeb-b589-4220-9bfb-51d890af09d1\",\"elementid\":\"12c92110-a8df-4173-95fc-9fc161ef40f7\",\"modelid\":\"9a26f8d8-ab11-492a-8014-0fc49f352102\"}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\")\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "9a26f8d8-ab11-492a-8014-0fc49f352102"
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90% percentile length: 152\n"
     ]
    }
   ],
   "source": [
    "train_comment_text = train['comment_text']\n",
    "sentences_lengths = train_comment_text.str.split().apply(len)\n",
    "print(\"Max length = {0}\".format(max(sentences_lengths)))\n",
    "plot_length_histogram(sentences_lengths, x_range=[0,300])\n",
    "print(\"90% percentile length: {:.0f}\".format(sentences_lengths.quantile(.90)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f1a77ee6780>,\n",
       "        <matplotlib.axes._subplots.AxesSubplot object at 0x7f1a77e38748>],\n",
       "       [<matplotlib.axes._subplots.AxesSubplot object at 0x7f1a77e58eb8>,\n",
       "        <matplotlib.axes._subplots.AxesSubplot object at 0x7f1a77dfbeb8>],\n",
       "       [<matplotlib.axes._subplots.AxesSubplot object at 0x7f1a77e1cdd8>,\n",
       "        <matplotlib.axes._subplots.AxesSubplot object at 0x7f1a77dc2eb8>]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.hist(grid=False, figsize=(12,12), column=['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lets pick cases where 'toxic' is 0 and 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline classifiers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV score for class toxic is 0.9571334034203997\n",
      "CV score for class severe_toxic is 0.984818091962158\n",
      "CV score for class obscene is 0.9799023282524604\n",
      "CV score for class threat is 0.9772003759212947\n",
      "CV score for class insult is 0.969468069332137\n",
      "CV score for class identity_hate is 0.9672945249629655\n",
      "Total CV score is 0.9726361323085692\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from scipy.sparse import hstack\n",
    "\n",
    "class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n",
    "\n",
    "train_text = train['comment_text']\n",
    "test_text = test['comment_text']\n",
    "all_text = pd.concat([train_text, test_text])\n",
    "\n",
    "word_vectorizer = TfidfVectorizer(\n",
    "    sublinear_tf=True,\n",
    "    strip_accents='unicode',\n",
    "    analyzer='word',\n",
    "    token_pattern=r'\\w{1,}',\n",
    "    stop_words='english',\n",
    "    ngram_range=(1, 1),\n",
    "    max_features=10000)\n",
    "\n",
    "word_vectorizer.fit(all_text)\n",
    "train_word_features = word_vectorizer.transform(train_text)\n",
    "test_word_features = word_vectorizer.transform(test_text)\n",
    "\n",
    "submission = pd.DataFrame.from_dict({'id': test['id']})\n",
    "scores = []\n",
    "for class_name in class_names:\n",
    "    train_target = train[class_name]\n",
    "    classifier = LogisticRegression(C=0.1, solver='sag')\n",
    "\n",
    "    cv_score = np.mean(cross_val_score(classifier, train_word_features, train_target, cv=3, scoring='roc_auc'))\n",
    "    scores.append(cv_score)\n",
    "    print('CV score for class {} is {}'.format(class_name, cv_score))\n",
    "\n",
    "    classifier.fit(train_word_features, train_target)\n",
    "    submission[class_name] = classifier.predict_proba(test_word_features)[:, 1]\n",
    "\n",
    "print('Total CV score is {}'.format(np.mean(scores)))\n",
    "\n",
    "submission.to_csv('submission_logreg.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Below we show an example of batching on a tiny dataset. Checkout tiny_data.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded tiny_data.csv: Num rows = 8, Num cols = 8\n",
      "Column names: ['id' 'comment_text' 'toxic' 'severe_toxic' 'obscene' 'threat' 'insult'\n",
      " 'identity_hate']\n",
      "\n",
      "Batch 1\n",
      "----------\n",
      "b'This is severe_toxic comment' [1 1 0 0 0 0]\n",
      "b'This is toxic comment' [1 0 0 0 0 0]\n",
      "\n",
      "Batch 2\n",
      "----------\n",
      "b'This is ok comment' [0 0 0 0 0 0]\n",
      "b'This is obscene comment' [1 0 1 0 0 0]\n",
      "\n",
      "Batch 3\n",
      "----------\n",
      "b'This is threat comment' [0 1 0 1 0 0]\n",
      "b'This is hate comment' [0 0 0 0 1 1]\n",
      "\n",
      "Batch 4\n",
      "----------\n",
      "b'This is really bad comment' [1 1 1 1 1 1]\n",
      "b'This is insult comment' [0 0 0 0 1 0]\n",
      "*************Epoch 1 finished*************\n",
      "\n",
      "\n",
      "Batch 5\n",
      "----------\n",
      "b'This is obscene comment' [1 0 1 0 0 0]\n",
      "b'This is threat comment' [0 1 0 1 0 0]\n",
      "\n",
      "Batch 6\n",
      "----------\n",
      "b'This is hate comment' [0 0 0 0 1 1]\n",
      "b'This is severe_toxic comment' [1 1 0 0 0 0]\n",
      "\n",
      "Batch 7\n",
      "----------\n",
      "b'This is ok comment' [0 0 0 0 0 0]\n",
      "b'This is insult comment' [0 0 0 0 1 0]\n",
      "\n",
      "Batch 8\n",
      "----------\n",
      "b'This is toxic comment' [1 0 0 0 0 0]\n",
      "b'This is really bad comment' [1 1 1 1 1 1]\n",
      "*************Epoch 2 finished*************\n",
      "\n",
      "\n",
      "Batch 9\n",
      "----------\n",
      "b'This is hate comment' [0 0 0 0 1 1]\n",
      "b'This is insult comment' [0 0 0 0 1 0]\n",
      "\n",
      "Batch 10\n",
      "----------\n",
      "b'This is threat comment' [0 1 0 1 0 0]\n",
      "b'This is really bad comment' [1 1 1 1 1 1]\n",
      "\n",
      "Batch 11\n",
      "----------\n",
      "b'This is ok comment' [0 0 0 0 0 0]\n",
      "b'This is severe_toxic comment' [1 1 0 0 0 0]\n",
      "\n",
      "Batch 12\n",
      "----------\n",
      "b'This is toxic comment' [1 0 0 0 0 0]\n",
      "b'This is obscene comment' [1 0 1 0 0 0]\n",
      "*************Epoch 3 finished*************\n",
      "\n"
     ]
    }
   ],
   "source": [
    "reload(data_utils)\n",
    "data_loader = data_utils.DataLoader(path=\"./data\")\n",
    "tiny_data = data_loader.readTinyData(verbose=True)\n",
    "bg = data_loader.batchGenerator(tiny_data, batch_size=2, num_epochs=3, verbose=True)\n",
    "\n",
    "for i, (batch_comments, batch_labels) in enumerate(bg):\n",
    "    print(\"\\nBatch {0}\\n----------\".format(i+1))\n",
    "    for c,l in zip(batch_comments, batch_labels):\n",
    "        print(c,l)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}