Supervised NLP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package gutenberg to /home/jovyan/nltk_data...\n",
      "[nltk_data]   Package gutenberg is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: en_core_web_sm==2.0.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.0.0/en_core_web_sm-2.0.0.tar.gz#egg=en_core_web_sm==2.0.0 in /opt/conda/lib/python3.7/site-packages (2.0.0)\n",
      "\n",
      "\u001b[93m    Linking successful\u001b[0m\n",
      "    /opt/conda/lib/python3.7/site-packages/en_core_web_sm -->\n",
      "    /opt/conda/lib/python3.7/site-packages/spacy/data/en\n",
      "\n",
      "    You can now load the model via spacy.load('en')\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy\n",
    "import sklearn\n",
    "import spacy\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import re\n",
    "from nltk.corpus import gutenberg, stopwords\n",
    "from collections import Counter\n",
    "import nltk\n",
    "\n",
    "nltk.download('gutenberg')\n",
    "!python -m spacy download en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Utility function for standard text cleaning.\n",
    "def text_cleaner(text):\n",
    "    # Visual inspection identifies a form of punctuation spaCy does not\n",
    "    # recognize: the double dash '--'.  Better get rid of it now!\n",
    "    text = re.sub(r'--',' ',text)\n",
    "    text = re.sub(\"[\\[].*?[\\]]\", \"\", text)\n",
    "    text = ' '.join(text.split())\n",
    "    return text\n",
    "    \n",
    "# Load and clean the data.\n",
    "persuasion = gutenberg.raw('austen-persuasion.txt')\n",
    "alice = gutenberg.raw('carroll-alice.txt')\n",
    "\n",
    "# The Chapter indicator is idiosyncratic\n",
    "persuasion = re.sub(r'Chapter \\d+', '', persuasion)\n",
    "alice = re.sub(r'CHAPTER .*', '', alice)\n",
    "    \n",
    "alice = text_cleaner(alice[:int(len(alice)/10)])\n",
    "persuasion = text_cleaner(persuasion[:int(len(persuasion)/10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse the cleaned novels. This can take a bit.\n",
    "nlp = spacy.load('en')\n",
    "alice_doc = nlp(alice)\n",
    "persuasion_doc = nlp(persuasion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(Alice, was, beginning, to, get, very, tired, ...</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(So, she, was, considering, in, her, own, mind...</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(There, was, nothing, so, VERY, remarkable, in...</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(Oh, dear, !)</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(I, shall, be, late, !, ')</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   0        1\n",
       "0  (Alice, was, beginning, to, get, very, tired, ...  Carroll\n",
       "1  (So, she, was, considering, in, her, own, mind...  Carroll\n",
       "2  (There, was, nothing, so, VERY, remarkable, in...  Carroll\n",
       "3                                      (Oh, dear, !)  Carroll\n",
       "4                         (I, shall, be, late, !, ')  Carroll"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Group into sentences.\n",
    "alice_sents = [[sent, \"Carroll\"] for sent in alice_doc.sents]\n",
    "persuasion_sents = [[sent, \"Austen\"] for sent in persuasion_doc.sents]\n",
    "\n",
    "# Combine the sentences from the two novels into one data frame.\n",
    "sentences = pd.DataFrame(alice_sents + persuasion_sents)\n",
    "sentences.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "source": [
    "Time to bag some words!  Since spaCy has already tokenized and labelled our data, we can move directly to recording how often various words occur.  We will exclude stopwords and punctuation.  In addition, in an attempt to keep our feature space from exploding, we will work with lemmas (root words) rather than the raw text terms, and we'll only use the 2000 most common words for each text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Utility function to create a list of the 2000 most common words.\n",
    "def bag_of_words(text):\n",
    "    \n",
    "    # Filter out punctuation and stop words.\n",
    "    allwords = [token.lemma_\n",
    "                for token in text\n",
    "                if not token.is_punct\n",
    "                and not token.is_stop]\n",
    "    \n",
    "    # Return the most common words.\n",
    "    return [item[0] for item in Counter(allwords).most_common(2000)]\n",
    "    \n",
    "\n",
    "# Creates a data frame with features for each word in our common word set.\n",
    "# Each value is the count of the times the word appears in each sentence.\n",
    "def bow_features(sentences, common_words):\n",
    "    \n",
    "    # Scaffold the data frame and initialize counts to zero.\n",
    "    df = pd.DataFrame(columns=common_words)\n",
    "    df['text_sentence'] = sentences[0]\n",
    "    df['text_source'] = sentences[1]\n",
    "    df.loc[:, common_words] = 0\n",
    "    \n",
    "    # Process each row, counting the occurrence of words in each sentence.\n",
    "    for i, sentence in enumerate(df['text_sentence']):\n",
    "        \n",
    "        # Convert the sentence to lemmas, then filter out punctuation,\n",
    "        # stop words, and uncommon words.\n",
    "        words = [token.lemma_\n",
    "                 for token in sentence\n",
    "                 if (\n",
    "                     not token.is_punct\n",
    "                     and not token.is_stop\n",
    "                     and token.lemma_ in common_words\n",
    "                 )]\n",
    "        \n",
    "        # Populate the row with word counts.\n",
    "        for word in words:\n",
    "            df.loc[i, word] += 1\n",
    "        \n",
    "        # This counter is just to make sure the kernel didn't hang.\n",
    "        if i % 50 == 0:\n",
    "            print(\"Processing row {}\".format(i))\n",
    "            \n",
    "    return df\n",
    "\n",
    "# Set up the bags.\n",
    "alicewords = bag_of_words(alice_doc)\n",
    "persuasionwords = bag_of_words(persuasion_doc)\n",
    "\n",
    "# Combine bags to create a set of unique words.\n",
    "common_words = set(alicewords + persuasionwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing row 0\n",
      "Processing row 50\n",
      "Processing row 100\n",
      "Processing row 150\n",
      "Processing row 200\n",
      "Processing row 250\n",
      "Processing row 300\n",
      "Processing row 350\n",
      "Processing row 400\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>easy</th>\n",
       "      <th>desperate</th>\n",
       "      <th>contentment</th>\n",
       "      <th>certain</th>\n",
       "      <th>and</th>\n",
       "      <th>match</th>\n",
       "      <th>mindedness</th>\n",
       "      <th>quit</th>\n",
       "      <th>peg</th>\n",
       "      <th>steady</th>\n",
       "      <th>...</th>\n",
       "      <th>solicit</th>\n",
       "      <th>hang</th>\n",
       "      <th>flavour</th>\n",
       "      <th>suddenly</th>\n",
       "      <th>pink</th>\n",
       "      <th>daisy</th>\n",
       "      <th>raise</th>\n",
       "      <th>light</th>\n",
       "      <th>text_sentence</th>\n",
       "      <th>text_source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(Alice, was, beginning, to, get, very, tired, ...</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(So, she, was, considering, in, her, own, mind...</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(There, was, nothing, so, VERY, remarkable, in...</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(Oh, dear, !)</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(I, shall, be, late, !, ')</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1649 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  easy desperate contentment certain and match mindedness quit peg steady  \\\n",
       "0    0         0           0       0   0     0          0    0   0      0   \n",
       "1    0         0           0       0   0     0          0    0   0      0   \n",
       "2    0         0           0       0   0     0          0    0   0      0   \n",
       "3    0         0           0       0   0     0          0    0   0      0   \n",
       "4    0         0           0       0   0     0          0    0   0      0   \n",
       "\n",
       "   ... solicit hang flavour suddenly pink daisy raise light  \\\n",
       "0  ...       0    0       0        0    0     0     0     0   \n",
       "1  ...       0    0       0        1    1     2     0     0   \n",
       "2  ...       0    0       0        0    0     0     0     0   \n",
       "3  ...       0    0       0        0    0     0     0     0   \n",
       "4  ...       0    0       0        0    0     0     0     0   \n",
       "\n",
       "                                       text_sentence text_source  \n",
       "0  (Alice, was, beginning, to, get, very, tired, ...     Carroll  \n",
       "1  (So, she, was, considering, in, her, own, mind...     Carroll  \n",
       "2  (There, was, nothing, so, VERY, remarkable, in...     Carroll  \n",
       "3                                      (Oh, dear, !)     Carroll  \n",
       "4                         (I, shall, be, late, !, ')     Carroll  \n",
       "\n",
       "[5 rows x 1649 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create our data frame with features. This can take a while to run.\n",
    "word_counts = bow_features(sentences, common_words)\n",
    "word_counts.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Parts of Speech POS</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import ensemble\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "rfc = ensemble.RandomForestClassifier()\n",
    "Y = word_counts['text_source']\n",
    "X = np.array(word_counts.drop(['text_sentence','text_source'], 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "source": [
    "## Trying out BoW\n",
    "\n",
    "Now let's give the bag of words features a whirl by trying a random forest."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set score: 0.988\n",
      "\n",
      "Test set score: 0.8143712574850299\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/ensemble/forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "                                                    Y,\n",
    "                                                    test_size=0.4,\n",
    "                                                    random_state=0)\n",
    "train = rfc.fit(X_train, y_train)\n",
    "\n",
    "print('Training set score:', rfc.score(X_train, y_train))\n",
    "print('\\nTest set score:', rfc.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "source": [
    "## BoW with Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(250, 1647) (250,)\n",
      "Training set score: 0.976\n",
      "\n",
      "Test set score: 0.8862275449101796\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lr = LogisticRegression(penalty='l2') # No need to specify l2 as it's the default. But we put it for demonstration.\n",
    "train = lr.fit(X_train, y_train)\n",
    "print(X_train.shape, y_train.shape)\n",
    "print('Training set score:', lr.score(X_train, y_train))\n",
    "print('\\nTest set score:', lr.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "source": [
    "# BoW with Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set score: 0.968\n",
      "\n",
      "Test set score: 0.8143712574850299\n"
     ]
    }
   ],
   "source": [
    "clf = ensemble.GradientBoostingClassifier()\n",
    "train = clf.fit(X_train, y_train)\n",
    "\n",
    "print('Training set score:', clf.score(X_train, y_train))\n",
    "print('\\nTest set score:', clf.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "source": [
    "Looks like logistic regression is the winner, but there's room for improvement.\n",
    "\n",
    "# Same model, new inputs\n",
    "\n",
    "What if we feed the model a different novel by Jane Austen, like _Emma_?  Will it be able to distinguish Austen from Carroll with the same level of accuracy if we insert a different sample of Austen's writing?\n",
    "\n",
    "First, we need to process _Emma_ the same way we processed the other data, and combine it with the Alice data. Remember that for computation time concerns, we only took the first tenth of the Alice text. Emma is pretty long. **So in order to get comparable length texts, we take the first sixtieth of Emma**. Again, if you want to experiment, you can take the whole texts of each."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emma Woodhouse, handsome, clever, and rich, with a comfortable home and happy disposition, seemed to\n"
     ]
    }
   ],
   "source": [
    "# Clean the Emma data.\n",
    "emma = gutenberg.raw('austen-emma.txt')\n",
    "emma = re.sub(r'VOLUME \\w+', '', emma)\n",
    "emma = re.sub(r'CHAPTER \\w+', '', emma)\n",
    "emma = text_cleaner(emma[:int(len(emma)/60)])\n",
    "print(emma[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse our cleaned data.\n",
    "emma_doc = nlp(emma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Group into sentences.\n",
    "persuasion_sents = [[sent, \"Austen\"] for sent in persuasion_doc.sents]\n",
    "emma_sents = [[sent, \"Austen\"] for sent in emma_doc.sents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing row 0\n",
      "Processing row 50\n",
      "Processing row 100\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "# Build a new Bag of Words data frame for Emma word counts.\n",
    "# We'll use the same common words from Alice and Persuasion.\n",
    "emma_sentences = pd.DataFrame(emma_sents)\n",
    "emma_bow = bow_features(emma_sentences, common_words)\n",
    "\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set score: 0.6330275229357798\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>Austen</th>\n",
       "      <th>Carroll</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Austen</th>\n",
       "      <td>120</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Carroll</th>\n",
       "      <td>60</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0    Austen  Carroll\n",
       "row_0                   \n",
       "Austen      120       20\n",
       "Carroll      60       18"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we can model it!\n",
    "# Let's use logistic regression again.\n",
    "\n",
    "# Combine the Emma sentence data with the Alice data from the test set.\n",
    "X_Emma_test = np.concatenate((\n",
    "    X_train[y_train[y_train=='Carroll'].index],\n",
    "    emma_bow.drop(['text_sentence','text_source'], 1)\n",
    "), axis=0)\n",
    "y_Emma_test = pd.concat([y_train[y_train=='Carroll'],pd.Series(['Austen'] * emma_bow.shape[0])])\n",
    "\n",
    "# Model.\n",
    "print('\\nTest set score:', lr.score(X_Emma_test, y_Emma_test))\n",
    "lr_Emma_predicted = lr.predict(X_Emma_test)\n",
    "pd.crosstab(y_Emma_test, lr_Emma_predicted)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "source": [
    "Well look at that!  NLP approaches are generally effective on the same type of material as they were trained on. It looks like this model is actually able to differentiate multiple works by Austen from Alice in Wonderland.  Now the question is whether the model is very good at identifying Austen, or very good at identifying Alice in Wonderland, or both...\n",
    "\n",
    "# Challenge 0:\n",
    "\n",
    "Recall that the logistic regression model's best performance on the test set was 93%.  See what you can do to improve performance.  Suggested avenues of investigation include: Other modeling techniques (SVM?), making more features that take advantage of the spaCy information (include grammar, phrases, POS, etc), making sentence-level features (number of words, amount of punctuation), or including contextual information (length of previous and next sentences, words repeated from one sentence to the next, etc), and anything else your heart desires.  Make sure to design your models on the test set, or use cross_validation with multiple folds, and see if you can get accuracy above 90%.  \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pos_df(sentences):\n",
    "    pos = []\n",
    "    for sent in sentences[0]:\n",
    "        for token in sent:\n",
    "            pos = pos + [token.pos_]\n",
    "    pos = set(pos)\n",
    "    pos_init = np.zeros(shape=(len(pos),len(sentences)))\n",
    "    df = pd.DataFrame(data=pos_init.T,index=np.arange(0,len(sentences)),columns=pos)\n",
    "    \n",
    "    for sent,ind in zip(sentences[0],np.arange(1,len(sentences))):    \n",
    "        for token in sent:\n",
    "            df.loc[ind,token.pos_] += 1 \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pos_df(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = word_counts.drop(['text_sentence','text_source'],1)\n",
    "X = pd.concat([X,df],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set score: 0.996\n",
      "\n",
      "Test set score: 0.8682634730538922\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "                                                    Y,\n",
    "                                                    test_size=0.4,\n",
    "                                                    random_state=0)\n",
    "train = rfc.fit(X_train, y_train)\n",
    "\n",
    "print('Training set score:', rfc.score(X_train, y_train))\n",
    "print('\\nTest set score:', rfc.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BoW and POS with Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(250, 1660) (250,)\n",
      "Training set score: 0.976\n",
      "\n",
      "Test set score: 0.9101796407185628\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "lr = LogisticRegression(penalty='l2') # No need to specify l2 as it's the default. But we put it for demonstration.\n",
    "train = lr.fit(X_train, y_train)\n",
    "print(X_train.shape, y_train.shape)\n",
    "print('Training set score:', lr.score(X_train, y_train))\n",
    "print('\\nTest set score:', lr.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BoW and POS with Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set score: 0.976\n",
      "\n",
      "Test set score: 0.844311377245509\n"
     ]
    }
   ],
   "source": [
    "clf = ensemble.GradientBoostingClassifier()\n",
    "train = clf.fit(X_train, y_train)\n",
    "\n",
    "print('Training set score:', clf.score(X_train, y_train))\n",
    "print('\\nTest set score:', clf.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Challenge 1:\n",
    "Find out whether your new model is good at identifying Alice in Wonderland vs any other work, Persuasion vs any other work, or Austen vs any other work.  This will involve pulling a new book from the Project Gutenberg corpus (print(gutenberg.fileids()) for a list) and processing it.\n",
    "\n",
    "Record your work for each challenge in a notebook and submit it below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['austen-emma.txt', 'austen-persuasion.txt', 'austen-sense.txt', 'bible-kjv.txt', 'blake-poems.txt', 'bryant-stories.txt', 'burgess-busterbrown.txt', 'carroll-alice.txt', 'chesterton-ball.txt', 'chesterton-brown.txt', 'chesterton-thursday.txt', 'edgeworth-parents.txt', 'melville-moby_dick.txt', 'milton-paradise.txt', 'shakespeare-caesar.txt', 'shakespeare-hamlet.txt', 'shakespeare-macbeth.txt', 'whitman-leaves.txt']\n"
     ]
    }
   ],
   "source": [
    "print(gutenberg.fileids())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "To Edmund Clerihew Bentley A cloud was on the mind of men, and wailing went the weather, Yea, a sick\n"
     ]
    }
   ],
   "source": [
    "# Clean the Emma data.\n",
    "chester = gutenberg.raw('chesterton-thursday.txt')\n",
    "chester = re.sub(r'VOLUME \\w+', '', chester)\n",
    "chester = re.sub(r'CHAPTER \\w+', '', chester)\n",
    "chester = text_cleaner(chester[:int(len(chester)/30)])\n",
    "print(chester[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse our cleaned data.\n",
    "chester_doc = nlp(chester)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Group into sentences.\n",
    "chester_sents = [[sent, \"Chester\"] for sent in chester_doc.sents]\n",
    "alice_sents = [[sent, \"Carroll\"] for sent in alice_doc.sents]\n",
    "sentences = pd.DataFrame(alice_sents + chester_sents)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up the bags.\n",
    "alicewords = bag_of_words(alice_doc)\n",
    "chesterwords = bag_of_words(chester_doc)\n",
    "\n",
    "# Combine bags to create a set of unique words.\n",
    "common_words = set(alicewords + chesterwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing row 0\n",
      "Processing row 50\n",
      "Processing row 100\n",
      "Processing row 150\n",
      "Processing row 200\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tell</th>\n",
       "      <th>pleasure</th>\n",
       "      <th>sane</th>\n",
       "      <th>peal</th>\n",
       "      <th>blind</th>\n",
       "      <th>desperate</th>\n",
       "      <th>certain</th>\n",
       "      <th>and</th>\n",
       "      <th>fort</th>\n",
       "      <th>sure</th>\n",
       "      <th>...</th>\n",
       "      <th>unpoetical</th>\n",
       "      <th>pink</th>\n",
       "      <th>daisy</th>\n",
       "      <th>gloom</th>\n",
       "      <th>arresting</th>\n",
       "      <th>expect</th>\n",
       "      <th>light</th>\n",
       "      <th>mauve</th>\n",
       "      <th>text_sentence</th>\n",
       "      <th>text_source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(Alice, was, beginning, to, get, very, tired, ...</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(So, she, was, considering, in, her, own, mind...</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(There, was, nothing, so, VERY, remarkable, in...</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(Oh, dear, !)</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(I, shall, be, late, !, ')</td>\n",
       "      <td>Carroll</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 939 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  tell pleasure sane peal blind desperate certain and fort sure  ...  \\\n",
       "0    0        0    0    0     0         0       0   0    0    0  ...   \n",
       "1    0        1    0    0     0         0       0   0    0    0  ...   \n",
       "2    0        0    0    0     0         0       0   0    0    0  ...   \n",
       "3    0        0    0    0     0         0       0   0    0    0  ...   \n",
       "4    0        0    0    0     0         0       0   0    0    0  ...   \n",
       "\n",
       "  unpoetical pink daisy gloom arresting expect light mauve  \\\n",
       "0          0    0     0     0         0      0     0     0   \n",
       "1          0    1     2     0         0      0     0     0   \n",
       "2          0    0     0     0         0      0     0     0   \n",
       "3          0    0     0     0         0      0     0     0   \n",
       "4          0    0     0     0         0      0     0     0   \n",
       "\n",
       "                                       text_sentence text_source  \n",
       "0  (Alice, was, beginning, to, get, very, tired, ...     Carroll  \n",
       "1  (So, she, was, considering, in, her, own, mind...     Carroll  \n",
       "2  (There, was, nothing, so, VERY, remarkable, in...     Carroll  \n",
       "3                                      (Oh, dear, !)     Carroll  \n",
       "4                         (I, shall, be, late, !, ')     Carroll  \n",
       "\n",
       "[5 rows x 939 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_counts = bow_features(sentences, common_words)\n",
    "word_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pos_df(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = word_counts.drop(['text_sentence','text_source'],1)\n",
    "X = pd.concat([X,df],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = word_counts['text_source']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set score: 0.8695652173913043\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>Carroll</th>\n",
       "      <th>Chester</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text_source</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Carroll</th>\n",
       "      <td>52</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chester</th>\n",
       "      <td>11</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0        Carroll  Chester\n",
       "text_source                  \n",
       "Carroll           52        1\n",
       "Chester           11       28"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "                                                    Y,\n",
    "                                                    test_size=0.4,\n",
    "                                                    random_state=0)\n",
    "train = lr.fit(X_train, y_train)\n",
    "\n",
    "# Model.\n",
    "print('\\nTest set score:', lr.score(X_test, y_test))\n",
    "lr_predicted = lr.predict(X_test)\n",
    "pd.crosstab(y_test, lr_predicted)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "49px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}