neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Attibution**: This code was adapted from the following 'Text Classification using Neural Networks' tutorial by gk_:\n",
    "https://machinelearnings.co/text-classification-using-neural-networks-f5cd7b8765c6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use natural language toolkit\n",
    "import nltk\n",
    "from nltk.stem.lancaster import LancasterStemmer\n",
    "ls = LancasterStemmer()\n",
    "\n",
    "import os\n",
    "import json\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1804874, 45)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def assign_toxicity(target):\n",
    "    if target > 0.8:\n",
    "        return \"super toxic\"\n",
    "    elif target > 0.5:\n",
    "        return \"toxic\"\n",
    "    else:\n",
    "        return \"not toxic\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['cat'] = train.target.apply(assign_toxicity,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "holdout = train[100000:150000]\n",
    "for_model = train[:100000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "super_toxic = for_model[for_model.cat == 'super toxic']\n",
    "\n",
    "toxic = for_model[for_model.cat == 'toxic']\n",
    "nontoxic = for_model[for_model.cat == 'not toxic']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1190, 46), (3944, 46), (94866, 46))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "super_toxic.shape, toxic.shape, nontoxic.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.94866"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nontoxic.shape[0] / for_model.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = super_toxic.head(10).append(toxic.head(20)).append(nontoxic.head(70))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = []\n",
    "classes = []\n",
    "documents = []\n",
    "stops = ['?']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 documents\n",
      "3 classes ['toxic', 'super toxic', 'not toxic']\n"
     ]
    }
   ],
   "source": [
    "# loop through each sentence in our training data\n",
    "for i,r in sample.iterrows():\n",
    "    # tokenize each word in the sentence\n",
    "    w = nltk.word_tokenize(r['comment_text'])\n",
    "    # add to our words list\n",
    "    words.extend(w)\n",
    "    # add to documents in our corpus\n",
    "    documents.append((w, r['cat']))\n",
    "    # add to our classes list\n",
    "    if r['cat'] not in classes:\n",
    "        classes.append(r['cat'])\n",
    "\n",
    "# stem and lower each word and remove duplicates\n",
    "words = [ls.stem(w.lower()) for w in words if w not in stops]\n",
    "words = list(set(words))\n",
    "\n",
    "# remove duplicates\n",
    "classes = list(set(classes))\n",
    "\n",
    "print (len(documents), \"documents\")\n",
    "print (len(classes), \"classes\", classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute sigmoid nonlinearity\n",
    "def sigmoid(x):\n",
    "    output = 1/(1+np.exp(-x))\n",
    "    return output\n",
    "\n",
    "# convert output of sigmoid function to its derivative\n",
    "def sigmoid_output_to_derivative(output):\n",
    "    return output*(1-output)\n",
    " \n",
    "def clean_up_sentence(sentence):\n",
    "    # tokenize the pattern\n",
    "    sentence_words = nltk.word_tokenize(sentence)\n",
    "    # stem each word\n",
    "    sentence_words = [ls.stem(word.lower()) for word in sentence_words]\n",
    "    return sentence_words\n",
    "\n",
    "# return bag of words array: 0 or 1 for each word in the bag that exists in the sentence\n",
    "def bow(sentence, words, show_details=False):\n",
    "    # tokenize the pattern\n",
    "    sentence_words = clean_up_sentence(sentence)\n",
    "    # bag of words\n",
    "    bag = [0]*len(words)  \n",
    "    for s in sentence_words:\n",
    "        for i,w in enumerate(words):\n",
    "            if w == s: \n",
    "                bag[i] = 1\n",
    "                if show_details:\n",
    "                    print (\"found in bag: %s\" % w)\n",
    "\n",
    "    return(np.array(bag))\n",
    "\n",
    "def think(sentence, show_details=False):\n",
    "    x = bow(sentence.lower(), words, show_details)\n",
    "    if show_details:\n",
    "        print (\"sentence:\", sentence, \"\\n bow:\", x)\n",
    "    # input layer is our bag of words\n",
    "    l0 = x\n",
    "    # matrix multiplication of input and hidden layer\n",
    "    l1 = sigmoid(np.dot(l0, synapse_0))\n",
    "    # output layer\n",
    "    l2 = sigmoid(np.dot(l1, synapse_1))\n",
    "    return l2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vary parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def train(X, y, hidden_neurons=10, alpha=1, epochs=5000, dropout=True, dropout_percent=0.5):\n",
    "\n",
    "    print (\"Number of Neurons: {}, alpha: {}, epochs: {}, dropout: {}, dropout_percent: {}\".format(hidden_neurons, alpha, epochs, dropout, dropout_percent))\n",
    "    np.random.seed(1)\n",
    "\n",
    "    last_mean_error = 1\n",
    "    # randomly initialize our weights with mean 0\n",
    "    synapse_0 = 2*np.random.random((len(X[0]), hidden_neurons)) - 1\n",
    "    synapse_1 = 2*np.random.random((hidden_neurons, len(classes))) - 1\n",
    "\n",
    "    prev_synapse_0_weight_update = np.zeros_like(synapse_0)\n",
    "    prev_synapse_1_weight_update = np.zeros_like(synapse_1)\n",
    "\n",
    "    synapse_0_direction_count = np.zeros_like(synapse_0)\n",
    "    synapse_1_direction_count = np.zeros_like(synapse_1)\n",
    "        \n",
    "    for j in iter(range(epochs+1)):\n",
    "\n",
    "        # Feed forward through layers 0, 1, and 2\n",
    "        layer_0 = X\n",
    "        layer_1 = sigmoid(np.dot(layer_0, synapse_0))\n",
    "                \n",
    "        if(dropout):\n",
    "            layer_1 *= np.random.binomial([np.ones((len(X),hidden_neurons))],1-dropout_percent)[0] * (1.0/(1-dropout_percent))\n",
    "\n",
    "        layer_2 = sigmoid(np.dot(layer_1, synapse_1))\n",
    "\n",
    "        # how much did we miss the target value?\n",
    "        layer_2_error = y - layer_2\n",
    "\n",
    "        if (j% 1000) == 0 and j > 5000:\n",
    "            # if this 10k iteration's error is greater than the last iteration, break out\n",
    "            if np.mean(np.abs(layer_2_error)) < last_mean_error:\n",
    "                print (\"delta after \"+str(j)+\" iterations:\" + str(np.mean(np.abs(layer_2_error))) )\n",
    "                last_mean_error = np.mean(np.abs(layer_2_error))\n",
    "            else:\n",
    "                print (\"break:\", np.mean(np.abs(layer_2_error)), \">\", last_mean_error )\n",
    "                break\n",
    "                \n",
    "        # in what direction is the target value?\n",
    "        # were we really sure? if so, don't change too much.\n",
    "        layer_2_delta = layer_2_error * sigmoid_output_to_derivative(layer_2)\n",
    "\n",
    "        # how much did each l1 value contribute to the l2 error (according to the weights)?\n",
    "        layer_1_error = layer_2_delta.dot(synapse_1.T)\n",
    "\n",
    "        # in what direction is the target l1?\n",
    "        # were we really sure? if so, don't change too much.\n",
    "        layer_1_delta = layer_1_error * sigmoid_output_to_derivative(layer_1)\n",
    "        \n",
    "        synapse_1_weight_update = (layer_1.T.dot(layer_2_delta))\n",
    "        synapse_0_weight_update = (layer_0.T.dot(layer_1_delta))\n",
    "        \n",
    "        if(j > 0):\n",
    "            synapse_0_direction_count += np.abs(((synapse_0_weight_update > 0)+0) - ((prev_synapse_0_weight_update > 0) + 0))\n",
    "            synapse_1_direction_count += np.abs(((synapse_1_weight_update > 0)+0) - ((prev_synapse_1_weight_update > 0) + 0))        \n",
    "        \n",
    "        synapse_1 += alpha * synapse_1_weight_update\n",
    "        synapse_0 += alpha * synapse_0_weight_update\n",
    "        \n",
    "        prev_synapse_0_weight_update = synapse_0_weight_update\n",
    "        prev_synapse_1_weight_update = synapse_1_weight_update\n",
    "\n",
    "    now = datetime.datetime.now()\n",
    "\n",
    "    # persist synapses\n",
    "    synapse = {'synapse0': synapse_0.tolist(), 'synapse1': synapse_1.tolist(),\n",
    "               'datetime': now.strftime(\"%Y-%m-%d %H:%M\"),\n",
    "               'words': words,\n",
    "               'classes': classes\n",
    "              }\n",
    "    synapse_file = \"synapses.json\"\n",
    "\n",
    "    with open(synapse_file, 'w') as outfile:\n",
    "        json.dump(synapse, outfile, indent=4, sort_keys=True)\n",
    "    print (\"saved synapses to:\", synapse_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify(sentence, show_details=False):\n",
    "    results = think(sentence, show_details)\n",
    "\n",
    "    results = [[i,r] for i,r in enumerate(results) if r>ERROR_THRESHOLD ] \n",
    "    results.sort(key=lambda x: x[1], reverse=True) \n",
    "    return_results =[[classes[r[0]],r[1]] for r in results]\n",
    "\n",
    "    return return_results\n",
    "\n",
    "def test_nn():\n",
    "    sample_holdout = holdout.sample(n=1000)\n",
    "    print(sample_holdout.cat.value_counts())\n",
    "    sample_holdout['classified'] = sample_holdout.comment_text.apply(classify,)\n",
    "    \n",
    "    output = sample_holdout[['target', 'comment_text', 'cat', 'classified']]\n",
    "    output['model_result'] = output.classified.apply(extract_result,)\n",
    "    print()\n",
    "    print(\"extracted results\")\n",
    "    print()\n",
    "\n",
    "    output['accuracy'] = output.cat == output.model_result\n",
    "    print(output.accuracy.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n"
     ]
    }
   ],
   "source": [
    "# create our training data\n",
    "training, output = [], []\n",
    "# create an empty array for our output\n",
    "output_empty = [0] * len(classes)\n",
    "n = 0 \n",
    "\n",
    "# training set, bag of words for each sentence\n",
    "for doc in documents:\n",
    "    n += 1\n",
    "    if n % 100 == 0:\n",
    "        print(n)\n",
    "    # initialize our bag of words\n",
    "    bag = []\n",
    "    # list of tokenized words for the pattern\n",
    "    pattern_words = doc[0]\n",
    "    # stem each word\n",
    "    pattern_words = [ls.stem(word.lower()) for word in pattern_words]\n",
    "    # create our bag of words array\n",
    "    for w in words:\n",
    "        bag.append(1) if w in pattern_words else bag.append(0)\n",
    "\n",
    "    training.append(bag)\n",
    "    # output is a '0' for each tag and '1' for current tag\n",
    "    output_row = list(output_empty)\n",
    "    output_row[classes.index(doc[1])] = 1\n",
    "    output.append(output_row)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Neurons: 10, alpha: 1, epochs: 6000, dropout: False, dropout_percent: 0.5\n",
      "delta after 6000 iterations:0.0009924492963152196\n",
      "saved synapses to: synapses.json\n",
      "\n",
      "processing time: 3.6982030868530273 seconds\n",
      "\n",
      "not toxic      926\n",
      "toxic           61\n",
      "super toxic     13\n",
      "Name: cat, dtype: int64\n",
      "\n",
      "extracted results\n",
      "\n",
      "True     651\n",
      "False    349\n",
      "Name: accuracy, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:16: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  app.launch_new_instance()\n",
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "X = np.array(training)\n",
    "y = np.array(output)\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "train(X, y, dropout=False, epochs = 6000)\n",
    "print()\n",
    "elapsed_time = time.time() - start_time\n",
    "print (\"processing time:\", elapsed_time, \"seconds\")\n",
    "# probability threshold\n",
    "ERROR_THRESHOLD = 0.2\n",
    "# load our calculated synapse values\n",
    "synapse_file = 'synapses.json' \n",
    "with open(synapse_file) as data_file: \n",
    "    synapse = json.load(data_file) \n",
    "    synapse_0 = np.asarray(synapse['synapse0']) \n",
    "    synapse_1 = np.asarray(synapse['synapse1'])\n",
    "\n",
    "print()\n",
    "test_nn()\n",
    "\n",
    "\n",
    "training, output = [], []\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "n = 100\n",
    "\n",
    "Number of Neurons: 10, alpha: 1, epochs: 6000, dropout: True, dropout_percent: 0.5\n",
    "\n",
    "True     932\n",
    "False     68"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "n = 100\n",
    "\n",
    "Number of Neurons: 10, alpha: 1, epochs: 6000, dropout: False, dropout_percent: 0.5\n",
    "\n",
    "True     651\n",
    "False    349"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}