Topic modelling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Topic modelling\n",
    "__Definition:__ A statistical model for discovering the abstract \"topics\" that occur in a collection of documents. \n",
    "\n",
    "For instance we can go through the articles published by a newspaper and check what the top 5 topics are. \n",
    "\n",
    "_Spoiler: We won't do that._\n",
    "\n",
    "We will try to extract the main topics in the King James' Bible, and see what they are.\n",
    "\n",
    "Please download and extract the .csv file from [here](https://www.kaggle.com/phyred23/bibleverses/downloads/bible_data_set.csv) and place it in the root of the project. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.552*\"ye\" + 0.245*\"king\" + 0.233*\"israel\" + 0.193*\"man\" + 0.190*\"son\" + 0.163*\"hath\" + 0.154*\"house\" + 0.153*\"people\" + 0.143*\"come\" + 0.135*\"children\"'),\n",
       " (1,\n",
       "  '-0.777*\"ye\" + 0.321*\"king\" + 0.297*\"son\" + 0.177*\"israel\" + 0.144*\"came\" + 0.135*\"house\" + 0.131*\"man\" + 0.092*\"people\" + 0.087*\"children\" + 0.080*\"david\"'),\n",
       " (2,\n",
       "  '-0.879*\"son\" + -0.189*\"ye\" + 0.153*\"hath\" + 0.131*\"people\" + 0.120*\"israel\" + 0.116*\"one\" + 0.102*\"let\" + 0.100*\"us\" + 0.086*\"come\" + 0.081*\"every\"'),\n",
       " (3,\n",
       "  '-0.669*\"king\" + 0.468*\"man\" + 0.319*\"hath\" + 0.219*\"son\" + -0.170*\"israel\" + 0.130*\"every\" + -0.129*\"ye\" + 0.116*\"us\" + 0.108*\"one\" + 0.088*\"let\"'),\n",
       " (4,\n",
       "  '-0.563*\"israel\" + -0.551*\"children\" + 0.440*\"king\" + 0.244*\"hath\" + 0.162*\"man\" + -0.118*\"land\" + 0.112*\"let\" + 0.083*\"us\" + -0.075*\"moses\" + -0.066*\"son\"'),\n",
       " (5,\n",
       "  '-0.767*\"hath\" + 0.489*\"man\" + 0.195*\"one\" + 0.165*\"every\" + 0.140*\"came\" + -0.108*\"us\" + -0.093*\"israel\" + -0.084*\"children\" + -0.073*\"son\" + 0.069*\"men\"'),\n",
       " (6,\n",
       "  '0.564*\"man\" + 0.292*\"king\" + 0.269*\"israel\" + 0.245*\"hath\" + -0.243*\"us\" + -0.225*\"came\" + -0.223*\"people\" + -0.197*\"come\" + 0.192*\"children\" + -0.151*\"let\"'),\n",
       " (7,\n",
       "  '-0.724*\"one\" + -0.218*\"offering\" + 0.216*\"man\" + 0.182*\"people\" + -0.177*\"every\" + 0.173*\"land\" + 0.170*\"let\" + 0.166*\"us\" + 0.159*\"come\" + -0.138*\"hath\"'),\n",
       " (8,\n",
       "  '-0.579*\"came\" + 0.419*\"us\" + 0.415*\"let\" + -0.206*\"hath\" + -0.193*\"day\" + 0.163*\"one\" + -0.160*\"pass\" + 0.155*\"king\" + 0.142*\"go\" + -0.133*\"went\"'),\n",
       " (9,\n",
       "  '-0.868*\"house\" + 0.201*\"came\" + 0.186*\"one\" + 0.178*\"children\" + 0.141*\"us\" + 0.127*\"king\" + 0.099*\"people\" + 0.090*\"saying\" + -0.082*\"made\" + 0.076*\"day\"')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "stopWordsSet = set(stopwords.words('english'))\n",
    "otherWords = 'shall unto yee ye lord god thou thy thee said upon shalt saith'.split(' ')\n",
    "for word in otherWords:\n",
    "    stopWordsSet.add(word)\n",
    "def getWords(text):\n",
    "    return [x for x in re.compile('\\w+').findall(text) if x not in stopWordsSet]\n",
    "\n",
    "# read line by line, remove \\n, split to columns, skip 1st line\n",
    "print(2)\n",
    "lines = [x.strip('\\n').split(',') for x in open('bible_data_set.csv', 'r').readlines() if ',' in x][1:]\n",
    "# keep book, chapter and text\n",
    "\n",
    "filteredLines = [[x[0], ' '.join(x[4:])] for x in lines]\n",
    "\n",
    "documentDictionary = {}\n",
    "for line in filteredLines:\n",
    "    if line[0] not in documentDictionary:\n",
    "        documentDictionary[line[0]] = getWords(line[1].lower())\n",
    "    else:\n",
    "        documentDictionary[line[0]] += getWords(line[1].lower())\n",
    "        \n",
    "from gensim import corpora, models\n",
    "dictionary = corpora.Dictionary(documentDictionary.values())\n",
    "dictionary.save('bible.dict')\n",
    "corpus = [dictionary.doc2bow(line) for line in documentDictionary.values()]\n",
    "lsi = models.lsimodel.LsiModel(corpus=corpus, id2word=dictionary, num_topics=10)\n",
    "\n",
    "# print the most contributing words (both positively and negatively) for each of the first ten topics\n",
    "lsi.print_topics(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.021*\"ye\" + 0.021*\"moses\" + 0.018*\"eat\" + 0.016*\"saying\" + 0.015*\"day\" + 0.013*\"spake\" + 0.012*\"came\" + 0.012*\"holy\" + 0.011*\"bread\" + 0.011*\"people\"'),\n",
       " (1,\n",
       "  '0.023*\"sons\" + 0.016*\"son\" + 0.015*\"children\" + 0.013*\"speak\" + 0.012*\"ye\" + 0.010*\"israel\" + 0.009*\"man\" + 0.009*\"daughters\" + 0.007*\"wisdom\" + 0.007*\"men\"'),\n",
       " (2,\n",
       "  '0.024*\"one\" + 0.013*\"every\" + 0.013*\"daughter\" + 0.012*\"man\" + 0.011*\"sin\" + 0.011*\"day\" + 0.010*\"young\" + 0.009*\"saul\" + 0.009*\"month\" + 0.008*\"beast\"'),\n",
       " (3,\n",
       "  '0.042*\"king\" + 0.031*\"israel\" + 0.027*\"land\" + 0.023*\"came\" + 0.019*\"children\" + 0.016*\"david\" + 0.016*\"people\" + 0.016*\"men\" + 0.014*\"judah\" + 0.013*\"went\"'),\n",
       " (4,\n",
       "  '0.037*\"ye\" + 0.018*\"jesus\" + 0.016*\"father\" + 0.014*\"christ\" + 0.012*\"hath\" + 0.012*\"us\" + 0.012*\"come\" + 0.011*\"man\" + 0.011*\"also\" + 0.010*\"things\"'),\n",
       " (5,\n",
       "  '0.018*\"let\" + 0.018*\"hath\" + 0.016*\"us\" + 0.015*\"thine\" + 0.015*\"man\" + 0.012*\"go\" + 0.011*\"hast\" + 0.010*\"may\" + 0.010*\"people\" + 0.010*\"hand\"'),\n",
       " (6,\n",
       "  '0.017*\"voice\" + 0.017*\"earth\" + 0.016*\"great\" + 0.013*\"heaven\" + 0.012*\"hear\" + 0.011*\"sea\" + 0.011*\"waters\" + 0.010*\"light\" + 0.008*\"heard\" + 0.008*\"cry\"'),\n",
       " (7,\n",
       "  '0.026*\"son\" + 0.025*\"offering\" + 0.017*\"priest\" + 0.015*\"two\" + 0.012*\"altar\" + 0.012*\"burnt\" + 0.012*\"fire\" + 0.010*\"sons\" + 0.010*\"offer\" + 0.009*\"offerings\"'),\n",
       " (8,\n",
       "  '0.024*\"hundred\" + 0.022*\"years\" + 0.019*\"three\" + 0.017*\"thousand\" + 0.015*\"five\" + 0.015*\"old\" + 0.015*\"thereof\" + 0.014*\"four\" + 0.013*\"word\" + 0.013*\"one\"'),\n",
       " (9,\n",
       "  '0.020*\"ye\" + 0.019*\"house\" + 0.013*\"made\" + 0.011*\"one\" + 0.009*\"set\" + 0.009*\"things\" + 0.009*\"also\" + 0.007*\"cubits\" + 0.007*\"hath\" + 0.006*\"gate\"')]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lsi = models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=10)\n",
    "\n",
    "# print the most contributing words (both positively and negatively) for each of the first ten topics\n",
    "lsi.print_topics(10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}