Introduction to TextBlob.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import textblob\n",
    "import requests\n",
    "import bs4\n",
    "import collections\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = requests.get('http://www.imsdb.com/scripts/Toy-Story.html').text\n",
    "soup = bs4.BeautifulSoup(html, 'html5lib')\n",
    "for tag in soup.body.find_all('script'):\n",
    "    tag.clear()\n",
    "text = soup.body.select('.scrtext')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "blob = textblob.TextBlob(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = textblob.TextBlob('run')\n",
    "s = b.words[0].synsets[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WordList(['geese', 'nunchuks', 'octopodes', 'mice', 'fish', 'dies'])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textblob.TextBlob('goose nunchuk octopus mouse fish die').words.pluralize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WordList(['goose', 'octopus', 'mouse', 'water', 'dice'])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textblob.TextBlob('gooses octopi mice water dice').words.singularize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('the', 'DT'), 1188),\n",
       " (('to', 'TO'), 470),\n",
       " (('and', 'CC'), 443),\n",
       " (('Buzz', 'NNP'), 388),\n",
       " (('a', 'DT'), 365),\n",
       " (('of', 'IN'), 347),\n",
       " (('WOODY', 'NNP'), 274),\n",
       " (('Woody', 'NNP'), 272),\n",
       " ((\"'s\", 'POS'), 236),\n",
       " (('I', 'PRP'), 217)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parts_of_speech = collections.Counter(map(tuple, chain.from_iterable(sentence.tags for sentence in blob.sentences)))\n",
    "parts_of_speech.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('What', 'WP'),\n",
       " (\"'s\", 'VBZ'),\n",
       " ('a', 'DT'),\n",
       " ('good', 'JJ'),\n",
       " ('sentence', 'NN')]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textblob.TextBlob(\"What's a good sentence?\").tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('He', 'PRP'),\n",
       " ('ran', 'VBD'),\n",
       " ('because', 'IN'),\n",
       " ('they', 'PRP'),\n",
       " ('scored', 'VBD'),\n",
       " ('a', 'DT'),\n",
       " ('run', 'NN')]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textblob.TextBlob(\"He ran because they scored a run.\").tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = textblob.TextBlob(\"The running runner ran because they made a run.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('The', 'DT'),\n",
       " ('running', 'NN'),\n",
       " ('runner', 'NN'),\n",
       " ('ran', 'VBD'),\n",
       " ('because', 'IN'),\n",
       " ('they', 'PRP'),\n",
       " ('made', 'VBD'),\n",
       " ('a', 'DT'),\n",
       " ('run', 'NN')]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WordList([])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.noun_phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sentence(\"Potato Head taped to it.\")"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blob.sentences[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WordList(['potato head'])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blob.sentences[5].noun_phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(Sentiment(polarity=1.0, subjectivity=0.75),\n",
       "  Sentence(\"Oh, this looks GREAT, Mom!\")),\n",
       " (Sentiment(polarity=1.0, subjectivity=1.0), Sentence(\"WOODY\n",
       "                     (whispering)\n",
       "              Just gather everyone up for a staff\n",
       "              meeting and be happy!!\")),\n",
       " (Sentiment(polarity=1.0, subjectivity=1.0), Sentence(\"WOODY\n",
       "              Be HAPPY!\")),\n",
       " (Sentiment(polarity=1.0, subjectivity=0.75), Sentence(\"Great!\")),\n",
       " (Sentiment(polarity=-1.0, subjectivity=0.9), Sentence(\"I hate to break up the\n",
       "              staff meeting, but THEY'RE HERE!\")),\n",
       " (Sentiment(polarity=1.0, subjectivity=0.3), Sentence(\"They're the best.\")),\n",
       " (Sentiment(polarity=-1.0, subjectivity=1.0),\n",
       "  Sentence(\"As a member of the elite\n",
       "              Universe Protection Unit of the\n",
       "              Space Ranger Corps, I protect the\n",
       "              galaxy from the threat of invasion\n",
       "              from the Evil Emperor Zurg, sworn\n",
       "              enemy of the Galactic Alliance.\")),\n",
       " (Sentiment(polarity=1.0, subjectivity=1.0), Sentence(\"HAMM\n",
       "              Oh, impressive wingspan.\")),\n",
       " (Sentiment(polarity=1.0, subjectivity=0.7800000000000001),\n",
       "  Sentence(\"Very good!\")),\n",
       " (Sentiment(polarity=1.0, subjectivity=0.9), Sentence(\"He can't fly!\"))]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted([(sentence.sentiment, sentence) for sentence in blob.sentences], key=lambda t: -abs(t[0].polarity))[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_counts = collections.Counter(map(tuple, chain.from_iterable(sentence.ngrams(2) for sentence in blob.sentences)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('of', 'the'), 152),\n",
       " (('to', 'the'), 85),\n",
       " (('on', 'the'), 76),\n",
       " (('in', 'the'), 73),\n",
       " (('out', 'of'), 66),\n",
       " (('POTATO', 'HEAD'), 61),\n",
       " (('into', 'the'), 54),\n",
       " (('Potato', 'Head'), 47),\n",
       " (('I', \"'m\"), 47),\n",
       " (('from', 'the'), 45)]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigram_counts.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "152"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigram_counts.most_common(1)[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "trigram_counts = collections.Counter(map(tuple, chain.from_iterable(sentence.ngrams(3) for sentence in blob.sentences)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('out', 'of', 'the'), 32),\n",
       " (('Woody', 'and', 'Buzz'), 22),\n",
       " (('ANDY', \"'S\", 'BEDROOM'), 18),\n",
       " (('WOODY', 'VOICE', 'BOX'), 15),\n",
       " (('on', 'the', 'floor'), 14),\n",
       " (('I', 'ca', \"n't\"), 13),\n",
       " (('MRS', 'DAVIS', 'O.S'), 12),\n",
       " (('I', 'do', \"n't\"), 12),\n",
       " (('Buzz', 'and', 'Woody'), 12),\n",
       " (('of', 'the', 'truck'), 12)]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigram_counts.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigram_counts.most_common(1)[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('sight', 'of')]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.choices(list(bigram_counts.elements()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'lassoing'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beginning = random.choice([t for t in bigram_counts.elements()])[0]\n",
    "beginning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the'"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "second = random.choice([t for t in bigram_counts.elements() if t[0] == beginning])[1]\n",
    "second"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Buzz'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "third = random.choice([t for t in bigram_counts.elements() if t[0] == second])[1]\n",
    "third"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'makes'"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fourth = random.choice([t for t in bigram_counts.elements() if t[0] == third])[1]\n",
    "fourth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fifth = random.choice([t for t in bigram_counts.elements() if t[0] == fourth])[1]\n",
    "fifth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lassoing the Buzz makes a\n"
     ]
    }
   ],
   "source": [
    "print(beginning, second, third, fourth, fifth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}