Capstone Project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#importing neccessary libraries\n",
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import bs4\n",
    "from datetime import datetime\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing relevant spreadsheets\n",
    "approval = pd.read_csv('C:/Users/brian/Documents/Python/approval_polllist.csv', parse_dates=['enddate'])\n",
    "sentiment_corpus = pd.read_csv('C:/Users/brian/Documents/Python/nrc.csv')\n",
    "pos_neg_corpus = pd.read_csv('C:/Users/brian/Documents/Python/bing.csv')\n",
    "policy_corpus = pd.read_excel('C:/Users/brian/Documents/Policy Corpus.xlsx', header= 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving the URL that contain all other speech URL\n",
    "links_from_2017 = 'http://www.presidency.ucsb.edu/ws/index.php?month=&year=2017'\n",
    "links_from_2018 = 'http://www.presidency.ucsb.edu/ws/index.php?month=&year=2018'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#writing the function to pull out the URL for each speech\n",
    "links = []\n",
    "def get_speeches(url):\n",
    "    url_links = requests.get(url)\n",
    "    content = url_links.content\n",
    "    \n",
    "    #parse the url through html parser Beautiful Soup\n",
    "    html = bs4.BeautifulSoup(content, 'html.parser')\n",
    "    \n",
    "    #Pull out the table from the url that contains the info we want\n",
    "    html = html.contents[-1]\n",
    "    \n",
    "    #pulls out the number of documents this particular url contains\n",
    "    records = html.find_all('span')[4]\n",
    "    x = records.text[-3:]\n",
    "    x = int(x)\n",
    "\n",
    "    #filter out superflous lines of code that have to deal with html formatting\n",
    "    section = html.find_all('tr')\n",
    "    body = section[3:-2]\n",
    "        \n",
    "    #for loop that filters out any other author that gave the speeck besides Trump\n",
    "    for i in range(x):\n",
    "        if 'Trump' in str(body[i].contents):\n",
    "\n",
    "    #code that pulls out the query index and attaches it to the main body of the url\n",
    "            link = 'http://www.presidency.ucsb.edu/ws/' + str(body[i].find('a')['href'])\n",
    "    \n",
    "    #appends the new url to the list links         \n",
    "            links.append(link)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pull speech URLs from 2017\n",
    "get_speeches(links_from_2017)\n",
    "#pull speeche URLs from 2018\n",
    "get_speeches(links_from_2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "texts = []\n",
    "dates = []\n",
    "titles = []\n",
    "#for loop to loop over ever speech and pull out the speeches\n",
    "for i in links:\n",
    "#pull and get url then parse url\n",
    "    doc_url = requests.get(i)\n",
    "    doc = doc_url.content\n",
    "    parsed = bs4.BeautifulSoup(doc, 'html.parser')\n",
    "        \n",
    "    #get the title of the speech\n",
    "    title = str(parsed.title)[24:-8]\n",
    "    titles.append(title)\n",
    "\n",
    "    #get the text of the speech\n",
    "    text = [i.text for i in parsed.find_all(class_ = 'displaytext')]\n",
    "    text1 = [i.text for i in parsed.find_all('p')]\n",
    "    string = text + text1\n",
    "    words = ' '.join(string)\n",
    "    texts.append(words)\n",
    "    \n",
    "    #get the date of the speech\n",
    "    date = [i.text for i in parsed.find_all(class_ = 'docdate')]\n",
    "    date = datetime.strptime(date[0], '%B %d, %Y')\n",
    "    date1 = datetime.strftime(date, '%m/%d/%Y')\n",
    "    dates.append(date1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating a corpus of stop words and adding our own words to it\n",
    "stop_words = stopwords.words('english')\n",
    "additional = ['us', 'president', 'trump', 'donald', 'vice']\n",
    "for i in additional:\n",
    "    stop_words.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#we are going to clean up the text by getting rid of most puncuation and making sure all words are lower case\n",
    "#we the create tokens out of each word of each speech\n",
    "#by the end we have a list of single word tokens nested inside a list that represents the speech\n",
    "clean_speeches = []\n",
    "tokens = []\n",
    "for i in texts:\n",
    "    x = i.replace(',', ' ')\n",
    "    x = x.replace('.', ' ')\n",
    "    x = x.replace('?', ' ')\n",
    "    x = x.replace(\"\\ \", ' ')\n",
    "    x = x.replace(\"'\", ' ')\n",
    "    x = x.replace(':', ' ')\n",
    "    x = x.replace(';', ' ')\n",
    "    x = x.replace('!', ' ')\n",
    "    x = x.replace('-', ' ')\n",
    "    x = x.replace('—', ' ')\n",
    "    x = x.replace('\"', ' ')\n",
    "    x = x.replace('*', ' ')\n",
    "    x = x.replace('(', ' ')\n",
    "    x = x.replace(')', ' ')\n",
    "    x = x.replace('&', ' ')\n",
    "    x = x.replace('#', ' ')\n",
    "    x = x.replace('%', ' ')\n",
    "    x = x.replace('$', ' ')\n",
    "    x = x.replace('[ ', ' ')\n",
    "    x = x.replace('] ', ' ')\n",
    "    clean_speeches.append(x)\n",
    "    tokens.append(word_tokenize(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this is our function to remove stop words later we can have the length of the speeches without stop words\n",
    "def remove_stop(speeches):\n",
    "    ### this function is meant to remove stop words from a w list of speeches\n",
    "    cleaned = []\n",
    "    for a in speeches:\n",
    "        pos = []\n",
    "        for i, x in enumerate(a):\n",
    "            if x in stop_words:\n",
    "                pos.append(i)\n",
    "        for i in reversed(pos):\n",
    "            a.pop(i)\n",
    "        cleaned.append(a)\n",
    "    return cleaned\n",
    "no_stop = remove_stop(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#running the speeches through the count vectorizer to create the document-term matrix\n",
    "#we pass this matrix through Pandas Data Frame to get the bag-of-words model\n",
    "vectorizer = CountVectorizer(analyzer='word', stop_words=stop_words, strip_accents='ascii')\n",
    "X = vectorizer.fit_transform(clean_speeches)\n",
    "names = vectorizer.get_feature_names()\n",
    "speeches_df = pd.DataFrame(X.toarray(), columns=names, index=range(len(X.toarray())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#here is the approval data frame that will be merged with the speech df\n",
    "#we have to change the date column to a string format in order to perform a join later\n",
    "#the date type is a pd.Timestamp which does not want to join to a date time object\n",
    "mean_adj_approval = approval.groupby('enddate')[['adjusted_approve', 'adjusted_disapprove']].mean()\n",
    "mean_adj_approval = mean_adj_approval.reset_index()\n",
    "mean_adj_approval['enddate'] = mean_adj_approval['enddate'].dt.strftime('%m/%d/%Y')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#here is the function to count the number of words from our sentiment corpora \n",
    "def sentiment_variable(corpus_df, sentiment= 'feeling'):\n",
    "    temp = corpus_df[corpus_df['sentiment'] == sentiment]\n",
    "    temp = temp['word']\n",
    "    temp = list(temp)\n",
    "    df = speeches_df.T\n",
    "    sent_filter = df.reindex(temp)\n",
    "    sent_count = np.sum(sent_filter)\n",
    "    return sent_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this function counts the number of words in our policy corpus\n",
    "#both functions return a single column of counts for each document\n",
    "def policy_variable(corpus_df, policy= 'issue'):\n",
    "    temp = corpus_df[corpus_df['policy'] == policy]\n",
    "    temp = temp['word']\n",
    "    temp = list(temp)\n",
    "    df = speeches_df.T\n",
    "    sent_filter = df.reindex(temp)\n",
    "    sent_count = np.sum(sent_filter)\n",
    "    return sent_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#iterate our sentiment function for each sentiment variable \n",
    "trust = sentiment_variable(sentiment_corpus, 'trust')\n",
    "fear = sentiment_variable(sentiment_corpus, 'fear')\n",
    "sadness = sentiment_variable(sentiment_corpus, 'sadness')\n",
    "anger = sentiment_variable(sentiment_corpus, 'anger')\n",
    "surprise = sentiment_variable(sentiment_corpus, 'surprise')\n",
    "disgust = sentiment_variable(sentiment_corpus, 'disgust')\n",
    "joy = sentiment_variable(sentiment_corpus, 'joy')\n",
    "anticipation = sentiment_variable(sentiment_corpus, 'anticipation')\n",
    "positive = sentiment_variable(pos_neg_corpus, 'positive')\n",
    "negative = sentiment_variable(pos_neg_corpus, 'negative')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#iterate our policy function over each policy variable\n",
    "gender = policy_variable(policy_corpus, policy=\"woman's rights & sexual orientation\")\n",
    "constitution = policy_variable(policy_corpus, policy= 'constitutional rights')\n",
    "social = policy_variable(policy_corpus, policy = 'socio-cultural issues')\n",
    "environment = policy_variable(policy_corpus, policy = 'environmental issues')\n",
    "military = policy_variable(policy_corpus, policy= 'military and national security')\n",
    "foreign_relations = policy_variable(policy_corpus, policy= 'foreign relations')\n",
    "healthcare = policy_variable(policy_corpus, policy = 'healthcare & welfare')\n",
    "economy = policy_variable(policy_corpus, policy= 'us economy')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pull each variable into a dictionary\n",
    "speech_data = {}\n",
    "for i, j in enumerate(dates):\n",
    "    speech_data[i] = i+1, j, titles[i], len(no_stop[i]), len(pd.unique(no_stop[i])), positive[i], negative[i], trust[i], fear[i], sadness[i], anger[i], surprise[i], disgust[i], joy[i], anticipation[i], gender[i], constitution[i], social[i], environment[i], military[i], foreign_relations[i], healthcare[i], economy[i]\n",
    "speech_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create column names\n",
    "var_names = ['Doc Number', 'Date', 'Title', 'Speech Length', 'Unique Words', 'Positive', 'Negative', 'Trust', 'Fear', 'Sadness', 'Anger', 'Surprise', 'Disgust', 'Joy', 'Anticipation', 'Womens Issues','Constitution Issues', 'Social Issues', 'Environmental Issues', 'Military/Defense', ' Foreign Relations', 'Healthcare/Welfare', 'US Economy'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#create a data frame from our variables and clean it\n",
    "speeches_df = pd.DataFrame(speech_data)\n",
    "speeches_df = speeches_df.T\n",
    "speeches_df.columns = var_names\n",
    "speeches_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#need to join our speech data frame with the approval data\n",
    "cleaned = pd.merge(speeches_df, mean_adj_approval, left_on= 'Date', right_on='enddate', how = 'left')\n",
    "cleaned[['adjusted_approve', 'adjusted_disapprove']] = cleaned[['adjusted_approve', 'adjusted_disapprove']].bfill()\n",
    "cleaned.pop('enddate')\n",
    "cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we finally export the final data frame to an excel spreadsheet.\n",
    "cleaned.to_excel('C:/Users/brian/Documents/Python/speech_data.xlsx', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#def remove_stop(speeches):\n",
    "    ### this function is meant to remove stop words from a w list of speeches\n",
    "    #cleaned = []\n",
    "    #for a in speeches:\n",
    "        #pos = []\n",
    "        #for i, x in enumerate(a):\n",
    "            #if x in stop_words:\n",
    "                #pos.append(i)\n",
    "        #for i in reversed(pos):\n",
    "            #a.pop(i)\n",
    "        #cleaned.append(a)\n",
    "    #return cleaned\n",
    "#FUNCTION WAS WRITTED TO ITERATE THROUGH LIST OF LISTS AND REMOVE STOP WORDS\n",
    "\n",
    "#THIS CHUNK OF CODE SHOULD STILL SERVE A PURPOSE AND IS SOME OF THE BEST CODE YOU HAVE WRITTEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#THE CODE IN # IS VERY TIME CONSUMING TO RUN SO I HAVE BLOCKED IT OUT\n",
    "#it produces a term frequency plot\n",
    "\n",
    "#distinct_words = []\n",
    "#for i in range(842):\n",
    "    #vectorizer = CountVectorizer(analyzer='word', stop_words=stop_words, strip_accents='ascii', min_df=i/1020)\n",
    "    #X = vectorizer.fit_transform(clean_speeches)\n",
    "    #distinct_words.append(X.toarray().shape[1])\n",
    "#_ = plt.plot(distinct_words)\n",
    "#_ = plt.ylabel('# of Unique Words')\n",
    "#_ = plt.xlabel('# of Documents')\n",
    "#_ = plt.title('Term Frequency Plot')\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#viewing how much percentage of a speech is disregarded when combining it with the corpus we use\n",
    "#I THINK THAT IT IS IMPORTANT TO NOTE HOW MUCH OF THE SPEECH WE ARE LOOSING WHEN WE ARE COMPARING IT TO THE DIFFERENT\n",
    "#VARIABLE CORPORA\n",
    "#import numpy as np\n",
    "#test = pd.DataFrame(no_stop_words, columns=['Word'])\n",
    "#test2 = pd.merge(test, sentiment_corpus, left_on= 'Word', right_on='word', how='left')\n",
    "#t2 = test2.isna()\n",
    "#np.sum(t2['sentiment'])/len(t2['sentiment'])"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}