model_1_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import nltk\n",
    "import re\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "import contractions\n",
    "from collections import Counter\n",
    "from num2words import num2words\n",
    "import numpy as np\n",
    "import tweepy\n",
    "from sklearn.metrics import accuracy_score,confusion_matrix,classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('tweet_timeline.pkl','rb') as f:\n",
    "        tweet_timeline_dict = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "teams_hash_dict={\"#mi\":\"mumbai indians\",\"#csk\":\"chennai super kings\",\"#rcb\":\"royal challengers bangalore\",\"#kxip\":\"kings xi punjab\",\n",
    "            \"#kkr\":\"kolkata knight riders\",\"#dc\":\"delhi capitals\",\"#srh\":\"sunrisers hyderabad\",\"#rr\":\"rajasthan royals\",\"#delhicapitals\":\"delhi capitals\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-03-23 a win to start the season for csk. @chennaiipl #ipl\n",
      "2019-03-24 match 3. it is all over! delhi capitals won by 37 runs  #mivdc #vivoipl\n",
      "2019-03-25 match 4. it is all over! kings xi punjab won by 14 runs  #rrvkxip #vivoipl\n",
      "2019-03-26 match 5. it is all over! chennai super kings won by 6 wickets  #dcvcsk #vivoipl\n",
      "2019-03-27 match 6. it is all over! kolkata knight riders won by 28 runs  #kkrvkxip #vivoipl\n",
      "2019-03-28 match 7. it is all over! mumbai indians won by 6 runs  #rcbvmi #vivoipl\n",
      "2019-03-29 match 8. it is all over! sunrisers hyderabad won by 5 wickets  #srhvrr #vivoipl\n",
      "2019-03-30 delhi capitals win! kagiso rabada concedes only 7 runs in the super over and closes the game for @delhicapitals. #dcvkkr #ipl2019\n",
      "2019-03-31 match 12. it is all over! chennai super kings won by 8 runs  #cskvrr #vivoipl\n",
      "2019-04-01 kings xi punjab have done the unbelievable! delhi capitals came down from 144/3 to 152/10! they lost 7 wickets for just 9 runs! kxip beat dc by 14 runs! #kxipvdc #ipl2019\n",
      "2019-04-02 shreyas gopal -- 4-1-12-3 -- 1 0 w 2 4 0 -- 0 0 w 0 0 0 -- w 0 0 0 1 0 -- 0 1 1 1 0 1 #ipl2019 #rrvrcb\n",
      "2019-04-03 congratulations to @mipaltan on becoming the first team to win 100 matches in the #ipl. #mivcsk #ipl2019\n",
      "2019-04-04 nabi, bairstow take @sunrisers to the top of the #ipl2019 table  #ipl2019 #dcvsrh\n",
      "2019-04-05  andre russell has smashed it kkr's way  #rcbvskkr #ipl2019\n",
      "2019-04-06 match report: alzarri joseph creates history on his #ipl debut; @mipaltan bowl out @sunrisers for 96 to win by 40 runs. #srhvmi #ipl2019\n",
      "2019-04-07 match 21. it is all over! kolkata knight riders won by 8 wickets  #rrvkkr #vivoipl\n",
      "2019-04-08 lowest total successfully defended in ipl matches in mohali: 150 by srh vs kxip, 2013 150 by srh vs kxip, 2015 today's defence is exactly 150 as well .. #ipl2019 #kxipvsrh\n",
      "2019-04-09 as comprehensive as it can get for @chennaiipl at the chepauk. csk beat kkr by 7 wickets and 16 balls to spare #cskvkkr #vivoipl\n",
      "2019-04-10 5 players with the most consecutive appearances for a team in the ipl:  #ipl #ipl2019 mumbai indians #mivkxip\n",
      "2019-04-11 breaking: ms dhoni has been fined 50% of his match fees for breaching the ipl code of conduct during csk's match against rajasthan royals in jaipur #rrvcsk #ipl2019\n",
      "2019-04-12 news alert: dale steyn replaces nathan coulter-nile in the rcb squad for #ipl2019 season. #ipl #ipl2019 #rcb\n",
      "2019-04-13 a victory at mohali as @rcbtweets register their first win of the #vivoipl 2019 season \n",
      "2019-04-14 101-3 106-4 106-5 110-6 110-7 112-8 116-9 116 all out  #srhvdc #ipl\n",
      "2019-04-15 pawan negi to hardik pandya: 18.1 >> dot 18.2 >> six 18.3 >> four 18.4 >> four 18.5 >> six 18.5 >> wide 18.6 >> single hardik finishes it for mumbai indians as they romp home by 5 wickets  #mivrcb #ipl2019\n",
      "2019-04-16 match 32. it is all over! kings xi punjab won by 12 runs  #kxipvrr #vivoipl\n",
      "2019-04-17 wicket! warner tries to get another boundary but finds du plessis at mid-off chennai super kings get their first, sunrisers hyderabad 66-1 (5.4) #srhvcsk #ipl2019\n",
      "2019-04-18 mumbai indians take the second spot. #ipl #ipl2019 #dcvmi\n",
      "2019-04-19 virat kohli -- 5th ipl century !! 57 balls | 9 fours | 4 sixes -- he is lifted rcb to 213/4 (20 overs) #ipl2019 #kkrvrcb\n",
      "2019-04-20 just in: steve smith to replace ajinkya rahane as the captain of rajasthan royals for the remainder of #ipl2019\n",
      "2019-04-21 royal challengers bangalore have won a thriller against chennai super kings by 1 run!! absolute nail-biter! #rcbvcsk #ipl2019\n",
      "2019-04-22 ajinkya rahane's stats against @delhicapitals look even more impressive after his unbeaten ton tonight. rajasthan royals #rrvdc #ipl2019\n",
      "2019-04-23 guess who is back on top of the table? 'csk, csk'  #cskvsrh #ipl2019\n",
      "2019-04-24 match 42. it is all over! royal challengers bangalore won by 17 runs  #rcbvkxip #vivoipl\n",
      "2019-04-25 match 43. it is all over! rajasthan royals won by 3 wickets  #kkrvrr #vivoipl\n",
      "2019-04-26 no dhoni, no cheer for csk  #ipl2019 #cskvmi\n",
      "2019-04-27 csk becomes the first team to qualify for ipl 2019 playoffs. #ipl2019\n",
      "2019-04-28 what a season it has been for andre russell  #kkrvmi #ipl2019\n",
      "2019-04-29 match 48. it is all over! sunrisers hyderabad won by 45 runs  #srhvkxip #vivoipl\n",
      "2019-04-30 rcb are the first team out of the #ipl2019 playoffs race  #rcbvrr\n",
      "2019-05-01 match 50. it is all over! chennai super kings won by 80 runs  #cskvdc #vivoipl\n",
      "2019-05-02 mumbai indians beat sunrisers hyderabad in the super over and qualify for the playoffs!! #mivsrh #srhvmi #ipl2019\n",
      "2019-05-03 fastest individual 50 in #ipl2019: 17 - h pandya (91) - mi v kkr (kolkata) 18 - pant (78*) - dc v mi (mumbai) 21 - russell (65) - kkr v rcb (kolkata) 22 - pollard (83) - mi v kxip (mumbai) 23 - russell (62) - kkr v dc (delhi) 23 - curran - kxip v kkr (mohali) - today #kxipvkkr\n",
      "2019-05-04 royal challengers bangalore beat sunrisers hyderabad by 4 wickets and end their season on a high! #rcbvsrh #ipl2019\n",
      "2019-05-05 mumbai indians beat kolkata knight riders and finish the league stage at top of the table! they will face chennai super kings on 7 may in chennai! #mivkkr #ipl2019\n",
      "2019-05-06 match 1. it is all over! trailblazers won by 2 runs  #tblvsno #ipl #wipl\n",
      "2019-05-07 csk are down, but not out...  #ipl2019 #mivcsk\n",
      "2019-05-08 first win for delhi capitals in an ipl knockout game. previous times 2008: lost semis to rr by 105 runs 2009: lost semis to deccan by six wkts 2012: lost qualifier 1 to kkr by 18 runs 2012: lost qualifier 2 to csk by 86 runs #dcvsrh #ipl2019\n",
      "2019-05-09 match 3. it is all over! supernovas won by 12 runs  #snovvel #ipl #wipl\n",
      "2019-05-10 csk into the ipl final for the 8th time in their 10 seasons #cskvdc\n",
      "2019-05-11 team circle of cricket congratulates @imharmanpreet on leading #supernovas to victory in the women's t20 challenge cup. #wipl\n",
      "2019-05-12 mi beat csk for the fourth time this season to become the most successful team in the ipl #mivcsk #ipl2019\n"
     ]
    }
   ],
   "source": [
    "for key,val in tweet_timeline_dict.items():\n",
    "    val=val.lower()\n",
    "    for team in list(teams_hash_dict):\n",
    "        val=re.sub(\"{} \".format(team),\"{} \".format(teams_hash_dict[team]),val)\n",
    "    tweet_timeline_dict[key]=val\n",
    "    print(key,val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#connection to Twitter API using tweepy\n",
    "\n",
    "consumer_key = \"CtmrTSUsnIRHKSgj4ktqK4NKb\"\n",
    "consumer_secret = \"1MNfKmptTsrJcq7WMYxOoVwEgFCK5DNMQeC43IZDkPucPObCnZ\"\n",
    "access_key = \"1231240089600057344-rqYnkeVvMaB1XwfvrERfI7aF38r69L\"\n",
    "access_secret = \"KmRfAoexHxfkHKJaC3hwB0sgtfocyz58IqdzYiWeaXSGO\"\n",
    "# Authorization to consumer key and consumer secret\n",
    "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "\n",
    "# Access to user's access key and access secret\n",
    "auth.set_access_token(access_key, access_secret)\n",
    "\n",
    "# Calling api\n",
    "api = tweepy.API(auth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_username(username):\n",
    "    # 200 tweets to be extracted\n",
    "    #number_of_tweets=1\n",
    "    try:\n",
    "        tweets = api.user_timeline(screen_name=username)\n",
    "\n",
    "        #tweets_for_csv = [tweet.text for tweet in tweets]  # CSV file created\n",
    "        for tweet in tweets:\n",
    "            #print(tweet)\n",
    "            return tweet._json['user']['name']\n",
    "    except tweepy.error.TweepError:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "teams_dict={\"mi\":\"mumbai indians\",\"csk\":\"chennai super kings\",\"rcb\":\"royal challengers bangalore\",\"kxip\":\"kings xi punjab\",\n",
    "            \"kkr\":\"kolkata knight riders\",\"dc\":\"delhi capitals\",\"srh\":\"sunrisers hyderabad\",\"rr\":\"rajasthan royals\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix_list=[\"mumbai\",\"chennai\",\"royal\",\"kings\",\"kolkata\",\"delhi\",\"sunrisers\",\"rajasthan\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "punct=string.punctuation\n",
    "english_stopwords=stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key,val in tweet_timeline_dict.items():\n",
    "        final_tokens = []\n",
    "\n",
    "        #remove url from tweet text \n",
    "        tweet_text=re.sub(r\"(http|https)\\S+\", \"\", val)\n",
    "        tweet_text=re.sub(r'\\B#\\w*[a-zA-Z]+\\w*',\"\",tweet_text)\n",
    "\n",
    "        #Fix contractions\n",
    "        tweet_text=contractions.fix(tweet_text)\n",
    "        #print(tweet_text)\n",
    "\n",
    "\n",
    "        #tokenize\n",
    "        token_list=nltk.word_tokenize(tweet_text)\n",
    "        i=0\n",
    "        while i<len(token_list):\n",
    "            if(token_list[i]=='@'): #replace twitter handle with screen name\n",
    "                temp=token_list[i]+token_list[i+1]\n",
    "                #print(temp)\n",
    "                if get_username(temp) is not None:\n",
    "                    #print(get_username(temp))\n",
    "                    temp_list=nltk.word_tokenize(get_username(temp))\n",
    "                    for t in temp_list:\n",
    "                        final_tokens.append(t.lower())\n",
    "                i=i+1\n",
    "            #elif token_list[i].isnumeric(): #ignore digits from token list\n",
    "            #    temp = num2words(token_list[i])\n",
    "                #final_tokens.append(temp)\n",
    "            #    i=i+1\n",
    "            else:\n",
    "                flag=1\n",
    "                #remove punctuation from each string \n",
    "                str_temp = token_list[i].translate(str.maketrans('', '', punct))\n",
    "                #print(str_temp)\n",
    "\n",
    "                #convert to lower case\n",
    "                str_temp=str_temp.lower()\n",
    "\n",
    "                #replace team abbreviations with team name\n",
    "                for team,team_name in teams_dict.items():\n",
    "                    if str_temp==team:\n",
    "                        temp_list = nltk.word_tokenize(team_name)\n",
    "                        #print(temp_list)\n",
    "                        for t in temp_list:\n",
    "                             final_tokens.append(t)\n",
    "                        flag=0\n",
    "                if flag==0:\n",
    "                    i=i+1\n",
    "                    continue\n",
    "\n",
    "                #stop word and punctuation tokens removal\n",
    "                if str_temp not in english_stopwords:\n",
    "                    #str_temp = str_temp.replace(\"'\", \"\")\n",
    "                    final_tokens.append(str_temp)\n",
    "                i+=1\n",
    "        tweet_timeline_dict[key]=final_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('model_1_evaluation_data.pkl','rb') as f:\n",
    "        schedule_dict = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81.25% match with ground truth\n"
     ]
    }
   ],
   "source": [
    "matches=0\n",
    "y_pred=[]\n",
    "y_true=[\"Match\" for x in range(len(list(schedule_dict)))]\n",
    "for key,val in schedule_dict.items():\n",
    "    intersect=set(val).intersection(set(tweet_timeline_dict[key]))\n",
    "    if(len(intersect)<=3):\n",
    "        if tweet_timeline_dict[key][0] in prefix_list or len(set([\"win\",\"won\",\"romp\",\"wins\",\"match\"]).intersection(intersect))>0:\n",
    "            matches+=1\n",
    "            y_pred.append(\"Match\")\n",
    "        else:\n",
    "            y_pred.append(\"No Match\")\n",
    "    else:\n",
    "        matches+=1\n",
    "        y_pred.append(\"Match\")\n",
    "print(\"{}% match with ground truth\".format(matches/len(list(schedule_dict))*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  81.25 %\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \",accuracy_score(y_true, y_pred)*100,'%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix: \n",
      " [[39  9]\n",
      " [ 0  0]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Confusion Matrix: \\n\",confusion_matrix(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "       Match       1.00      0.81      0.90        48\n",
      "    No Match       0.00      0.00      0.00         0\n",
      "\n",
      "    accuracy                           0.81        48\n",
      "   macro avg       0.50      0.41      0.45        48\n",
      "weighted avg       1.00      0.81      0.90        48\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\aksha\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "print(\"Classification Report: \\n\",classification_report(y_true, y_pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}