SPAM detector.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imoporting of required libraries\n",
    "import csv\n",
    "import numpy as np\n",
    "from nltk.tokenize  import word_tokenize,sent_tokenize\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer,PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
     "# function for pre-procesing of text\n",
    "def text_processing(text):\n",
    "    \n",
    "    # prepartion of tokens from the given sentences\n",
    "    words = [w for sent in nltk.sent_tokenize(text) for w in nltk.word_tokenize(sent)]\n",
    "    # remove noises from the file\n",
    "    stop_words=stopwords.words(\"english\")\n",
    "    tokens=[w for w in words if w not in stop_words]\n",
    "    # sometimes words less than donot have dominating importance\n",
    "    tokens=[w for w in  tokens if len(w)>=3]\n",
    "    \n",
    "    tokens=[word.lower() for word in tokens]\n",
    "     # lemmatizer helps in finding the root and replace word which helps in making text simpler for further processing\n",
    "    lemma=WordNetLemmatizer\n",
    "    last=[]\n",
    "    for word in tokens:\n",
    "        last.append(lemma.lemmatize(word,\"n\"))\n",
    "    pre_processed_text=\" \".join(tokens)\n",
    "    return pre_processed_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
     "# importing of smsspam collection file \n",
    "sms_data=open(\"C:\\\\Users\\\\DELL\\\\Desktop\\\\MY STUFF\\\\DATA SCIENCE\\\\DATA\\\\Raw data\\\\SMSSpamCollection\")\n",
    "sms_datadata=[]\n",
    "sms_datalabels=[]\n",
    "sms=csv.reader(sms_data,delimiter='\\t')\n",
    "\n",
    "for w in sms:\n",
    "    text=w[1]\n",
    "    # preparation of labels for the spam identification\n",
    "    # spam - spam ; ham- not spam\n",
    "    sms_datalabels.append(w[0])\n",
    "    sms_datadata.append(text_processing(text))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ham', 'ham', 'spam', 'ham', 'ham']\n",
      "['jurong point crazy.. available bugis great world buffet ... cine got amore wat ...', 'lar ... joking wif oni ...', 'free entry wkly comp win cup final tkts 21st may 2005 text 87121 receive entry question std txt rate apply 08452810075over18', 'dun say early hor ... already say ...', \"nah n't think goes usf lives around though\"]\n"
     ]
    }
   ],
   "source": [
    "# verification of data splitting into lables and input features\n",
    "print(sms_datalabels[0:5])\n",
    "print(sms_datadata[0:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# splitting of data into train and test set with a testset size of 0.3 randomly\n",
    "from sklearn.model_selection import train_test_split\n",
    "x_train,x_test,y_train,y_test=train_test_split(sms_datadata,sms_datalabels,test_size=0.3,random_state=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# conversion of input text data to numerical features by TFID vectorizer method\n",
    "# tfid ---  a method which is replaced by the probabilty value\n",
    "vectorizer = TfidfVectorizer(min_df=2, ngram_range=(1, 2), stop_words='english', strip_accents='unicode', norm='l2')\n",
    "# ngram_ range----pos tagger,unigram tagger and bigram tagger\n",
    "# stop_words-----removing noises\n",
    "#strip_accents------ Remove accents and perform other character normalization during the preprocessing step\n",
    "#normalization--l2\n",
    "X_train = vectorizer.fit_transform(x_train)\n",
    "\n",
    "X_test = vectorizer.transform(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.97      1.00      0.98      1451\n",
      "       spam       1.00      0.79      0.88       221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      1672\n",
      "\n",
      "[[1451    0]\n",
      " [  47  174]]\n"
     ]
    }
   ],
   "source": [
    "# initilisaing naive bayes classifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "# confusion matrix for easy understanding\n",
    "from sklearn.metrics import confusion_matrix\n",
    "# classification report for precision and recall,f1-score\n",
    "from sklearn.metrics import classification_report\n",
    "clf = MultinomialNB().fit(X_train, y_train)\n",
    "y_pred= clf.predict(X_test)\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "CR= classification_report(y_test,y_pred)\n",
    "print(CR)\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}