13. Наивный Байес.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({(False, False): 704, (True, True): 101, (True, False): 38, (False, True): 33})\n",
      "самые спамные среди не-спамных (spammiest_hams) [('Attn programmers: support offered [FLOSS-Sarai Initiative]', False, 0.9756129605142172), ('2000+ year old Greek computer reinterpreted', False, 0.9835355008104623), ('What to look for in your next smart phone (Tech Update)', False, 0.9898719206903337), ('[ILUG-Social] Re: Important - reenactor insurance needed', False, 0.999534905780338), ('[ILUG-Social] Re: Important - reenactor insurance needed', False, 0.999534905780338)]\n",
      "\n",
      "самые не-спамные среди спамных (hammiest_spams) [('Re: girls', True, 0.0009525186158413672), ('Introducing Chase Platinum for Students with a 0% Introductory APR', True, 0.0012566691211097502), ('.Message report from your contact page....//ytu855 rkq', True, 0.0015109358288607473), ('Testing a system, please delete', True, 0.0026920538836854905), ('Never pay for the goodz again (8SimUgQ)', True, 0.005911623221924926)]\n",
      "\n",
      "самые спамные слова [('year', 0.028767123287671233, 0.00022893772893772894), ('rates', 0.031506849315068496, 0.00022893772893772894), ('sale', 0.031506849315068496, 0.00022893772893772894), ('systemworks', 0.036986301369863014, 0.00022893772893772894), ('money', 0.03972602739726028, 0.00022893772893772894)]\n",
      "\n",
      "самые не-спамные слова [('spambayes', 0.0013698630136986301, 0.04601648351648352), ('users', 0.0013698630136986301, 0.036401098901098904), ('razor', 0.0013698630136986301, 0.030906593406593408), ('zzzzteana', 0.0013698630136986301, 0.029075091575091576), ('sadev', 0.0013698630136986301, 0.026785714285714284)]\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "# naive_bayes.py\n",
    "\n",
    "import sys\n",
    "sys.path.append(\"../code-python3-ru\")\n",
    "\n",
    "from collections import Counter, defaultdict\n",
    "from lib.machine_learning import split_data\n",
    "import math, random, re, glob\n",
    "\n",
    "def tokenize(message):\n",
    "    message = message.lower()                       # преобразовать с строчные\n",
    "    all_words = re.findall(\"[a-z0-9']+\", message)   # извлечь слова\n",
    "    return set(all_words)                           # удалить повторы\n",
    "\n",
    "def count_words(training_set):\n",
    "    \"\"\"обучающая выборка состоит из пар (сообщение, спам?\"\"\"\n",
    "    counts = defaultdict(lambda: [0, 0])\n",
    "    for message, is_spam in training_set:\n",
    "        for word in tokenize(message):\n",
    "            counts[word][0 if is_spam else 1] += 1\n",
    "    return counts\n",
    "\n",
    "def word_probabilities(counts, total_spams, total_non_spams, k=0.5):\n",
    "    \"\"\"преобразовать частотности word_counts в список триплетов\n",
    "    слово w, p(w | spam) и p(w | ~spam)\"\"\"\n",
    "    return [(w,\n",
    "             (spam + k) / (total_spams + 2 * k),\n",
    "             (non_spam + k) / (total_non_spams + 2 * k))\n",
    "             for w, (spam, non_spam) in counts.items()]\n",
    "\n",
    "def spam_probability(word_probs, message):\n",
    "    message_words = tokenize(message)\n",
    "    log_prob_if_spam = log_prob_if_not_spam = 0.0\n",
    "\n",
    "    for word, prob_if_spam, prob_if_not_spam in word_probs:\n",
    "\n",
    "        # если СЛОВО в сообщении появляется, то\n",
    "        # добавить лог-вероятность встретить его в сообщении\n",
    "        if word in message_words:\n",
    "            log_prob_if_spam += math.log(prob_if_spam)\n",
    "            log_prob_if_not_spam += math.log(prob_if_not_spam)\n",
    "\n",
    "        # если СЛОВО в сообщении не появляется, то\n",
    "        # добавить лог-вероятность НЕ встретить его в сообщении,\n",
    "        # вычисляемое как log(1 - вероятность встретить его в сообщении)\n",
    "        else:\n",
    "            log_prob_if_spam += math.log(1.0 - prob_if_spam)\n",
    "            log_prob_if_not_spam += math.log(1.0 - prob_if_not_spam)\n",
    "\n",
    "    prob_if_spam = math.exp(log_prob_if_spam)\n",
    "    prob_if_not_spam = math.exp(log_prob_if_not_spam)\n",
    "    return prob_if_spam / (prob_if_spam + prob_if_not_spam)\n",
    "\n",
    "\n",
    "class NaiveBayesClassifier:\n",
    "\n",
    "    def __init__(self, k=0.5):\n",
    "        self.k = k\n",
    "        self.word_probs = []\n",
    "\n",
    "    def train(self, training_set):\n",
    "\n",
    "        # подсчитать спамные и неспамные сообщения\n",
    "        num_spams = len([is_spam\n",
    "                         for message, is_spam in training_set\n",
    "                         if is_spam])\n",
    "        num_non_spams = len(training_set) - num_spams\n",
    "\n",
    "        # пропустить обучающую выборку через \"конвейер\"\n",
    "        word_counts = count_words(training_set)\n",
    "        self.word_probs = word_probabilities(word_counts,\n",
    "                                             num_spams,\n",
    "                                             num_non_spams,\n",
    "                                             self.k)\n",
    "\n",
    "    def classify(self, message):\n",
    "        return spam_probability(self.word_probs, message)\n",
    "\n",
    "\n",
    "def get_subject_data(path):\n",
    "\n",
    "    data = []\n",
    "\n",
    "    # регулярное выражение для удаления начального слова \"Subject:\" \n",
    "    # и любых пробельных символов после него\n",
    "    subject_regex = re.compile(r\"^Subject:\\s+\")\n",
    "\n",
    "    # glob.glob возвращает имена файлов,\n",
    "    # соответствующие шаблону поиска по указанному пути    \n",
    "    for fn in glob.glob(path):\n",
    "        is_spam = \"ham\" not in fn\n",
    "\n",
    "        with open(fn,'r',encoding='ISO-8859-1') as file:\n",
    "            for line in file:\n",
    "                if line.startswith(\"Subject:\"):\n",
    "                    subject = subject_regex.sub(\"\", line).strip()\n",
    "                    data.append((subject, is_spam))\n",
    "\n",
    "    return data\n",
    "\n",
    "def p_spam_given_word(word_prob):\n",
    "    word, prob_if_spam, prob_if_not_spam = word_prob\n",
    "    return prob_if_spam / (prob_if_spam + prob_if_not_spam)\n",
    "\n",
    "def train_and_test_model(path):\n",
    "\n",
    "    data = get_subject_data(path)\n",
    "    random.seed(0)      # чтобы получить повторимые ответы\n",
    "    train_data, test_data = split_data(data, 0.75)\n",
    "\n",
    "    classifier = NaiveBayesClassifier()\n",
    "    classifier.train(train_data)\n",
    "\n",
    "    classified = [(subject, is_spam, classifier.classify(subject))\n",
    "              for subject, is_spam in test_data]\n",
    "\n",
    "    counts = Counter((is_spam, spam_probability > 0.5) # (фактическая, прогнозная)\n",
    "                     for _, is_spam, spam_probability in classified)\n",
    "\n",
    "    print(counts)\n",
    "\n",
    "    classified.sort(key=lambda row: row[2])\n",
    "    spammiest_hams = list(filter(lambda row: not row[1], classified))[-5:]\n",
    "    hammiest_spams = list(filter(lambda row: row[1], classified))[:5]\n",
    "\n",
    "    print(\"самые спамные среди не-спамных (spammiest_hams)\", spammiest_hams)\n",
    "    print()\n",
    "    print(\"самые не-спамные среди спамных (hammiest_spams)\", hammiest_spams)\n",
    "\n",
    "    words = sorted(classifier.word_probs, key=p_spam_given_word)\n",
    "\n",
    "    spammiest_words = words[-5:]\n",
    "    hammiest_words = words[:5]\n",
    "    \n",
    "    print()\n",
    "    print(\"самые спамные слова\", spammiest_words)\n",
    "    print()\n",
    "    print(\"самые не-спамные слова\", hammiest_words)\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    train_and_test_model(r\"../code-python3-ru/data/spam/*/*\")\n",
    "    #train_and_test_model(r\"c:\\spam\\*\\*\")\n",
    "    #train_and_test_model(r\"/home/joel/src/spam/*/*\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}