TF-IDF手工实现.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 输入语料库\n",
    "corpus = ['I graduated from BUPT BUPT is beijing university of posts and telecommunications',\n",
    "        'I work in ByteDance',\n",
    "        'I am living in guangdong province',\n",
    "        'my favorite sports is playing basketball']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['I', 'graduated', 'from', 'BUPT', 'BUPT', 'is', 'beijing', 'university', 'of', 'posts', 'and', 'telecommunications'], ['I', 'work', 'in', 'ByteDance'], ['I', 'am', 'living', 'in', 'guangdong', 'province'], ['my', 'favorite', 'sports', 'is', 'playing', 'basketball']]\n"
     ]
    }
   ],
   "source": [
    "words_list = list()\n",
    "for i in range(len(corpus)):\n",
    "    words_list.append(corpus[i].split(' '))\n",
    "print(words_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Counter({'BUPT': 2, 'I': 1, 'graduated': 1, 'from': 1, 'is': 1, 'beijing': 1, 'university': 1, 'of': 1, 'posts': 1, 'and': 1, 'telecommunications': 1}), Counter({'I': 1, 'work': 1, 'in': 1, 'ByteDance': 1}), Counter({'I': 1, 'am': 1, 'living': 1, 'in': 1, 'guangdong': 1, 'province': 1}), Counter({'my': 1, 'favorite': 1, 'sports': 1, 'is': 1, 'playing': 1, 'basketball': 1})]\n"
     ]
    }
   ],
   "source": [
    "count_list = list()\n",
    "for i in range(len(words_list)):\n",
    "    count = Counter(words_list[i])\n",
    "    count_list.append(count)\n",
    "    # print(count)\n",
    "print(count_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "def tf(word, count):\n",
    "    return count[word] / sum(count.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 词频TF（Term Frequncy），也就是词在单个文档中出现的频率。\n",
    "在该函数中词频为count[word]\n",
    "所有计数的总数为sum(count.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def idf(word, count_list):\n",
    "    n_contain = sum([1 for count in count_list if word in count])\n",
    "    return math.log(len(count_list) / (1 + n_contain))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 逆文档频率IDF（Inverse Document Frequency）\n",
    "$IDF = log(N/n)$ \\\n",
    "其中，N代表语料库中文档的总数，n代表某个Word在几个文档中出现过；当一个Word出现地越频繁，那么IDF就越小。\n",
    "显然，IDF用于惩罚那些常用词汇，而TF用于奖励那些在特定文档中出现频繁的词汇。二者的乘积TF X IDF用来表示词汇的权重，显然合理性大大增强。\n",
    "### 另一种写法\n",
    "$IDF = log[N/(n+1)]$ \\\n",
    "为什么这里要加 1 呢？主要是防止包含词条 n 的数量为 0 从而导致运算出错的现象发生。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tf_idf(word, count, count_list):\n",
    "    return tf(word, count) * idf(word, count_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 1 个文档 TF-IDF 统计信息\n",
      "\tword: BUPT, TF-IDF: 0.11552\n",
      "\tword: graduated, TF-IDF: 0.05776\n",
      "\tword: from, TF-IDF: 0.05776\n",
      "\tword: beijing, TF-IDF: 0.05776\n",
      "\tword: university, TF-IDF: 0.05776\n",
      "\tword: of, TF-IDF: 0.05776\n",
      "\tword: posts, TF-IDF: 0.05776\n",
      "\tword: and, TF-IDF: 0.05776\n",
      "\tword: telecommunications, TF-IDF: 0.05776\n",
      "\tword: is, TF-IDF: 0.02397\n",
      "\tword: I, TF-IDF: 0.0\n",
      "第 2 个文档 TF-IDF 统计信息\n",
      "\tword: work, TF-IDF: 0.17329\n",
      "\tword: ByteDance, TF-IDF: 0.17329\n",
      "\tword: in, TF-IDF: 0.07192\n",
      "\tword: I, TF-IDF: 0.0\n",
      "第 3 个文档 TF-IDF 统计信息\n",
      "\tword: am, TF-IDF: 0.11552\n",
      "\tword: living, TF-IDF: 0.11552\n",
      "\tword: guangdong, TF-IDF: 0.11552\n",
      "\tword: province, TF-IDF: 0.11552\n",
      "\tword: in, TF-IDF: 0.04795\n",
      "\tword: I, TF-IDF: 0.0\n",
      "第 4 个文档 TF-IDF 统计信息\n",
      "\tword: my, TF-IDF: 0.11552\n",
      "\tword: favorite, TF-IDF: 0.11552\n",
      "\tword: sports, TF-IDF: 0.11552\n",
      "\tword: playing, TF-IDF: 0.11552\n",
      "\tword: basketball, TF-IDF: 0.11552\n",
      "\tword: is, TF-IDF: 0.04795\n"
     ]
    }
   ],
   "source": [
    "for i, count in enumerate(count_list):\n",
    "    print(\"第 {} 个文档 TF-IDF 统计信息\".format(i + 1))\n",
    "    scores = {word : tf_idf(word, count, count_list) for word in count}\n",
    "    sorted_word = sorted(scores.items(), key = lambda x : x[1], reverse=True)\n",
    "    for word, score in sorted_word:\n",
    "        print(\"\\tword: {}, TF-IDF: {}\".format(word, round(score, 5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}