使用 sklearn 算法包实现TF-IDF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 输入语料库\n",
    "corpus = ['I graduated from BUPT BUPT is beijing university of posts and telecommunications',\n",
    "        'I work in ByteDance',\n",
    "        'I am living in guangdong province',\n",
    "        'my favorite sports is playing basketball']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['am', 'and', 'basketball', 'beijing', 'bupt', 'bytedance', 'favorite', 'from', 'graduated', 'guangdong', 'in', 'is', 'living', 'my', 'of', 'playing', 'posts', 'province', 'sports', 'telecommunications', 'university', 'work']\n",
      "{'graduated': 8, 'from': 7, 'bupt': 4, 'is': 11, 'beijing': 3, 'university': 20, 'of': 14, 'posts': 16, 'and': 1, 'telecommunications': 19, 'work': 21, 'in': 10, 'bytedance': 5, 'am': 0, 'living': 12, 'guangdong': 9, 'province': 17, 'my': 13, 'favorite': 6, 'sports': 18, 'playing': 15, 'basketball': 2}\n",
      "[[0.         0.281477   0.         0.281477   0.56295401 0.\n",
      "  0.         0.281477   0.281477   0.         0.         0.2219197\n",
      "  0.         0.         0.281477   0.         0.281477   0.\n",
      "  0.         0.281477   0.281477   0.        ]\n",
      " [0.         0.         0.         0.         0.         0.61761437\n",
      "  0.         0.         0.         0.         0.48693426 0.\n",
      "  0.         0.         0.         0.         0.         0.\n",
      "  0.         0.         0.         0.61761437]\n",
      " [0.46516193 0.         0.         0.         0.         0.\n",
      "  0.         0.         0.         0.46516193 0.36673901 0.\n",
      "  0.46516193 0.         0.         0.         0.         0.46516193\n",
      "  0.         0.         0.         0.        ]\n",
      " [0.         0.         0.42176478 0.         0.         0.\n",
      "  0.42176478 0.         0.         0.         0.         0.3325242\n",
      "  0.         0.42176478 0.         0.42176478 0.         0.\n",
      "  0.42176478 0.         0.         0.        ]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf_vec = TfidfVectorizer()\n",
    "tfidf_matrix = tfidf_vec.fit_transform(corpus)\n",
    "# 得到语料库所有不重复的词\n",
    "print(tfidf_vec.get_feature_names())\n",
    "# 得到每个单词对应的id值\n",
    "print(tfidf_vec.vocabulary_)\n",
    "# 得到每个句子所对应的向量，向量里数字的顺序是按照词语的id顺序来的\n",
    "print(tfidf_matrix.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}