ml/logistic_regression_with_word2vec.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "logistic regression with word2vec",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/manishiitg/ML_Experments/blob/master/ml/logistic_regression_with_word2vec.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EG2quMxGhtHW",
        "colab_type": "text"
      },
      "source": [
        "# Text classification with logistic regression. \n",
        "\n",
        "Very simple experiment to use sklearn logistic regression. \n",
        "Using word2vec embeddings to classify text in 2 labels "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bWLHHVWhatWr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# http://nadbordrozd.github.io/blog/2016/05/20/text-classification-with-word2vec/\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import spacy \n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "\n",
        "from sklearn.datasets import fetch_20newsgroups\n",
        "\n",
        "news = fetch_20newsgroups(subset=\"train\")\n",
        "\n",
        "from gensim.models import KeyedVectors\n",
        "\n",
        "from sklearn.cluster import KMeans;\n",
        "from sklearn.neighbors import KDTree;\n",
        "\n",
        "from wordcloud import WordCloud, ImageColorGenerator\n",
        "import matplotlib.pyplot as plt;\n",
        "from itertools import cycle;\n",
        "import matplotlib.pyplot as plt;\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "\n",
        "from collections import defaultdict\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_xnWy2sGa1OQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def normalize(comment, lowercase=True, remove_stopwords=True):\n",
        "    if lowercase:\n",
        "        comment = comment.lower()\n",
        "    lines = comment.splitlines()\n",
        "    lines = [x.strip(' ') for x in lines]\n",
        "    lines = [x.replace('\"', '') for x in lines]\n",
        "    lines = [x.replace('\\\\\"', '') for x in lines]\n",
        "    lines = [x.replace(u'\\xa0', u'') for x in lines]\n",
        "    comment = \" \".join(lines)\n",
        "    doc = nlp(comment)\n",
        "\n",
        "    # for token in doc:\n",
        "    #   print(token.text, token.lemma_, token.pos_, token.tag_, token.dep_,\n",
        "    #     token.shape_, token.is_alpha, token.is_stop)\n",
        "\n",
        "    words = [token for token in doc if token.is_stop !=\n",
        "             True and token.is_punct != True]\n",
        "    # return \" \".join(words)\n",
        "    lemmatized = list()\n",
        "    for word in words:\n",
        "        lemma = word.lemma_.strip()\n",
        "        if lemma:\n",
        "            lemmatized.append(lemma)\n",
        "    return lemmatized"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PLpFOCgYiBG9",
        "colab_type": "text"
      },
      "source": [
        "Cleaning up data, lemma is optional"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FpjyH6LZa3sV",
        "colab_type": "code",
        "outputId": "e9e5554c-f9a5-46f7-8894-683540d5963d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "import os\n",
        "\n",
        "def writetofile(dir, filename, data):\n",
        "    if not os.path.exists(dir):\n",
        "        os.makedirs(dir)\n",
        "    f = os.path.join(dir,str(filename))\n",
        "\n",
        "    with open(f, 'wb') as the_file:\n",
        "      the_file.write(data)\n",
        "\n",
        "news = fetch_20newsgroups(subset=\"train\", categories=['alt.atheism', 'comp.graphics'])\n",
        "\n",
        "\n",
        "clean_data  = []\n",
        "\n",
        "# print(news.keys())\n",
        "\n",
        "# print(news[\"filenames\"][:10])\n",
        "\n",
        "# print(len(news[\"data\"][:1000]))\n",
        "\n",
        "# print(news[\"target_names\"][:50])\n",
        "# print(news[\"target\"][:10])\n",
        "\n",
        "\n",
        "max_limit = 1000\n",
        "\n",
        "targets = news[\"target\"][:max_limit]\n",
        "filenames = news[\"filenames\"][:max_limit]\n",
        "news = news[\"data\"][:max_limit]\n",
        "\n",
        "dir = \"news_group_cleaned\"\n",
        "\n",
        "print(\"cleaning data\")\n",
        "for i, row in enumerate(news):\n",
        "  filename = filenames[i]\n",
        "  filename = filename[(filename.rfind('/'))+1:]\n",
        "  if os.path.exists(filename):\n",
        "    with open(os.path.join(dir, filename), 'r') as content_file:\n",
        "      data = content_file.read()\n",
        "      cleaned = data.split(\" \")\n",
        "  else:\n",
        "    cleaned = normalize(row)\n",
        "    writetofile(dir, filename, \" \".join(cleaned).encode(\"utf-8\"))\n",
        "    \n",
        "  \n",
        "  clean_data.append(cleaned)\n",
        "\n",
        "print(\"data cleaned\")\n",
        "\n",
        "# print(clean_data[10])"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cleaning data\n",
            "data cleaned\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q4i0ggX-iFou",
        "colab_type": "text"
      },
      "source": [
        "Just using two category for simple binary classification "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z2LQStZya6a4",
        "colab_type": "code",
        "outputId": "940aa354-1880-4daf-fcbd-ac5983897345",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 589
        }
      },
      "source": [
        "import gensim, logging\n",
        "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
        "\n",
        "model = gensim.models.Word2Vec(clean_data)\n",
        "\n",
        "\n",
        "#https://stackoverflow.com/questions/45159693/word2vec-models-consist-of-characters-instead-of-words"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-11-08 05:49:56,012 : INFO : collecting all words and their counts\n",
            "2019-11-08 05:49:56,013 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
            "2019-11-08 05:49:56,053 : INFO : collected 18870 word types from a corpus of 166087 raw words and 1000 sentences\n",
            "2019-11-08 05:49:56,054 : INFO : Loading a fresh vocabulary\n",
            "2019-11-08 05:49:56,072 : INFO : effective_min_count=5 retains 4338 unique words (22% of original 18870, drops 14532)\n",
            "2019-11-08 05:49:56,073 : INFO : effective_min_count=5 leaves 143114 word corpus (86% of original 166087, drops 22973)\n",
            "2019-11-08 05:49:56,089 : INFO : deleting the raw counts dictionary of 18870 items\n",
            "2019-11-08 05:49:56,091 : INFO : sample=0.001 downsamples 30 most-common words\n",
            "2019-11-08 05:49:56,092 : INFO : downsampling leaves estimated 121517 word corpus (84.9% of prior 143114)\n",
            "2019-11-08 05:49:56,106 : INFO : estimated required memory for 4338 words and 100 dimensions: 5639400 bytes\n",
            "2019-11-08 05:49:56,107 : INFO : resetting layer weights\n",
            "2019-11-08 05:49:56,904 : INFO : training model with 3 workers on 4338 vocabulary and 100 features, using sg=0 hs=0 sample=0.001 negative=5 window=5\n",
            "2019-11-08 05:49:57,115 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-08 05:49:57,116 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-08 05:49:57,131 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-08 05:49:57,132 : INFO : EPOCH - 1 : training on 166087 raw words (121614 effective words) took 0.2s, 547582 effective words/s\n",
            "2019-11-08 05:49:57,335 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-08 05:49:57,338 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-08 05:49:57,356 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-08 05:49:57,358 : INFO : EPOCH - 2 : training on 166087 raw words (121533 effective words) took 0.2s, 556045 effective words/s\n",
            "2019-11-08 05:49:57,571 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-08 05:49:57,581 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-08 05:49:57,589 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-08 05:49:57,590 : INFO : EPOCH - 3 : training on 166087 raw words (121469 effective words) took 0.2s, 538825 effective words/s\n",
            "2019-11-08 05:49:57,805 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-08 05:49:57,820 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-08 05:49:57,824 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-08 05:49:57,825 : INFO : EPOCH - 4 : training on 166087 raw words (121588 effective words) took 0.2s, 528343 effective words/s\n",
            "2019-11-08 05:49:58,036 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-08 05:49:58,038 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-08 05:49:58,055 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-08 05:49:58,056 : INFO : EPOCH - 5 : training on 166087 raw words (121427 effective words) took 0.2s, 540433 effective words/s\n",
            "2019-11-08 05:49:58,057 : INFO : training on a 830435 raw words (607631 effective words) took 1.2s, 527376 effective words/s\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DdcpAwoAbAXz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "class MeanEmbeddingVectorizer(object):\n",
        "    def __init__(self, word2vec):\n",
        "        self.word2vec = word2vec\n",
        "        # if a text is empty we should return a vector of zeros\n",
        "        # with the same dimensionality as all the other vectors\n",
        "        self.dim = 100\n",
        "\n",
        "    def fit(self, X, y):\n",
        "        return self\n",
        "\n",
        "    def transform(self, X):\n",
        "        return np.array([\n",
        "            np.mean([self.word2vec[w] for w in words if w in self.word2vec]\n",
        "                    or [np.zeros(self.dim)], axis=0)\n",
        "            for words in X\n",
        "        ])\n",
        "\n",
        "w2v = dict(zip(model.wv.index2word, model.wv.vectors))\n",
        "\n",
        "\n",
        "em = MeanEmbeddingVectorizer(w2v)\n",
        "mean_embeddings = em.transform(clean_data)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TRsCr4X3iLgR",
        "colab_type": "text"
      },
      "source": [
        "This will simple take mean of all vectors of words  in a single sentence/document to represent it. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z7o9AseTjB7i",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "a8e63657-95f0-428e-ac30-e3bab63a221e"
      },
      "source": [
        "print(len(mean_embeddings))\n",
        "print(len(targets))\n",
        "\n",
        "X_train, X_test, Y_train, Y_test = train_test_split(mean_embeddings, targets, test_size=0.33, random_state=42)\n",
        "\n",
        "scikit_log_reg = LogisticRegression(verbose=1, solver='liblinear', random_state=0, C=5, penalty='l2', max_iter=1000)\n",
        "lr_model = scikit_log_reg.fit(X_train, Y_train)\n",
        "\n",
        "print(\"Score:\", scikit_log_reg.score(X_test, Y_test))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000\n",
            "1000\n",
            "[LibLinear]Score: 0.9636363636363636\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UrXwDvY4bFIN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class TfidfEmbeddingVectorizer(object):\n",
        "    def __init__(self, word2vec):\n",
        "        self.word2vec = word2vec\n",
        "        self.word2weight = None\n",
        "        self.dim = 100\n",
        "\n",
        "    def fit(self, X, y):\n",
        "        tfidf = TfidfVectorizer(analyzer=lambda x: x)\n",
        "        tfidf.fit(X)\n",
        "        # if a word was never seen - it must be at least as infrequent\n",
        "        # as any of the known words - so the default idf is the max of \n",
        "        # known idf's\n",
        "        max_idf = max(tfidf.idf_)\n",
        "        #https://stackoverflow.com/a/5900634\n",
        "        self.word2weight = defaultdict(\n",
        "            lambda: max_idf,\n",
        "            [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()])\n",
        "\n",
        "        return self\n",
        "\n",
        "    def transform(self, X):\n",
        "        return np.array([\n",
        "                np.mean([self.word2vec[w] * self.word2weight[w]\n",
        "                         for w in words if w in self.word2vec] or\n",
        "                        [np.zeros(self.dim)], axis=0)\n",
        "                for words in X\n",
        "            ])\n",
        "\n",
        "w2v = dict(zip(model.wv.index2word, model.wv.vectors))\n",
        "\n",
        "tf = TfidfEmbeddingVectorizer(w2v)\n",
        "tf.fit(clean_data, targets)\n",
        "tf_idf_mean_embeddings = tf.transform(clean_data)   \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HqghIpFmbG6B",
        "colab_type": "code",
        "outputId": "b1e181d8-0dfc-40b8-97e7-a2f0e0fd6b21",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "print(len(tf_idf_mean_embeddings))\n",
        "print(len(targets))\n",
        "\n",
        "X_train, X_test, Y_train, Y_test = train_test_split(tf_idf_mean_embeddings, targets, test_size=0.33, random_state=42)\n",
        "\n",
        "scikit_log_reg = LogisticRegression(verbose=1, solver='liblinear', random_state=0, C=5, penalty='l2', max_iter=1000)\n",
        "lr_model2 = scikit_log_reg.fit(X_train, Y_train)\n",
        "\n",
        "print(\"Score:\", scikit_log_reg.score(X_test, Y_test))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000\n",
            "1000\n",
            "[LibLinear]Score: 0.9787878787878788\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KEOtKp5ybILu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}