text-clustering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/fabio/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /home/fabio/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "nltk.download(\"punkt\")\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from string import punctuation\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of the Dataframe (5749, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genre</th>\n",
       "      <th>filename</th>\n",
       "      <th>year</th>\n",
       "      <th>index</th>\n",
       "      <th>score</th>\n",
       "      <th>sentence1</th>\n",
       "      <th>sentence2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>main-captions</td>\n",
       "      <td>MSRvid</td>\n",
       "      <td>2012test</td>\n",
       "      <td>1</td>\n",
       "      <td>5.00</td>\n",
       "      <td>A plane is taking off.</td>\n",
       "      <td>An air plane is taking off.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>main-captions</td>\n",
       "      <td>MSRvid</td>\n",
       "      <td>2012test</td>\n",
       "      <td>4</td>\n",
       "      <td>3.80</td>\n",
       "      <td>A man is playing a large flute.</td>\n",
       "      <td>A man is playing a flute.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main-captions</td>\n",
       "      <td>MSRvid</td>\n",
       "      <td>2012test</td>\n",
       "      <td>5</td>\n",
       "      <td>3.80</td>\n",
       "      <td>A man is spreading shreded cheese on a pizza.</td>\n",
       "      <td>A man is spreading shredded cheese on an uncoo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>main-captions</td>\n",
       "      <td>MSRvid</td>\n",
       "      <td>2012test</td>\n",
       "      <td>6</td>\n",
       "      <td>2.60</td>\n",
       "      <td>Three men are playing chess.</td>\n",
       "      <td>Two men are playing chess.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>main-captions</td>\n",
       "      <td>MSRvid</td>\n",
       "      <td>2012test</td>\n",
       "      <td>9</td>\n",
       "      <td>4.25</td>\n",
       "      <td>A man is playing the cello.</td>\n",
       "      <td>A man seated is playing the cello.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>main-captions</td>\n",
       "      <td>MSRvid</td>\n",
       "      <td>2012test</td>\n",
       "      <td>11</td>\n",
       "      <td>4.25</td>\n",
       "      <td>Some men are fighting.</td>\n",
       "      <td>Two men are fighting.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>main-captions</td>\n",
       "      <td>MSRvid</td>\n",
       "      <td>2012test</td>\n",
       "      <td>12</td>\n",
       "      <td>0.50</td>\n",
       "      <td>A man is smoking.</td>\n",
       "      <td>A man is skating.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>main-captions</td>\n",
       "      <td>MSRvid</td>\n",
       "      <td>2012test</td>\n",
       "      <td>13</td>\n",
       "      <td>1.60</td>\n",
       "      <td>The man is playing the piano.</td>\n",
       "      <td>The man is playing the guitar.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>main-captions</td>\n",
       "      <td>MSRvid</td>\n",
       "      <td>2012test</td>\n",
       "      <td>14</td>\n",
       "      <td>2.20</td>\n",
       "      <td>A man is playing on a guitar and singing.</td>\n",
       "      <td>A woman is playing an acoustic guitar and sing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>main-captions</td>\n",
       "      <td>MSRvid</td>\n",
       "      <td>2012test</td>\n",
       "      <td>16</td>\n",
       "      <td>5.00</td>\n",
       "      <td>A person is throwing a cat on to the ceiling.</td>\n",
       "      <td>A person throws a cat on the ceiling.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           genre filename      year  index  score  \\\n",
       "0  main-captions   MSRvid  2012test      1   5.00   \n",
       "1  main-captions   MSRvid  2012test      4   3.80   \n",
       "2  main-captions   MSRvid  2012test      5   3.80   \n",
       "3  main-captions   MSRvid  2012test      6   2.60   \n",
       "4  main-captions   MSRvid  2012test      9   4.25   \n",
       "5  main-captions   MSRvid  2012test     11   4.25   \n",
       "6  main-captions   MSRvid  2012test     12   0.50   \n",
       "7  main-captions   MSRvid  2012test     13   1.60   \n",
       "8  main-captions   MSRvid  2012test     14   2.20   \n",
       "9  main-captions   MSRvid  2012test     16   5.00   \n",
       "\n",
       "                                       sentence1  \\\n",
       "0                         A plane is taking off.   \n",
       "1                A man is playing a large flute.   \n",
       "2  A man is spreading shreded cheese on a pizza.   \n",
       "3                   Three men are playing chess.   \n",
       "4                    A man is playing the cello.   \n",
       "5                         Some men are fighting.   \n",
       "6                              A man is smoking.   \n",
       "7                  The man is playing the piano.   \n",
       "8      A man is playing on a guitar and singing.   \n",
       "9  A person is throwing a cat on to the ceiling.   \n",
       "\n",
       "                                           sentence2  \n",
       "0                        An air plane is taking off.  \n",
       "1                          A man is playing a flute.  \n",
       "2  A man is spreading shredded cheese on an uncoo...  \n",
       "3                         Two men are playing chess.  \n",
       "4                 A man seated is playing the cello.  \n",
       "5                              Two men are fighting.  \n",
       "6                                  A man is skating.  \n",
       "7                     The man is playing the guitar.  \n",
       "8  A woman is playing an acoustic guitar and sing...  \n",
       "9              A person throws a cat on the ceiling.  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def on_bad_line(values):\n",
    "    return values[:7]\n",
    "\n",
    "columns_mapping = {\n",
    "    0: 'genre',\n",
    "    1: 'filename',\n",
    "    2: 'year',\n",
    "    3: 'index',\n",
    "    4: 'score',\n",
    "    5: 'sentence1',\n",
    "    6: 'sentence2'\n",
    "}\n",
    "\n",
    "# quotings 3 = csv.QUOTE_NONE\n",
    "df = pd.read_csv('Dataset/sts-train.csv', sep=\"\\t\", on_bad_lines=on_bad_line, engine='python', header=None, encoding='utf-8', quoting=3).rename(columns=columns_mapping)\n",
    "print(f'shape of the Dataframe {df.shape}')\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from scipy.cluster.hierarchy import dendrogram, linkage\n",
    "from tqdm import tqdm\n",
    "import scipy.sparse as scs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11498, 2877)\n"
     ]
    }
   ],
   "source": [
    "sentences1 = df[\"sentence1\"]\n",
    "sentences2 = df[\"sentence2\"]\n",
    "raw_corpus = np.concatenate((sentences1, sentences2))\n",
    "\n",
    "stop_words = stopwords.words('english')\n",
    "\n",
    "regex = f'[{punctuation}]'\n",
    "corpus = np.array([re.sub(regex,'',string).lower() for string in raw_corpus])\n",
    "\n",
    "# creazione della matrice di similarità\n",
    "tfidf = TfidfVectorizer(max_df=0.8, min_df=5, stop_words='english', strip_accents='ascii')\n",
    "tfidf_matrix = tfidf.fit_transform(corpus)\n",
    "similarity_matrix = cosine_similarity(tfidf_matrix)\n",
    "vocab = tfidf.get_feature_names_out()\n",
    "\n",
    "print(tfidf_matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['04' '07' '10' ... 'zimbabwe' 'zimmerman' 'zone']\n",
      "the word dog is present in 433 documents. Here's the first 5\n",
      "['a dog is trying to get bacon off his back' 'a dog rides a skateboard'\n",
      " 'a dog licks a baby' 'a dog is eating water melon'\n",
      " 'a small dog is chasing a yoga ball']\n"
     ]
    }
   ],
   "source": [
    "# Get the terms (columns) of the matrix\n",
    "terms = tfidf.get_feature_names_out()\n",
    "print(terms)\n",
    "\n",
    "word = 'dog'\n",
    "word_id = tfidf.transform([word]).nonzero()[1]\n",
    "filter = tfidf_matrix.getcol(word_id).nonzero()[0]\n",
    "print(f'the word {word} is present in {len(filter)} documents. Here\\'s the first 5')\n",
    "print(corpus[filter][:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 2805)\t0.2952740316253059\n",
      "  (0, 1902)\t0.405078758431841\n",
      "  (0, 1695)\t0.3874740562788338\n",
      "  (0, 467)\t0.7129072478984504\n",
      "  (0, 345)\t0.30058535779332396\n",
      "  (1, 2759)\t0.6709395536042083\n",
      "  (1, 1026)\t0.5748842386096671\n",
      "  (1, 467)\t0.46834626891603315\n",
      "  (2, 1902)\t0.5759350456688125\n",
      "  (2, 1794)\t0.6414452697963486\n",
      "  (2, 467)\t0.50680054165949\n"
     ]
    }
   ],
   "source": [
    "docs = ['the black cat is on the plane, near the white cat', 'another cat is waiting for its food', 'càt orange plane']\n",
    "dt_matrix = tfidf.transform(docs)\n",
    "print(dt_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialization complete\n",
      "Iteration 0, inertia 19312.619734942713.\n",
      "Iteration 1, inertia 10543.630351754517.\n",
      "Iteration 2, inertia 10491.200139302586.\n",
      "Iteration 3, inertia 10459.384151765547.\n",
      "Iteration 4, inertia 10451.401064134441.\n",
      "Iteration 5, inertia 10444.212212742817.\n",
      "Iteration 6, inertia 10437.963965781455.\n",
      "Iteration 7, inertia 10434.200905290123.\n",
      "Iteration 8, inertia 10429.810610515426.\n",
      "Iteration 9, inertia 10427.145002937279.\n",
      "Iteration 10, inertia 10423.162686221849.\n",
      "Iteration 11, inertia 10417.501258545039.\n",
      "Iteration 12, inertia 10389.402654884878.\n",
      "Iteration 13, inertia 10384.753471356591.\n",
      "Iteration 14, inertia 10384.109599476968.\n",
      "Iteration 15, inertia 10383.487712229798.\n",
      "Iteration 16, inertia 10383.0420604167.\n",
      "Iteration 17, inertia 10382.811282508505.\n",
      "Iteration 18, inertia 10382.681314184527.\n",
      "Iteration 19, inertia 10382.62037156886.\n",
      "Iteration 20, inertia 10382.603029655944.\n",
      "Iteration 21, inertia 10382.595329231895.\n",
      "Iteration 22, inertia 10382.58875625161.\n",
      "Converged at iteration 22: strict convergence.\n",
      "Initialization complete\n",
      "Iteration 0, inertia 11245.979177239282.\n",
      "Iteration 1, inertia 10966.794365346706.\n",
      "Iteration 2, inertia 10822.480029135966.\n",
      "Iteration 3, inertia 10777.426517960032.\n",
      "Iteration 4, inertia 10750.292332492647.\n",
      "Iteration 5, inertia 10735.161754774119.\n",
      "Iteration 6, inertia 10729.98373419228.\n",
      "Iteration 7, inertia 10727.275971523128.\n",
      "Iteration 8, inertia 10723.424683794037.\n",
      "Iteration 9, inertia 10720.783962385449.\n",
      "Iteration 10, inertia 10720.418778987641.\n",
      "Iteration 11, inertia 10720.343687827237.\n",
      "Iteration 12, inertia 10720.213586520102.\n",
      "Iteration 13, inertia 10720.117217480758.\n",
      "Iteration 14, inertia 10720.030650030752.\n",
      "Iteration 15, inertia 10719.865541951842.\n",
      "Iteration 16, inertia 10719.631485520033.\n",
      "Iteration 17, inertia 10719.471031970244.\n",
      "Iteration 18, inertia 10719.404556882004.\n",
      "Iteration 19, inertia 10719.359952977513.\n",
      "Iteration 20, inertia 10719.348139754986.\n",
      "Converged at iteration 20: strict convergence.\n"
     ]
    }
   ],
   "source": [
    "k = 30\n",
    "kmeans = KMeans(n_clusters=k, max_iter=100, n_init=2, verbose=True)\n",
    "cluster_matrix = kmeans.fit_transform(tfidf_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top terms per cluster:\n",
      "Cluster 0:\t['women', 'sitting', 'smiling', 'standing', 'sofa']\n",
      "Cluster 1:\t['sand', 'dog', 'running', 'brown', 'dogs']\n",
      "Cluster 2:\t['running', 'dog', 'grass', 'dogs', 'beach']\n",
      "Cluster 3:\t['people', 'group', 'sitting', 'table', 'restaurant']\n",
      "Cluster 4:\t['korea', 'north', 'south', 'nuclear', 'missiles']\n",
      "Cluster 5:\t['girl', 'new', 'boy', 'police', 'dead']\n",
      "Cluster 6:\t['dancing', 'woman', 'man', 'rain', 'group']\n",
      "Cluster 7:\t['percent', 'shares', 'points', 'nasdaq', 'index']\n",
      "Cluster 8:\t['iran', 'nuclear', 'talks', 'peace', 'deal']\n",
      "Cluster 9:\t['snow', 'dog', 'dogs', 'black', 'running']\n",
      "Cluster 10:\t['playing', 'guitar', 'man', 'flute', 'piano']\n",
      "Cluster 11:\t['man', 'slicing', 'cutting', 'walking', 'rope']\n",
      "Cluster 12:\t['drug', 'capital', 'trafficking', 'police', 'laws']\n",
      "Cluster 13:\t['riding', 'horse', 'man', 'woman', 'motorcycle']\n",
      "Cluster 14:\t['china', 'usd', 'yuan', 'stocks', 'strengthens']\n",
      "Cluster 15:\t['white', 'black', 'cat', 'cow', 'dog']\n",
      "Cluster 16:\t['dont', 'exist', 'fine', 'know', 'want']\n",
      "Cluster 17:\t['child', 'bird', 'small', 'perched', 'branch']\n",
      "Cluster 18:\t['president', 'egypt', 'new', 'brotherhood', 'morsi']\n",
      "Cluster 19:\t['pakistan', 'drone', 'kills', 'strike', 'taliban']\n",
      "Cluster 20:\t['woman', 'slicing', 'cutting', 'onion', 'man']\n",
      "Cluster 21:\t['officials', 'children', 'said', 'stated', 'police']\n",
      "Cluster 22:\t['train', 'tracks', 'station', 'passenger', 'crash']\n",
      "Cluster 23:\t['syria', 'chemical', 'weapons', 'rebels', 'opposition']\n",
      "Cluster 24:\t['car', 'driving', 'man', 'bomb', 'kills']\n",
      "Cluster 25:\t['water', 'bus', 'decker', 'red', 'double']\n",
      "Cluster 26:\t['person', 'slicing', 'cutting', 'bike', 'onion']\n",
      "Cluster 27:\t['dog', 'brown', 'black', 'runs', 'grass']\n",
      "Cluster 28:\t['killed', 'afghanistan', 'nato', 'iraq', 'soldiers']\n",
      "Cluster 29:\t['said', 'spokesman', 'statement', 'mr', 'director']\n"
     ]
    }
   ],
   "source": [
    "print(\"Top terms per cluster:\")\n",
    "vocab = tfidf.get_feature_names_out()\n",
    "\n",
    "for i in range(kmeans.n_clusters):\n",
    "    centroid = kmeans.cluster_centers_[i]    \n",
    "    sorted_terms = centroid.argsort()[::-1]\n",
    "    print(f\"Cluster {i}:\\t{[vocab[j] for j in sorted_terms[:5]]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2892\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Word2Vec\n",
    "\n",
    "analyzer = tfidf.build_analyzer()\n",
    "tokenized_corpus = [analyzer(w) for w in corpus]\n",
    "model = Word2Vec(tokenized_corpus, vector_size=30, min_count=5, window=10)\n",
    "print(len(model.wv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# todo fare con operazioni vettoriali\n",
    "def mix(vectorizer: TfidfVectorizer, embeddings: Word2Vec, document) -> np.array:\n",
    "    vectorized = vectorizer.transform([document])\n",
    "    cols = vectorized.nonzero()[1]\n",
    "    result = np.zeros(30)\n",
    "    weights = 0\n",
    "    terms = tfidf.get_feature_names_out()\n",
    "\n",
    "    for x in cols:\n",
    "        word = terms[x]\n",
    "        try:\n",
    "            embed = embeddings.wv[word]\n",
    "        except:\n",
    "            continue\n",
    "        weight = vectorized[0, x]\n",
    "        result += weight * embed\n",
    "        weights += weight\n",
    "        \n",
    "    result = result / weights\n",
    "    if (np.isnan(result).any() or weights == 0):\n",
    "        return np.zeros(30)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_25521/2236178505.py:18: RuntimeWarning: invalid value encountered in divide\n",
      "  result = result / weights\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11498, 30)\n",
      "Initialization complete\n",
      "Iteration 0, inertia 298.8018460750988.\n",
      "Iteration 1, inertia 219.55690562598215.\n",
      "Iteration 2, inertia 214.6848562331368.\n",
      "Iteration 3, inertia 212.857355852252.\n",
      "Iteration 4, inertia 212.23368635999134.\n",
      "Iteration 5, inertia 211.92467648426236.\n",
      "Iteration 6, inertia 211.6976908654495.\n",
      "Iteration 7, inertia 211.47532825632214.\n",
      "Iteration 8, inertia 211.1146978506815.\n",
      "Iteration 9, inertia 210.45744775429492.\n",
      "Iteration 10, inertia 209.69076390074764.\n",
      "Iteration 11, inertia 208.97420058796783.\n",
      "Iteration 12, inertia 208.90743118556685.\n",
      "Iteration 13, inertia 208.83627551403782.\n",
      "Iteration 14, inertia 208.81854240977623.\n",
      "Iteration 15, inertia 208.79941843379348.\n",
      "Iteration 16, inertia 208.7858940110073.\n",
      "Iteration 17, inertia 208.77601484051328.\n",
      "Iteration 18, inertia 208.77324308712764.\n",
      "Converged at iteration 18: center shift 1.1176360309289543e-06 within tolerance 1.626664907705768e-06.\n",
      "Initialization complete\n",
      "Iteration 0, inertia 300.90241853391024.\n",
      "Iteration 1, inertia 224.8844079841024.\n",
      "Iteration 2, inertia 217.76265006915352.\n",
      "Iteration 3, inertia 214.95449578373402.\n",
      "Iteration 4, inertia 213.17639388824608.\n",
      "Iteration 5, inertia 212.10050379461882.\n",
      "Iteration 6, inertia 211.13841068984598.\n",
      "Iteration 7, inertia 210.73581642710312.\n",
      "Iteration 8, inertia 210.56651356045575.\n",
      "Iteration 9, inertia 210.36159250886703.\n",
      "Iteration 10, inertia 210.17194262021.\n",
      "Iteration 11, inertia 209.99665299742225.\n",
      "Iteration 12, inertia 209.85094440834348.\n",
      "Iteration 13, inertia 209.75963391224624.\n",
      "Iteration 14, inertia 209.68673541390942.\n",
      "Iteration 15, inertia 209.62472859792024.\n",
      "Iteration 16, inertia 209.54301270548913.\n",
      "Iteration 17, inertia 209.34782901773198.\n",
      "Iteration 18, inertia 209.25823172394814.\n",
      "Iteration 19, inertia 209.17139786211868.\n",
      "Iteration 20, inertia 209.12233818403593.\n",
      "Iteration 21, inertia 209.0827272898957.\n",
      "Iteration 22, inertia 208.99330718110485.\n",
      "Iteration 23, inertia 208.9143295194209.\n",
      "Iteration 24, inertia 208.83077344959494.\n",
      "Iteration 25, inertia 208.79702768404275.\n",
      "Iteration 26, inertia 208.7725910106367.\n",
      "Iteration 27, inertia 208.73972609677224.\n",
      "Iteration 28, inertia 208.70110633708634.\n",
      "Iteration 29, inertia 208.67530244060296.\n",
      "Iteration 30, inertia 208.65722824559592.\n",
      "Iteration 31, inertia 208.64103989520126.\n",
      "Iteration 32, inertia 208.6127594726541.\n",
      "Iteration 33, inertia 208.57658804421155.\n",
      "Iteration 34, inertia 208.54647562901928.\n",
      "Iteration 35, inertia 208.51308887883218.\n",
      "Iteration 36, inertia 208.4900699545808.\n",
      "Iteration 37, inertia 208.467313292989.\n",
      "Iteration 38, inertia 208.45129415105328.\n",
      "Iteration 39, inertia 208.4400786813158.\n",
      "Iteration 40, inertia 208.4351354972298.\n",
      "Iteration 41, inertia 208.43193965949067.\n",
      "Iteration 42, inertia 208.4259189557053.\n",
      "Iteration 43, inertia 208.42054810852713.\n",
      "Iteration 44, inertia 208.4175631396192.\n",
      "Converged at iteration 44: center shift 1.269570197700746e-06 within tolerance 1.626664907705768e-06.\n"
     ]
    }
   ],
   "source": [
    "mixed_corpus = np.array([mix(tfidf, model, w) for w in corpus])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialization complete\n",
      "Iteration 0, inertia 540.7065096318718.\n",
      "Iteration 1, inertia 415.4677448122001.\n",
      "Iteration 2, inertia 410.50759191062025.\n",
      "Iteration 3, inertia 407.1811913459393.\n",
      "Iteration 4, inertia 405.4516020070937.\n",
      "Iteration 5, inertia 404.376890767678.\n",
      "Iteration 6, inertia 403.62590788286354.\n",
      "Iteration 7, inertia 403.05747903241394.\n",
      "Iteration 8, inertia 402.33969978692943.\n",
      "Iteration 9, inertia 401.26514013216115.\n",
      "Iteration 10, inertia 400.4569337750206.\n",
      "Iteration 11, inertia 400.14070382054194.\n",
      "Iteration 12, inertia 399.9643201921365.\n",
      "Iteration 13, inertia 399.86032076568824.\n",
      "Iteration 14, inertia 399.81950869380347.\n",
      "Iteration 15, inertia 399.7941575168503.\n",
      "Iteration 16, inertia 399.75739545827275.\n",
      "Iteration 17, inertia 399.72824703792094.\n",
      "Iteration 18, inertia 399.70391499328633.\n",
      "Iteration 19, inertia 399.6838494806509.\n",
      "Iteration 20, inertia 399.67250939027207.\n",
      "Iteration 21, inertia 399.6685164150547.\n",
      "Iteration 22, inertia 399.6585553670636.\n",
      "Iteration 23, inertia 399.6545778144234.\n",
      "Iteration 24, inertia 399.64976865696303.\n",
      "Iteration 25, inertia 399.64669922410116.\n",
      "Iteration 26, inertia 399.6452061588247.\n",
      "Converged at iteration 26: center shift 1.1507840604210159e-06 within tolerance 1.626664907705768e-06.\n",
      "Initialization complete\n",
      "Iteration 0, inertia 546.9535896601617.\n",
      "Iteration 1, inertia 430.5528568156626.\n",
      "Iteration 2, inertia 422.1827675746573.\n",
      "Iteration 3, inertia 419.56408606930756.\n",
      "Iteration 4, inertia 418.0855282089156.\n",
      "Iteration 5, inertia 416.85664717622535.\n",
      "Iteration 6, inertia 415.57519202608364.\n",
      "Iteration 7, inertia 414.45206836752686.\n",
      "Iteration 8, inertia 413.32408685692917.\n",
      "Iteration 9, inertia 412.33313401544433.\n",
      "Iteration 10, inertia 410.83905355655367.\n",
      "Iteration 11, inertia 406.03959069748953.\n",
      "Iteration 12, inertia 398.8772611730614.\n",
      "Iteration 13, inertia 392.50143522518954.\n",
      "Iteration 14, inertia 389.1554006313195.\n",
      "Iteration 15, inertia 387.3437278675274.\n",
      "Iteration 16, inertia 386.2979909518825.\n",
      "Iteration 17, inertia 385.5245761108707.\n",
      "Iteration 18, inertia 384.9880204890725.\n",
      "Iteration 19, inertia 384.6757488486848.\n",
      "Iteration 20, inertia 384.4341144173883.\n",
      "Iteration 21, inertia 384.2180106877555.\n",
      "Iteration 22, inertia 383.9562436230366.\n",
      "Iteration 23, inertia 383.70088599521296.\n",
      "Iteration 24, inertia 383.38727708788974.\n",
      "Iteration 25, inertia 382.88689701678715.\n",
      "Iteration 26, inertia 382.2982841957656.\n",
      "Iteration 27, inertia 381.594055301614.\n",
      "Iteration 28, inertia 380.6620906183337.\n",
      "Iteration 29, inertia 380.16726192994065.\n",
      "Iteration 30, inertia 379.87494224846677.\n",
      "Iteration 31, inertia 379.63561873829906.\n",
      "Iteration 32, inertia 379.54921184119604.\n",
      "Iteration 33, inertia 379.4692246863506.\n",
      "Iteration 34, inertia 379.39224110189.\n",
      "Iteration 35, inertia 379.34013509083576.\n",
      "Iteration 36, inertia 379.29919143166694.\n",
      "Iteration 37, inertia 379.2569592287264.\n",
      "Iteration 38, inertia 379.22271105501954.\n",
      "Iteration 39, inertia 379.18153632505545.\n",
      "Iteration 40, inertia 379.14517042257506.\n",
      "Iteration 41, inertia 379.1097715002627.\n",
      "Iteration 42, inertia 379.082225676842.\n",
      "Iteration 43, inertia 379.0730831741289.\n",
      "Iteration 44, inertia 379.06685490356494.\n",
      "Iteration 45, inertia 379.061099367117.\n",
      "Iteration 46, inertia 379.05853678450353.\n",
      "Converged at iteration 46: center shift 6.00260285751411e-07 within tolerance 1.626664907705768e-06.\n"
     ]
    }
   ],
   "source": [
    "k = 15\n",
    "kmeans = KMeans(n_clusters=k, max_iter=100, n_init=2, verbose=True)\n",
    "cluster_matrix = kmeans.fit_transform(mixed_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = np.argmax(cluster_matrix, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top term per cluster:\n",
      "[('said', 0.9998720288276672), ('billion', 0.9996214509010315), ('tuesday', 0.9996185302734375), ('million', 0.9996104836463928), ('state', 0.9995715618133545), ('government', 0.9995687007904053), ('president', 0.9995319247245789), ('police', 0.9995316863059998), ('years', 0.9995228052139282), ('military', 0.9994965195655823)]\n",
      "[('said', 0.9998518228530884), ('million', 0.9997186064720154), ('government', 0.9996440410614014), ('billion', 0.9996383786201477), ('tuesday', 0.9996023178100586), ('police', 0.9995832443237305), ('yesterday', 0.9995828866958618), ('state', 0.9995742440223694), ('military', 0.999573826789856), ('president', 0.9995699524879456)]\n",
      "[('indian', 0.999377429485321), ('said', 0.999259352684021), ('dies', 0.9992164969444275), ('told', 0.9991896152496338), ('air', 0.9991471171379089), ('court', 0.9991379976272583), ('arrested', 0.9990209937095642), ('workers', 0.9990055561065674), ('tuesday', 0.9990054368972778), ('billion', 0.9989449977874756)]\n",
      "[('said', 0.9998770952224731), ('million', 0.999714195728302), ('billion', 0.9996453523635864), ('government', 0.999639093875885), ('president', 0.9996323585510254), ('state', 0.9996278882026672), ('police', 0.9996159076690674), ('tuesday', 0.9996143579483032), ('military', 0.9995691776275635), ('security', 0.9995645880699158)]\n",
      "[('water', 0.9985550045967102), ('standing', 0.9978150129318237), ('walking', 0.9976155161857605), ('cat', 0.9973345398902893), ('wearing', 0.9969745874404907), ('green', 0.9968481063842773), ('large', 0.996844470500946), ('laying', 0.99665766954422), ('shirt', 0.9966006875038147), ('beach', 0.9963244199752808)]\n",
      "[('said', 0.9999098777770996), ('million', 0.9996146559715271), ('tuesday', 0.9995675086975098), ('government', 0.9995554089546204), ('military', 0.9995068311691284), ('billion', 0.9994760155677795), ('security', 0.9994750618934631), ('president', 0.9994511008262634), ('police', 0.9994447827339172), ('high', 0.9994063973426819)]\n",
      "[('said', 0.9997278451919556), ('billion', 0.9995597004890442), ('tuesday', 0.9995173215866089), ('indian', 0.9994814991950989), ('million', 0.9994781017303467), ('years', 0.999477744102478), ('state', 0.999445915222168), ('arrested', 0.9994416832923889), ('say', 0.9994085431098938), ('police', 0.9994056224822998)]\n",
      "[('indian', 0.9991279244422913), ('near', 0.9990482926368713), ('air', 0.9990220069885254), ('dies', 0.9990163445472717), ('told', 0.998894453048706), ('said', 0.9988046884536743), ('court', 0.9987913370132446), ('children', 0.9987783432006836), ('arrested', 0.9987454414367676), ('major', 0.9986768960952759)]\n",
      "[('said', 0.9998658895492554), ('million', 0.9996998310089111), ('government', 0.9996566772460938), ('billion', 0.9996330738067627), ('tuesday', 0.999629020690918), ('president', 0.9996071457862854), ('police', 0.9995977282524109), ('state', 0.9995574355125427), ('yesterday', 0.9995519518852234), ('military', 0.9995507001876831)]\n",
      "[('men', 0.9987792372703552), ('air', 0.9986473917961121), ('dies', 0.9986045360565186), ('near', 0.9985849857330322), ('indian', 0.9984719157218933), ('area', 0.9983622431755066), ('people', 0.9983249306678772), ('children', 0.9982901215553284), ('women', 0.9981650114059448), ('room', 0.9981545805931091)]\n",
      "[('water', 0.9987273812294006), ('cat', 0.9976967573165894), ('walking', 0.9975780248641968), ('large', 0.9974914789199829), ('standing', 0.9974740147590637), ('green', 0.9974294900894165), ('wearing', 0.9973902702331543), ('laying', 0.9971055388450623), ('small', 0.9969626665115356), ('shirt', 0.9969197511672974)]\n",
      "[('singing', 0.9872704148292542), ('man', 0.9872002005577087), ('playing', 0.9834654331207275), ('boy', 0.9824506640434265), ('girl', 0.9795342683792114), ('plays', 0.9770668148994446), ('dogs', 0.9769740700721741), ('young', 0.9747228622436523), ('shirt', 0.9738288521766663), ('sitting', 0.9731125831604004)]\n",
      "[('wearing', 0.9986225962638855), ('young', 0.9983915090560913), ('large', 0.9983195066452026), ('girl', 0.9982755184173584), ('water', 0.9982449412345886), ('small', 0.9979092478752136), ('green', 0.9978635311126709), ('woman', 0.997851550579071), ('blue', 0.997815728187561), ('shirt', 0.9975525140762329)]\n",
      "[('white', 0.9970255494117737), ('black', 0.9969334006309509), ('field', 0.9965169429779053), ('standing', 0.9959350228309631), ('runs', 0.9954880475997925), ('dog', 0.9940996170043945), ('walking', 0.9940255880355835), ('mouth', 0.9939853549003601), ('running', 0.9939004778862), ('cow', 0.9938557744026184)]\n",
      "[('said', 0.9998716115951538), ('billion', 0.9996348023414612), ('million', 0.9996046423912048), ('tuesday', 0.9995864033699036), ('state', 0.9995753765106201), ('government', 0.9995702505111694), ('president', 0.9995403289794922), ('police', 0.9995337724685669), ('years', 0.999531626701355), ('indian', 0.9994869232177734)]\n"
     ]
    }
   ],
   "source": [
    "print(\"Top term per cluster:\")\n",
    "\n",
    "for i in range(kmeans.n_clusters):\n",
    "    centroid = kmeans.cluster_centers_[i]\n",
    "    print(model.wv.most_similar(centroid, topn=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11498, 11498)\n"
     ]
    }
   ],
   "source": [
    "similarity_matrix2 = cosine_similarity(mixed_corpus)\n",
    "print(similarity_matrix2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0000000000000007\n"
     ]
    }
   ],
   "source": [
    "print(np.max(similarity_matrix)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}