visualise_embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from gensim.models import KeyedVectors\n",
    "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n",
    "import plotly.graph_objs as go\n",
    "import umap\n",
    "\n",
    "init_notebook_mode(connected=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model loading - only top 100 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KeyedVectors.load_word2vec_format(\"../model.vec\", binary=False, limit=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocess word dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "tokens=[]\n",
    "word_labels=[]\n",
    "\n",
    "for word, vocab_obj in model.vocab.items():\n",
    "    word_labels.append(word)\n",
    "    tokens.append(model[word])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dimensionality reduction via umap (https://arxiv.org/abs/1802.03426) (much faster than TSNE with comparable results)  \n",
    "We need to project 128 dimensional vector to 2D space (X, Y coordinates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = umap.UMAP().fit_transform(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scatter plot visualisation using plotly (be carefull how many words you want to visualise this way), you can kill your browser :))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "data": [
        {
         "mode": "text",
         "text": [
          "</s>",
          "self",
          "x",
          "the",
          "if",
          "a",
          "u",
          "def",
          "in",
          "return",
          "none",
          "for",
          "to",
          "ai",
          "is",
          "of",
          "s",
          "b",
          "import",
          "c",
          "e",
          "d",
          "f",
          "and",
          "from",
          "not",
          "name",
          "true",
          "class",
          "n",
          "or",
          "i",
          "false",
          "this",
          "value",
          "type",
          "data",
          "with",
          "else",
          "as",
          "be",
          "r",
          "path",
          "t",
          "get",
          "os",
          "v",
          "assertequal",
          "android",
          "that",
          "p",
          "key",
          "args",
          "it",
          "file",
          "len",
          "str",
          "list",
          "xc",
          "np",
          "raise",
          "m",
          "id",
          "xe",
          "xf",
          "__init__",
          "xa",
          "y",
          "_",
          "an",
          "xb",
          "l",
          "xd",
          "append",
          "int",
          "permission",
          "license",
          "object",
          "try",
          "by",
          "default",
          "except",
          "print",
          "set",
          "string",
          "http",
          "on",
          "result",
          "test",
          "are",
          "ensmusg",
          "assert",
          "models",
          "types",
          "version",
          "elif",
          "request",
          "all",
          "module",
          "text"
         ],
         "textfont": {
          "size": 9
         },
         "type": "scatter",
         "x": [
          -0.9321006109533267,
          -1.9811381747981371,
          -3.6896686433851404,
          0.5896693181154222,
          -0.8261907776667907,
          -3.5908699743405044,
          -3.87807295522537,
          -1.9255569632546783,
          -0.4127322720160416,
          -1.8166708315073175,
          -1.5459447898041256,
          -0.7574937137394102,
          0.7330040769998445,
          -3.2514893631331496,
          0.2987908692138859,
          0.5887447685627598,
          -3.8109580560360876,
          -3.4748991721872016,
          0.7567238682954887,
          -3.934094204208825,
          -4.12220567976359,
          -3.921325280420335,
          -3.730341221087853,
          -0.21139254028197788,
          0.6496013621622373,
          -0.23454197626723627,
          -1.8398937436730314,
          -1.021380448116528,
          0.3567221175829417,
          -3.5781963722064787,
          -0.418890708234346,
          -2.972848337040655,
          -1.089642419266283,
          0.7835879986933897,
          -1.5239567509515977,
          -1.4088726019611717,
          -1.3501683674303167,
          0.2221543160361711,
          -1.2779814487315848,
          0.6191115214793028,
          1.0103941481206282,
          -3.61101019936476,
          -0.6015983793436126,
          -3.82590873532012,
          -1.1905401719125015,
          -0.9392694068320665,
          -3.0093508354678833,
          -1.3563239820110653,
          1.0757032111110079,
          1.2414048510448317,
          -3.4172238168289493,
          -1.4135810692796908,
          -1.8349474366982552,
          1.1402538841171428,
          -0.14920716022119784,
          -1.0198921809218413,
          -1.8451416539234053,
          -0.055723176783912795,
          -5.139274040864563,
          -2.559188103983255,
          -0.06804213464906064,
          -3.4630872030480635,
          -1.8797485480671918,
          -5.042158216041142,
          -5.354294007274164,
          -1.9920729708413327,
          -4.977142790790284,
          -3.4298600904369,
          -2.9109689478155403,
          1.0894041320927756,
          -5.279431980817133,
          -3.201626363038833,
          -5.126115346529922,
          -1.0780407073638387,
          -2.0376087866384065,
          0.9400879947312886,
          -0.07320893535776905,
          0.3094827074531141,
          0.7170350582918629,
          0.7793707908573919,
          -1.0941867449096796,
          0.1652798865577898,
          -2.3793134529065694,
          -0.6055956824013045,
          -1.5670927575273506,
          0.24916500218182783,
          1.224636237344838,
          -1.6501458370442343,
          -1.416346389638424,
          0.7554617901589002,
          -5.190945701212861,
          -1.0467651473638866,
          -1.0367141076849204,
          -0.4503332800548929,
          -0.7554273993937082,
          -0.760512131943402,
          -1.1064835214311943,
          0.5333063117499789,
          -0.5349140041706334,
          -1.17044789818965
         ],
         "y": [
          0.0183928235048426,
          -0.7500957642771074,
          1.2732259103685117,
          -0.8728152372792206,
          -0.9168912961865506,
          0.8520245903199652,
          1.0761451060986975,
          -1.097881855108073,
          -1.948918228075009,
          -0.5993520651375636,
          -0.5244289116752041,
          -2.1316081160622162,
          -0.3032025557130091,
          1.3158280948733851,
          -1.193208015664651,
          -0.17977757896636404,
          -0.36863642916398276,
          1.1292244855724358,
          0.8714252164879417,
          0.484034849028246,
          0.24472560154093131,
          0.3689071666474655,
          0.7517389162777047,
          -1.2350684517422221,
          0.6844703487159254,
          -0.889230336800281,
          -0.08541799347001328,
          0.7119029423517081,
          0.7574049618360335,
          -0.19248193375334882,
          -1.186100127904087,
          0.1801041453277789,
          0.4543006332282434,
          -0.037505161526582245,
          0.12298783926077997,
          0.8632696177013831,
          -0.44804711593397567,
          -0.45015231154358254,
          -0.832333235296656,
          -1.053505075391515,
          -1.1651371118245006,
          -0.12405972707370802,
          -0.6824192948942759,
          -0.3092688281848418,
          -0.7308926209353345,
          -1.333856759020342,
          0.6610365119863829,
          -1.349683970125166,
          0.24356772099227383,
          -0.6575622136039599,
          -0.07350813158555607,
          -0.09156918586613083,
          -1.1417869243363348,
          -0.6261934178163544,
          -0.15929028789012467,
          -1.8226398137999111,
          0.3771221974061168,
          -1.7847970683047896,
          4.220760572413106,
          -0.23026493534774917,
          -0.5146734462995705,
          0.29495881979822247,
          -0.13449715470413653,
          4.077337949492663,
          4.016526543601403,
          -0.9721859197841829,
          4.275917314934964,
          0.5575144817696753,
          0.6594021379110054,
          -0.7428926144627773,
          4.295002139347254,
          0.05714091589947399,
          3.9307313336219774,
          -1.8678284354439212,
          0.3731475961916635,
          0.16692614593761865,
          0.2880193292493865,
          0.6376770218161772,
          -0.38476018812525314,
          -0.5750848793876352,
          0.9540244239021212,
          0.1045836474999511,
          -0.7664649319061941,
          -1.903880659076165,
          0.5803231163657688,
          -0.11960161431133824,
          -0.9927923282295302,
          -0.7407492339652544,
          -1.260853574850886,
          -1.4855845641206322,
          3.826248292198038,
          -1.3203883316132272,
          1.071204430228317,
          0.88656535289645,
          0.2460348772334141,
          -1.0076974499251965,
          -0.507785134015971,
          -1.506856696490323,
          -0.07980905958831866,
          0.1085051361741971
         ]
        }
       ],
       "layout": {}
      },
      "text/html": [
       "<div id=\"8269337f-d199-492f-b51c-e892263abb90\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"8269337f-d199-492f-b51c-e892263abb90\", [{\"type\": \"scatter\", \"x\": [-0.9321006109533267, -1.9811381747981371, -3.6896686433851404, 0.5896693181154222, -0.8261907776667907, -3.5908699743405044, -3.87807295522537, -1.9255569632546783, -0.4127322720160416, -1.8166708315073175, -1.5459447898041256, -0.7574937137394102, 0.7330040769998445, -3.2514893631331496, 0.2987908692138859, 0.5887447685627598, -3.8109580560360876, -3.4748991721872016, 0.7567238682954887, -3.934094204208825, -4.12220567976359, -3.921325280420335, -3.730341221087853, -0.21139254028197788, 0.6496013621622373, -0.23454197626723627, -1.8398937436730314, -1.021380448116528, 0.3567221175829417, -3.5781963722064787, -0.418890708234346, -2.972848337040655, -1.089642419266283, 0.7835879986933897, -1.5239567509515977, -1.4088726019611717, -1.3501683674303167, 0.2221543160361711, -1.2779814487315848, 0.6191115214793028, 1.0103941481206282, -3.61101019936476, -0.6015983793436126, -3.82590873532012, -1.1905401719125015, -0.9392694068320665, -3.0093508354678833, -1.3563239820110653, 1.0757032111110079, 1.2414048510448317, -3.4172238168289493, -1.4135810692796908, -1.8349474366982552, 1.1402538841171428, -0.14920716022119784, -1.0198921809218413, -1.8451416539234053, -0.055723176783912795, -5.139274040864563, -2.559188103983255, -0.06804213464906064, -3.4630872030480635, -1.8797485480671918, -5.042158216041142, -5.354294007274164, -1.9920729708413327, -4.977142790790284, -3.4298600904369, -2.9109689478155403, 1.0894041320927756, -5.279431980817133, -3.201626363038833, -5.126115346529922, -1.0780407073638387, -2.0376087866384065, 0.9400879947312886, -0.07320893535776905, 0.3094827074531141, 0.7170350582918629, 0.7793707908573919, -1.0941867449096796, 0.1652798865577898, -2.3793134529065694, -0.6055956824013045, -1.5670927575273506, 0.24916500218182783, 1.224636237344838, -1.6501458370442343, -1.416346389638424, 0.7554617901589002, -5.190945701212861, -1.0467651473638866, -1.0367141076849204, -0.4503332800548929, -0.7554273993937082, -0.760512131943402, -1.1064835214311943, 0.5333063117499789, -0.5349140041706334, -1.17044789818965], \"y\": [0.0183928235048426, -0.7500957642771074, 1.2732259103685117, -0.8728152372792206, -0.9168912961865506, 0.8520245903199652, 1.0761451060986975, -1.097881855108073, -1.948918228075009, -0.5993520651375636, -0.5244289116752041, -2.1316081160622162, -0.3032025557130091, 1.3158280948733851, -1.193208015664651, -0.17977757896636404, -0.36863642916398276, 1.1292244855724358, 0.8714252164879417, 0.484034849028246, 0.24472560154093131, 0.3689071666474655, 0.7517389162777047, -1.2350684517422221, 0.6844703487159254, -0.889230336800281, -0.08541799347001328, 0.7119029423517081, 0.7574049618360335, -0.19248193375334882, -1.186100127904087, 0.1801041453277789, 0.4543006332282434, -0.037505161526582245, 0.12298783926077997, 0.8632696177013831, -0.44804711593397567, -0.45015231154358254, -0.832333235296656, -1.053505075391515, -1.1651371118245006, -0.12405972707370802, -0.6824192948942759, -0.3092688281848418, -0.7308926209353345, -1.333856759020342, 0.6610365119863829, -1.349683970125166, 0.24356772099227383, -0.6575622136039599, -0.07350813158555607, -0.09156918586613083, -1.1417869243363348, -0.6261934178163544, -0.15929028789012467, -1.8226398137999111, 0.3771221974061168, -1.7847970683047896, 4.220760572413106, -0.23026493534774917, -0.5146734462995705, 0.29495881979822247, -0.13449715470413653, 4.077337949492663, 4.016526543601403, -0.9721859197841829, 4.275917314934964, 0.5575144817696753, 0.6594021379110054, -0.7428926144627773, 4.295002139347254, 0.05714091589947399, 3.9307313336219774, -1.8678284354439212, 0.3731475961916635, 0.16692614593761865, 0.2880193292493865, 0.6376770218161772, -0.38476018812525314, -0.5750848793876352, 0.9540244239021212, 0.1045836474999511, -0.7664649319061941, -1.903880659076165, 0.5803231163657688, -0.11960161431133824, -0.9927923282295302, -0.7407492339652544, -1.260853574850886, -1.4855845641206322, 3.826248292198038, -1.3203883316132272, 1.071204430228317, 0.88656535289645, 0.2460348772334141, -1.0076974499251965, -0.507785134015971, -1.506856696490323, -0.07980905958831866, 0.1085051361741971], \"mode\": \"text\", \"text\": [\"</s>\", \"self\", \"x\", \"the\", \"if\", \"a\", \"u\", \"def\", \"in\", \"return\", \"none\", \"for\", \"to\", \"ai\", \"is\", \"of\", \"s\", \"b\", \"import\", \"c\", \"e\", \"d\", \"f\", \"and\", \"from\", \"not\", \"name\", \"true\", \"class\", \"n\", \"or\", \"i\", \"false\", \"this\", \"value\", \"type\", \"data\", \"with\", \"else\", \"as\", \"be\", \"r\", \"path\", \"t\", \"get\", \"os\", \"v\", \"assertequal\", \"android\", \"that\", \"p\", \"key\", \"args\", \"it\", \"file\", \"len\", \"str\", \"list\", \"xc\", \"np\", \"raise\", \"m\", \"id\", \"xe\", \"xf\", \"__init__\", \"xa\", \"y\", \"_\", \"an\", \"xb\", \"l\", \"xd\", \"append\", \"int\", \"permission\", \"license\", \"object\", \"try\", \"by\", \"default\", \"except\", \"print\", \"set\", \"string\", \"http\", \"on\", \"result\", \"test\", \"are\", \"ensmusg\", \"assert\", \"models\", \"types\", \"version\", \"elif\", \"request\", \"all\", \"module\", \"text\"], \"textfont\": {\"size\": 9}}], {}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<div id=\"8269337f-d199-492f-b51c-e892263abb90\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"8269337f-d199-492f-b51c-e892263abb90\", [{\"type\": \"scatter\", \"x\": [-0.9321006109533267, -1.9811381747981371, -3.6896686433851404, 0.5896693181154222, -0.8261907776667907, -3.5908699743405044, -3.87807295522537, -1.9255569632546783, -0.4127322720160416, -1.8166708315073175, -1.5459447898041256, -0.7574937137394102, 0.7330040769998445, -3.2514893631331496, 0.2987908692138859, 0.5887447685627598, -3.8109580560360876, -3.4748991721872016, 0.7567238682954887, -3.934094204208825, -4.12220567976359, -3.921325280420335, -3.730341221087853, -0.21139254028197788, 0.6496013621622373, -0.23454197626723627, -1.8398937436730314, -1.021380448116528, 0.3567221175829417, -3.5781963722064787, -0.418890708234346, -2.972848337040655, -1.089642419266283, 0.7835879986933897, -1.5239567509515977, -1.4088726019611717, -1.3501683674303167, 0.2221543160361711, -1.2779814487315848, 0.6191115214793028, 1.0103941481206282, -3.61101019936476, -0.6015983793436126, -3.82590873532012, -1.1905401719125015, -0.9392694068320665, -3.0093508354678833, -1.3563239820110653, 1.0757032111110079, 1.2414048510448317, -3.4172238168289493, -1.4135810692796908, -1.8349474366982552, 1.1402538841171428, -0.14920716022119784, -1.0198921809218413, -1.8451416539234053, -0.055723176783912795, -5.139274040864563, -2.559188103983255, -0.06804213464906064, -3.4630872030480635, -1.8797485480671918, -5.042158216041142, -5.354294007274164, -1.9920729708413327, -4.977142790790284, -3.4298600904369, -2.9109689478155403, 1.0894041320927756, -5.279431980817133, -3.201626363038833, -5.126115346529922, -1.0780407073638387, -2.0376087866384065, 0.9400879947312886, -0.07320893535776905, 0.3094827074531141, 0.7170350582918629, 0.7793707908573919, -1.0941867449096796, 0.1652798865577898, -2.3793134529065694, -0.6055956824013045, -1.5670927575273506, 0.24916500218182783, 1.224636237344838, -1.6501458370442343, -1.416346389638424, 0.7554617901589002, -5.190945701212861, -1.0467651473638866, -1.0367141076849204, -0.4503332800548929, -0.7554273993937082, -0.760512131943402, -1.1064835214311943, 0.5333063117499789, -0.5349140041706334, -1.17044789818965], \"y\": [0.0183928235048426, -0.7500957642771074, 1.2732259103685117, -0.8728152372792206, -0.9168912961865506, 0.8520245903199652, 1.0761451060986975, -1.097881855108073, -1.948918228075009, -0.5993520651375636, -0.5244289116752041, -2.1316081160622162, -0.3032025557130091, 1.3158280948733851, -1.193208015664651, -0.17977757896636404, -0.36863642916398276, 1.1292244855724358, 0.8714252164879417, 0.484034849028246, 0.24472560154093131, 0.3689071666474655, 0.7517389162777047, -1.2350684517422221, 0.6844703487159254, -0.889230336800281, -0.08541799347001328, 0.7119029423517081, 0.7574049618360335, -0.19248193375334882, -1.186100127904087, 0.1801041453277789, 0.4543006332282434, -0.037505161526582245, 0.12298783926077997, 0.8632696177013831, -0.44804711593397567, -0.45015231154358254, -0.832333235296656, -1.053505075391515, -1.1651371118245006, -0.12405972707370802, -0.6824192948942759, -0.3092688281848418, -0.7308926209353345, -1.333856759020342, 0.6610365119863829, -1.349683970125166, 0.24356772099227383, -0.6575622136039599, -0.07350813158555607, -0.09156918586613083, -1.1417869243363348, -0.6261934178163544, -0.15929028789012467, -1.8226398137999111, 0.3771221974061168, -1.7847970683047896, 4.220760572413106, -0.23026493534774917, -0.5146734462995705, 0.29495881979822247, -0.13449715470413653, 4.077337949492663, 4.016526543601403, -0.9721859197841829, 4.275917314934964, 0.5575144817696753, 0.6594021379110054, -0.7428926144627773, 4.295002139347254, 0.05714091589947399, 3.9307313336219774, -1.8678284354439212, 0.3731475961916635, 0.16692614593761865, 0.2880193292493865, 0.6376770218161772, -0.38476018812525314, -0.5750848793876352, 0.9540244239021212, 0.1045836474999511, -0.7664649319061941, -1.903880659076165, 0.5803231163657688, -0.11960161431133824, -0.9927923282295302, -0.7407492339652544, -1.260853574850886, -1.4855845641206322, 3.826248292198038, -1.3203883316132272, 1.071204430228317, 0.88656535289645, 0.2460348772334141, -1.0076974499251965, -0.507785134015971, -1.506856696490323, -0.07980905958831866, 0.1085051361741971], \"mode\": \"text\", \"text\": [\"</s>\", \"self\", \"x\", \"the\", \"if\", \"a\", \"u\", \"def\", \"in\", \"return\", \"none\", \"for\", \"to\", \"ai\", \"is\", \"of\", \"s\", \"b\", \"import\", \"c\", \"e\", \"d\", \"f\", \"and\", \"from\", \"not\", \"name\", \"true\", \"class\", \"n\", \"or\", \"i\", \"false\", \"this\", \"value\", \"type\", \"data\", \"with\", \"else\", \"as\", \"be\", \"r\", \"path\", \"t\", \"get\", \"os\", \"v\", \"assertequal\", \"android\", \"that\", \"p\", \"key\", \"args\", \"it\", \"file\", \"len\", \"str\", \"list\", \"xc\", \"np\", \"raise\", \"m\", \"id\", \"xe\", \"xf\", \"__init__\", \"xa\", \"y\", \"_\", \"an\", \"xb\", \"l\", \"xd\", \"append\", \"int\", \"permission\", \"license\", \"object\", \"try\", \"by\", \"default\", \"except\", \"print\", \"set\", \"string\", \"http\", \"on\", \"result\", \"test\", \"are\", \"ensmusg\", \"assert\", \"models\", \"types\", \"version\", \"elif\", \"request\", \"all\", \"module\", \"text\"], \"textfont\": {\"size\": 9}}], {}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_coords = Y[:, 0]\n",
    "y_coords = Y[:, 1]\n",
    "\n",
    "trace1 = go.Scatter(\n",
    "    x=x_coords,\n",
    "    y=y_coords,\n",
    "\n",
    "    mode='text',\n",
    "    text=word_labels,\n",
    "    textfont=dict(\n",
    "        size=9,\n",
    "    )\n",
    ")\n",
    "\n",
    "data=[trace1]\n",
    "fig = go.Figure(data=data)\n",
    "iplot(fig)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}