notebooks/Checking the dumps.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "sys.path.append(os.path.join(os.path.dirname(os.path.realpath('__file__')), \"..\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openqas\n",
    "from openqas.retriever.retriever import WikiRetriever"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranker = WikiRetriever(\"../data/wiki_00\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12</td>\n",
       "      <td>Anarchism\\n\\nAnarchism is a political philosop...</td>\n",
       "      <td>Anarchism</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25</td>\n",
       "      <td>Autism\\n\\nAutism is a developmental disorder c...</td>\n",
       "      <td>Autism</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>39</td>\n",
       "      <td>Albedo\\n\\nAlbedo () (, meaning 'whiteness') is...</td>\n",
       "      <td>Albedo</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>290</td>\n",
       "      <td>A\\n\\nA (named , plural \"As\", \"A's\", \"a\"s, \"a's...</td>\n",
       "      <td>A</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>303</td>\n",
       "      <td>Alabama\\n\\nAlabama is a state in the southeast...</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>305</td>\n",
       "      <td>Achilles\\n\\nIn Greek mythology, Achilles or Ac...</td>\n",
       "      <td>Achilles</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>307</td>\n",
       "      <td>Abraham Lincoln\\n\\nAbraham Lincoln (February 1...</td>\n",
       "      <td>Abraham Lincoln</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>308</td>\n",
       "      <td>Aristotle\\n\\nAristotle (; \"Aristotélēs\",; 384–...</td>\n",
       "      <td>Aristotle</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>309</td>\n",
       "      <td>An American in Paris\\n\\nAn American in Paris i...</td>\n",
       "      <td>An American in Paris</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>316</td>\n",
       "      <td>Academy Award for Best Production Design\\n\\nTh...</td>\n",
       "      <td>Academy Award for Best Production Design</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>324</td>\n",
       "      <td>Academy Awards\\n\\nThe Academy Awards, also kno...</td>\n",
       "      <td>Academy Awards</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>330</td>\n",
       "      <td>Actrius\\n\\nActresses (Catalan: Actrius) is a 1...</td>\n",
       "      <td>Actrius</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>332</td>\n",
       "      <td>Animalia (book)\\n\\nAnimalia is an illustrated ...</td>\n",
       "      <td>Animalia (book)</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>334</td>\n",
       "      <td>International Atomic Time\\n\\nInternational Ato...</td>\n",
       "      <td>International Atomic Time</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>336</td>\n",
       "      <td>Altruism\\n\\nAltruism is the principle and mora...</td>\n",
       "      <td>Altruism</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>339</td>\n",
       "      <td>Ayn Rand\\n\\nAyn Rand (; born Alisa Zinovyevna ...</td>\n",
       "      <td>Ayn Rand</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>340</td>\n",
       "      <td>Alain Connes\\n\\nAlain Connes (; born 1 April 1...</td>\n",
       "      <td>Alain Connes</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>344</td>\n",
       "      <td>Allan Dwan\\n\\nAllan Dwan (3 April 1885 – 28 De...</td>\n",
       "      <td>Allan Dwan</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>358</td>\n",
       "      <td>Algeria\\n\\nAlgeria (; ', Algerian Arabic '; ),...</td>\n",
       "      <td>Algeria</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>359</td>\n",
       "      <td>List of Atlas Shrugged characters\\n\\nThis is a...</td>\n",
       "      <td>List of Atlas Shrugged characters</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>569</td>\n",
       "      <td>Anthropology\\n\\nAnthropology is the study of h...</td>\n",
       "      <td>Anthropology</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>572</td>\n",
       "      <td>Agricultural science\\n\\nAgricultural science i...</td>\n",
       "      <td>Agricultural science</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>573</td>\n",
       "      <td>Alchemy\\n\\nAlchemy (from Arabic: \"al-kīmiyā\") ...</td>\n",
       "      <td>Alchemy</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>579</td>\n",
       "      <td>Alien\\n\\nAlien primarily refers to:\\n\\nAlien(s...</td>\n",
       "      <td>Alien</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>580</td>\n",
       "      <td>Astronomer\\n\\nAn astronomer is a scientist in ...</td>\n",
       "      <td>Astronomer</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>586</td>\n",
       "      <td>ASCII\\n\\nASCII ( ), abbreviated from American ...</td>\n",
       "      <td>ASCII</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>590</td>\n",
       "      <td>Austin (disambiguation)\\n\\nAustin is the capit...</td>\n",
       "      <td>Austin (disambiguation)</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>593</td>\n",
       "      <td>Animation\\n\\nAnimation is a method in which pi...</td>\n",
       "      <td>Animation</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>594</td>\n",
       "      <td>Apollo\\n\\nApollo (Attic, Ionic, and Homeric Gr...</td>\n",
       "      <td>Apollo</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>595</td>\n",
       "      <td>Andre Agassi\\n\\nAndre Kirk Agassi ( ; born Apr...</td>\n",
       "      <td>Andre Agassi</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6163</th>\n",
       "      <td>12899</td>\n",
       "      <td>Gestapo\\n\\nThe Gestapo (), abbreviation of Geh...</td>\n",
       "      <td>Gestapo</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6164</th>\n",
       "      <td>12900</td>\n",
       "      <td>Grammatical conjugation\\n\\nIn linguistics, con...</td>\n",
       "      <td>Grammatical conjugation</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6165</th>\n",
       "      <td>12902</td>\n",
       "      <td>Gomoku\\n\\nGomoku, also called \"Five in a Row\",...</td>\n",
       "      <td>Gomoku</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6166</th>\n",
       "      <td>12903</td>\n",
       "      <td>Gegenschein\\n\\nGegenschein ( German for \"count...</td>\n",
       "      <td>Gegenschein</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6167</th>\n",
       "      <td>12904</td>\n",
       "      <td>Glyph\\n\\nIn typography, a glyph is an elementa...</td>\n",
       "      <td>Glyph</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6168</th>\n",
       "      <td>12905</td>\n",
       "      <td>Goth subculture\\n\\nThe goth subculture is a su...</td>\n",
       "      <td>Goth subculture</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6169</th>\n",
       "      <td>12908</td>\n",
       "      <td>Global warming potential\\n\\nGlobal warming pot...</td>\n",
       "      <td>Global warming potential</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6170</th>\n",
       "      <td>12910</td>\n",
       "      <td>Grothendieck topology\\n\\nIn category theory, a...</td>\n",
       "      <td>Grothendieck topology</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6171</th>\n",
       "      <td>12913</td>\n",
       "      <td>Greens\\n\\nGreens may refer to:\\n\\n\\n\\n\\n\\n\\n</td>\n",
       "      <td>Greens</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6172</th>\n",
       "      <td>12914</td>\n",
       "      <td>Ghost in the Shell\\n\\nAnimation studio Product...</td>\n",
       "      <td>Ghost in the Shell</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6173</th>\n",
       "      <td>12916</td>\n",
       "      <td>Gauss–Legendre algorithm\\n\\nThe Gauss–Legendre...</td>\n",
       "      <td>Gauss–Legendre algorithm</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6174</th>\n",
       "      <td>12917</td>\n",
       "      <td>Great Internet Mersenne Prime Search\\n\\nThe Gr...</td>\n",
       "      <td>Great Internet Mersenne Prime Search</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6175</th>\n",
       "      <td>12919</td>\n",
       "      <td>Game.com\\n\\nThe Game.com is a fifth-generation...</td>\n",
       "      <td>Game.com</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176</th>\n",
       "      <td>12920</td>\n",
       "      <td>General Packet Radio Service\\n\\nGeneral Packet...</td>\n",
       "      <td>General Packet Radio Service</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6177</th>\n",
       "      <td>12922</td>\n",
       "      <td>Gnosis\\n\\nGnosis is the common Greek noun for ...</td>\n",
       "      <td>Gnosis</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6178</th>\n",
       "      <td>12923</td>\n",
       "      <td>Georgian\\n\\nGeorgian may refer to:\\n\\n\\n\\n\\n\\n...</td>\n",
       "      <td>Georgian</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6179</th>\n",
       "      <td>12924</td>\n",
       "      <td>Georgian architecture\\n\\nGeorgian architecture...</td>\n",
       "      <td>Georgian architecture</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6180</th>\n",
       "      <td>12926</td>\n",
       "      <td>Goshen, Indiana\\n\\nGoshen is a city in and the...</td>\n",
       "      <td>Goshen, Indiana</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6181</th>\n",
       "      <td>12929</td>\n",
       "      <td>Gallipoli\\n\\nThe Gallipoli peninsula (; ; , \"C...</td>\n",
       "      <td>Gallipoli</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6182</th>\n",
       "      <td>12935</td>\n",
       "      <td>Gram stain\\n\\nGram stain or Gram staining, als...</td>\n",
       "      <td>Gram stain</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6183</th>\n",
       "      <td>12936</td>\n",
       "      <td>Gram-positive bacteria\\n\\nGram-positive bacter...</td>\n",
       "      <td>Gram-positive bacteria</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6184</th>\n",
       "      <td>12937</td>\n",
       "      <td>Gram-negative bacteria\\n\\nGram-negative bacter...</td>\n",
       "      <td>Gram-negative bacteria</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6185</th>\n",
       "      <td>12938</td>\n",
       "      <td>Greyhound\\n\\nThe Greyhound is a breed of dog, ...</td>\n",
       "      <td>Greyhound</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6186</th>\n",
       "      <td>12939</td>\n",
       "      <td>Geometric algebra\\n\\nThe geometric algebra (GA...</td>\n",
       "      <td>Geometric algebra</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6187</th>\n",
       "      <td>12942</td>\n",
       "      <td>Genetic\\n\\nGenetic may refer to:\\n\\n</td>\n",
       "      <td>Genetic</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6188</th>\n",
       "      <td>12945</td>\n",
       "      <td>George Benson\\n\\nGeorge Benson (born March 22,...</td>\n",
       "      <td>George Benson</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6189</th>\n",
       "      <td>12946</td>\n",
       "      <td>Grigory Barenblatt\\n\\nGrigory Isaakovich Baren...</td>\n",
       "      <td>Grigory Barenblatt</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6190</th>\n",
       "      <td>12947</td>\n",
       "      <td>Grammatical tense\\n\\nIn grammar, tense is a ca...</td>\n",
       "      <td>Grammatical tense</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6191</th>\n",
       "      <td>12948</td>\n",
       "      <td>Grammatical aspect\\n\\nAspect is a grammatical ...</td>\n",
       "      <td>Grammatical aspect</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6192</th>\n",
       "      <td>12950</td>\n",
       "      <td>Glucose\\n\\nGlucose (also called dextrose) is a...</td>\n",
       "      <td>Glucose</td>\n",
       "      <td>https://en.wikipedia.org/wiki?curid=12950</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6193 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                               text  \\\n",
       "0        12  Anarchism\\n\\nAnarchism is a political philosop...   \n",
       "1        25  Autism\\n\\nAutism is a developmental disorder c...   \n",
       "2        39  Albedo\\n\\nAlbedo () (, meaning 'whiteness') is...   \n",
       "3       290  A\\n\\nA (named , plural \"As\", \"A's\", \"a\"s, \"a's...   \n",
       "4       303  Alabama\\n\\nAlabama is a state in the southeast...   \n",
       "5       305  Achilles\\n\\nIn Greek mythology, Achilles or Ac...   \n",
       "6       307  Abraham Lincoln\\n\\nAbraham Lincoln (February 1...   \n",
       "7       308  Aristotle\\n\\nAristotle (; \"Aristotélēs\",; 384–...   \n",
       "8       309  An American in Paris\\n\\nAn American in Paris i...   \n",
       "9       316  Academy Award for Best Production Design\\n\\nTh...   \n",
       "10      324  Academy Awards\\n\\nThe Academy Awards, also kno...   \n",
       "11      330  Actrius\\n\\nActresses (Catalan: Actrius) is a 1...   \n",
       "12      332  Animalia (book)\\n\\nAnimalia is an illustrated ...   \n",
       "13      334  International Atomic Time\\n\\nInternational Ato...   \n",
       "14      336  Altruism\\n\\nAltruism is the principle and mora...   \n",
       "15      339  Ayn Rand\\n\\nAyn Rand (; born Alisa Zinovyevna ...   \n",
       "16      340  Alain Connes\\n\\nAlain Connes (; born 1 April 1...   \n",
       "17      344  Allan Dwan\\n\\nAllan Dwan (3 April 1885 – 28 De...   \n",
       "18      358  Algeria\\n\\nAlgeria (; ', Algerian Arabic '; ),...   \n",
       "19      359  List of Atlas Shrugged characters\\n\\nThis is a...   \n",
       "20      569  Anthropology\\n\\nAnthropology is the study of h...   \n",
       "21      572  Agricultural science\\n\\nAgricultural science i...   \n",
       "22      573  Alchemy\\n\\nAlchemy (from Arabic: \"al-kīmiyā\") ...   \n",
       "23      579  Alien\\n\\nAlien primarily refers to:\\n\\nAlien(s...   \n",
       "24      580  Astronomer\\n\\nAn astronomer is a scientist in ...   \n",
       "25      586  ASCII\\n\\nASCII ( ), abbreviated from American ...   \n",
       "26      590  Austin (disambiguation)\\n\\nAustin is the capit...   \n",
       "27      593  Animation\\n\\nAnimation is a method in which pi...   \n",
       "28      594  Apollo\\n\\nApollo (Attic, Ionic, and Homeric Gr...   \n",
       "29      595  Andre Agassi\\n\\nAndre Kirk Agassi ( ; born Apr...   \n",
       "...     ...                                                ...   \n",
       "6163  12899  Gestapo\\n\\nThe Gestapo (), abbreviation of Geh...   \n",
       "6164  12900  Grammatical conjugation\\n\\nIn linguistics, con...   \n",
       "6165  12902  Gomoku\\n\\nGomoku, also called \"Five in a Row\",...   \n",
       "6166  12903  Gegenschein\\n\\nGegenschein ( German for \"count...   \n",
       "6167  12904  Glyph\\n\\nIn typography, a glyph is an elementa...   \n",
       "6168  12905  Goth subculture\\n\\nThe goth subculture is a su...   \n",
       "6169  12908  Global warming potential\\n\\nGlobal warming pot...   \n",
       "6170  12910  Grothendieck topology\\n\\nIn category theory, a...   \n",
       "6171  12913       Greens\\n\\nGreens may refer to:\\n\\n\\n\\n\\n\\n\\n   \n",
       "6172  12914  Ghost in the Shell\\n\\nAnimation studio Product...   \n",
       "6173  12916  Gauss–Legendre algorithm\\n\\nThe Gauss–Legendre...   \n",
       "6174  12917  Great Internet Mersenne Prime Search\\n\\nThe Gr...   \n",
       "6175  12919  Game.com\\n\\nThe Game.com is a fifth-generation...   \n",
       "6176  12920  General Packet Radio Service\\n\\nGeneral Packet...   \n",
       "6177  12922  Gnosis\\n\\nGnosis is the common Greek noun for ...   \n",
       "6178  12923  Georgian\\n\\nGeorgian may refer to:\\n\\n\\n\\n\\n\\n...   \n",
       "6179  12924  Georgian architecture\\n\\nGeorgian architecture...   \n",
       "6180  12926  Goshen, Indiana\\n\\nGoshen is a city in and the...   \n",
       "6181  12929  Gallipoli\\n\\nThe Gallipoli peninsula (; ; , \"C...   \n",
       "6182  12935  Gram stain\\n\\nGram stain or Gram staining, als...   \n",
       "6183  12936  Gram-positive bacteria\\n\\nGram-positive bacter...   \n",
       "6184  12937  Gram-negative bacteria\\n\\nGram-negative bacter...   \n",
       "6185  12938  Greyhound\\n\\nThe Greyhound is a breed of dog, ...   \n",
       "6186  12939  Geometric algebra\\n\\nThe geometric algebra (GA...   \n",
       "6187  12942               Genetic\\n\\nGenetic may refer to:\\n\\n   \n",
       "6188  12945  George Benson\\n\\nGeorge Benson (born March 22,...   \n",
       "6189  12946  Grigory Barenblatt\\n\\nGrigory Isaakovich Baren...   \n",
       "6190  12947  Grammatical tense\\n\\nIn grammar, tense is a ca...   \n",
       "6191  12948  Grammatical aspect\\n\\nAspect is a grammatical ...   \n",
       "6192  12950  Glucose\\n\\nGlucose (also called dextrose) is a...   \n",
       "\n",
       "                                         title  \\\n",
       "0                                    Anarchism   \n",
       "1                                       Autism   \n",
       "2                                       Albedo   \n",
       "3                                            A   \n",
       "4                                      Alabama   \n",
       "5                                     Achilles   \n",
       "6                              Abraham Lincoln   \n",
       "7                                    Aristotle   \n",
       "8                         An American in Paris   \n",
       "9     Academy Award for Best Production Design   \n",
       "10                              Academy Awards   \n",
       "11                                     Actrius   \n",
       "12                             Animalia (book)   \n",
       "13                   International Atomic Time   \n",
       "14                                    Altruism   \n",
       "15                                    Ayn Rand   \n",
       "16                                Alain Connes   \n",
       "17                                  Allan Dwan   \n",
       "18                                     Algeria   \n",
       "19           List of Atlas Shrugged characters   \n",
       "20                                Anthropology   \n",
       "21                        Agricultural science   \n",
       "22                                     Alchemy   \n",
       "23                                       Alien   \n",
       "24                                  Astronomer   \n",
       "25                                       ASCII   \n",
       "26                     Austin (disambiguation)   \n",
       "27                                   Animation   \n",
       "28                                      Apollo   \n",
       "29                                Andre Agassi   \n",
       "...                                        ...   \n",
       "6163                                   Gestapo   \n",
       "6164                   Grammatical conjugation   \n",
       "6165                                    Gomoku   \n",
       "6166                               Gegenschein   \n",
       "6167                                     Glyph   \n",
       "6168                           Goth subculture   \n",
       "6169                  Global warming potential   \n",
       "6170                     Grothendieck topology   \n",
       "6171                                    Greens   \n",
       "6172                        Ghost in the Shell   \n",
       "6173                  Gauss–Legendre algorithm   \n",
       "6174      Great Internet Mersenne Prime Search   \n",
       "6175                                  Game.com   \n",
       "6176              General Packet Radio Service   \n",
       "6177                                    Gnosis   \n",
       "6178                                  Georgian   \n",
       "6179                     Georgian architecture   \n",
       "6180                           Goshen, Indiana   \n",
       "6181                                 Gallipoli   \n",
       "6182                                Gram stain   \n",
       "6183                    Gram-positive bacteria   \n",
       "6184                    Gram-negative bacteria   \n",
       "6185                                 Greyhound   \n",
       "6186                         Geometric algebra   \n",
       "6187                                   Genetic   \n",
       "6188                             George Benson   \n",
       "6189                        Grigory Barenblatt   \n",
       "6190                         Grammatical tense   \n",
       "6191                        Grammatical aspect   \n",
       "6192                                   Glucose   \n",
       "\n",
       "                                            url  \n",
       "0        https://en.wikipedia.org/wiki?curid=12  \n",
       "1        https://en.wikipedia.org/wiki?curid=25  \n",
       "2        https://en.wikipedia.org/wiki?curid=39  \n",
       "3       https://en.wikipedia.org/wiki?curid=290  \n",
       "4       https://en.wikipedia.org/wiki?curid=303  \n",
       "5       https://en.wikipedia.org/wiki?curid=305  \n",
       "6       https://en.wikipedia.org/wiki?curid=307  \n",
       "7       https://en.wikipedia.org/wiki?curid=308  \n",
       "8       https://en.wikipedia.org/wiki?curid=309  \n",
       "9       https://en.wikipedia.org/wiki?curid=316  \n",
       "10      https://en.wikipedia.org/wiki?curid=324  \n",
       "11      https://en.wikipedia.org/wiki?curid=330  \n",
       "12      https://en.wikipedia.org/wiki?curid=332  \n",
       "13      https://en.wikipedia.org/wiki?curid=334  \n",
       "14      https://en.wikipedia.org/wiki?curid=336  \n",
       "15      https://en.wikipedia.org/wiki?curid=339  \n",
       "16      https://en.wikipedia.org/wiki?curid=340  \n",
       "17      https://en.wikipedia.org/wiki?curid=344  \n",
       "18      https://en.wikipedia.org/wiki?curid=358  \n",
       "19      https://en.wikipedia.org/wiki?curid=359  \n",
       "20      https://en.wikipedia.org/wiki?curid=569  \n",
       "21      https://en.wikipedia.org/wiki?curid=572  \n",
       "22      https://en.wikipedia.org/wiki?curid=573  \n",
       "23      https://en.wikipedia.org/wiki?curid=579  \n",
       "24      https://en.wikipedia.org/wiki?curid=580  \n",
       "25      https://en.wikipedia.org/wiki?curid=586  \n",
       "26      https://en.wikipedia.org/wiki?curid=590  \n",
       "27      https://en.wikipedia.org/wiki?curid=593  \n",
       "28      https://en.wikipedia.org/wiki?curid=594  \n",
       "29      https://en.wikipedia.org/wiki?curid=595  \n",
       "...                                         ...  \n",
       "6163  https://en.wikipedia.org/wiki?curid=12899  \n",
       "6164  https://en.wikipedia.org/wiki?curid=12900  \n",
       "6165  https://en.wikipedia.org/wiki?curid=12902  \n",
       "6166  https://en.wikipedia.org/wiki?curid=12903  \n",
       "6167  https://en.wikipedia.org/wiki?curid=12904  \n",
       "6168  https://en.wikipedia.org/wiki?curid=12905  \n",
       "6169  https://en.wikipedia.org/wiki?curid=12908  \n",
       "6170  https://en.wikipedia.org/wiki?curid=12910  \n",
       "6171  https://en.wikipedia.org/wiki?curid=12913  \n",
       "6172  https://en.wikipedia.org/wiki?curid=12914  \n",
       "6173  https://en.wikipedia.org/wiki?curid=12916  \n",
       "6174  https://en.wikipedia.org/wiki?curid=12917  \n",
       "6175  https://en.wikipedia.org/wiki?curid=12919  \n",
       "6176  https://en.wikipedia.org/wiki?curid=12920  \n",
       "6177  https://en.wikipedia.org/wiki?curid=12922  \n",
       "6178  https://en.wikipedia.org/wiki?curid=12923  \n",
       "6179  https://en.wikipedia.org/wiki?curid=12924  \n",
       "6180  https://en.wikipedia.org/wiki?curid=12926  \n",
       "6181  https://en.wikipedia.org/wiki?curid=12929  \n",
       "6182  https://en.wikipedia.org/wiki?curid=12935  \n",
       "6183  https://en.wikipedia.org/wiki?curid=12936  \n",
       "6184  https://en.wikipedia.org/wiki?curid=12937  \n",
       "6185  https://en.wikipedia.org/wiki?curid=12938  \n",
       "6186  https://en.wikipedia.org/wiki?curid=12939  \n",
       "6187  https://en.wikipedia.org/wiki?curid=12942  \n",
       "6188  https://en.wikipedia.org/wiki?curid=12945  \n",
       "6189  https://en.wikipedia.org/wiki?curid=12946  \n",
       "6190  https://en.wikipedia.org/wiki?curid=12947  \n",
       "6191  https://en.wikipedia.org/wiki?curid=12948  \n",
       "6192  https://en.wikipedia.org/wiki?curid=12950  \n",
       "\n",
       "[6193 rows x 4 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ranker.wikidocs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranker.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys.getsizeof(ranker.retriever.tfidf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "152.7134132385254"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ranker.retriever.tfidf_matrix.data.nbytes + ranker.retriever.tfidf_matrix.indptr.nbytes + ranker.retriever.tfidf_matrix.indices.nbytes )/ (1024**2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "194.50710487365723"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ranker.wikidocs.memory_usage(deep=True, index=True).sum()/(1024**2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}