examples/notebooks/index_api.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Indexing Demo.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TNdMHD8LS2Bm",
        "colab_type": "text"
      },
      "source": [
        "# PyTerrier Index Analysis examples\n",
        "\n",
        "This notebook takes you through how to access an index directly in [Pyterrier](https://github.com/terrier-org/pyterrier)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pkMrNonZrpEg",
        "colab_type": "text"
      },
      "source": [
        "## Prerequisites\n",
        "\n",
        "You will need Pyterrier installed. Pyterrier also needs Java to be installed, and will find most installations."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JWLqWXBHeBRc",
        "colab_type": "code",
        "outputId": "618fe9cb-2bd3-47ff-fbdf-816d4b1cadce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 765
        }
      },
      "source": [
        "!pip install python-terrier\n",
        "# !pip install --upgrade git+https://github.com/terrier-org/pyterrier.git#egg=python-terrier"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting python-terrier\n",
            "  Cloning https://github.com/terrier-org/pyterrier.git to /tmp/pip-install-90ambft7/python-terrier\n",
            "  Running command git clone -q https://github.com/terrier-org/pyterrier.git /tmp/pip-install-90ambft7/python-terrier\n",
            "Collecting pyjnius~=1.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/50/098cb5fb76fb7c7d99d403226a2a63dcbfb5c129b71b7d0f5200b05de1f0/pyjnius-1.3.0-cp36-cp36m-manylinux2010_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from python-terrier) (1.18.4)\n",
            "Requirement already satisfied, skipping upgrade: pandas in /usr/local/lib/python3.6/dist-packages (from python-terrier) (1.0.4)\n",
            "Collecting wget\n",
            "  Downloading https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip\n",
            "Collecting pytrec_eval\n",
            "  Downloading https://files.pythonhosted.org/packages/36/0a/5809ba805e62c98f81e19d6007132712945c78e7612c11f61bac76a25ba3/pytrec_eval-0.4.tar.gz\n",
            "Requirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from python-terrier) (4.41.1)\n",
            "Collecting matchpy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/47/95/d265b944ce391bb2fa9982d7506bbb197bb55c5088ea74448a5ffcaeefab/matchpy-0.5.1-py3-none-any.whl (67kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 6.3MB/s \n",
            "\u001b[?25hCollecting deprecation\n",
            "  Downloading https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: cython in /usr/local/lib/python3.6/dist-packages (from pyjnius~=1.3.0->python-terrier) (0.29.19)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.7.0 in /usr/local/lib/python3.6/dist-packages (from pyjnius~=1.3.0->python-terrier) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas->python-terrier) (2.8.1)\n",
            "Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->python-terrier) (2018.9)\n",
            "Collecting multiset<3.0,>=2.0\n",
            "  Downloading https://files.pythonhosted.org/packages/a8/12/813a649f5bc9801865dc6cda95b8f169f784d996322db192907ebe399064/multiset-2.1.1-py2.py3-none-any.whl\n",
            "Collecting hopcroftkarp<2.0,>=1.2\n",
            "  Downloading https://files.pythonhosted.org/packages/6b/56/7b03eba3c43008c490c9d52e69ea5334b65955f66836eb4f1962f3b0d421/hopcroftkarp-1.2.5.tar.gz\n",
            "Requirement already satisfied, skipping upgrade: packaging in /usr/local/lib/python3.6/dist-packages (from deprecation->python-terrier) (20.4)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->deprecation->python-terrier) (2.4.7)\n",
            "Building wheels for collected packages: python-terrier, wget, pytrec-eval, hopcroftkarp\n",
            "  Building wheel for python-terrier (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-terrier: filename=python_terrier-0.1.3-cp36-none-any.whl size=29582 sha256=ca0ef83f096c225840fc2b4090ca2d292ac33b89a8755eb540c425683ba02a74\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-mab8ak6s/wheels/cc/bb/69/836d846a92c787b35ca6478119c0033762ab2b95d866eeb288\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-cp36-none-any.whl size=9682 sha256=8e2c7f0e5c7ee878b901dd3be2a833522f8963baaca72896d719ded8386a5796\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/15/30/7d8f7cea2902b4db79e3fea550d7d7b85ecb27ef992b618f3f\n",
            "  Building wheel for pytrec-eval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pytrec-eval: filename=pytrec_eval-0.4-cp36-cp36m-linux_x86_64.whl size=273892 sha256=9dedcd6e836e417c794af5a208cb0138968c57a8438221c59f5825d2f0a2c4b9\n",
            "  Stored in directory: /root/.cache/pip/wheels/58/30/73/8858a1b6e5e2674e2ea85c9904949c06addcf6fd34d59b5ea6\n",
            "  Building wheel for hopcroftkarp (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for hopcroftkarp: filename=hopcroftkarp-1.2.5-py2.py3-none-any.whl size=18092 sha256=ebd6fee3a8d74de3f94f8761d858d42006c514966b1ca96735339c0cf93d01ee\n",
            "  Stored in directory: /root/.cache/pip/wheels/2b/e1/c9/1993c7f7f114b7d3fb2d3e895e02157a7ebf554861e9e54e01\n",
            "Successfully built python-terrier wget pytrec-eval hopcroftkarp\n",
            "Installing collected packages: pyjnius, wget, pytrec-eval, multiset, hopcroftkarp, matchpy, deprecation, python-terrier\n",
            "Successfully installed deprecation-2.1.0 hopcroftkarp-1.2.5 matchpy-0.5.1 multiset-2.1.1 pyjnius-1.3.0 python-terrier-0.1.3 pytrec-eval-0.4 wget-3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ixzDjvtTOQbB",
        "colab_type": "text"
      },
      "source": [
        "## Init \n",
        "\n",
        "You must run `pt.init()` before other pyterrier functions and classes\n",
        "\n",
        "Optional Arguments:    \n",
        " - `version` - terrier IR version e.g. \"5.2\"    \n",
        " - `mem` - megabytes allocated to java e.g. \"4096\"      \n",
        " - `packages` - external java packages for Terrier to load e.g. [\"org.terrier:terrier.prf\"]\n",
        " - `logging` - logging level for Terrier. Defaults to \"WARN\", use \"INFO\" or \"DEBUG\" for more output.\n",
        "\n",
        "NB: Pyterrier needs Java 11 installed. If it cannot find your Java installation, you can set the `JAVA_HOME` environment variable."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k3ltUZ8PgWmz",
        "colab_type": "code",
        "outputId": "28681afb-8132-4cde-a0eb-6dc4c53da1b4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "import pyterrier as pt\n",
        "if not pt.started():\n",
        "  pt.init()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "terrier-assemblies 5.2  jar-with-dependencies not found, downloading to /root/.pyterrier...\n",
            "Done\n",
            "terrier-python-helper 0.0.2  jar not found, downloading to /root/.pyterrier...\n",
            "Done\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "By5UFYnRLgD0",
        "colab_type": "text"
      },
      "source": [
        "## Loading an Index\n",
        "\n",
        "Here, we are going to make use of Pyterrier's dataset API. We will use the [vaswani_npl corpus](http://ir.dcs.gla.ac.uk/resources/test_collections/npl/), a very small information retrieval test collection. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eRx5kIL9nmsB",
        "colab_type": "code",
        "outputId": "b6774593-d847-4d23-bd77-c266b8a32d2f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "dataset = pt.datasets.get_dataset(\"vaswani\")\n",
        "\n",
        "indexref = dataset.get_index()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading vaswani index to /root/.pyterrier/corpora/vaswani/index\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "orZfC5vY-NQT",
        "colab_type": "text"
      },
      "source": [
        "Lets have a look at the statistics of this index."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YxW1gSJh-MLr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "d0ef80f7-de1a-4e1f-ddb3-dd8d98d634e7"
      },
      "source": [
        "index = pt.IndexFactory.of(indexref)\n",
        "\n",
        "print(index.getCollectionStatistics().toString())"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of documents: 11429\n",
            "Number of terms: 7756\n",
            "Number of fields: 0\n",
            "Field names: []\n",
            "Number of tokens: 271581\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KLlaucxq-i4b",
        "colab_type": "text"
      },
      "source": [
        "## Using a Terrier index in your own code"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W7zH4-Y2-n7a",
        "colab_type": "text"
      },
      "source": [
        "### How many documents does term X occur in?\n",
        "\n",
        "As our index is stemmed, we used the stemmed form of the word 'chemical' which is 'chemic'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r1en_8ga-Y0d",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8feff0b9-53ab-4acd-be51-150359b9c3ff"
      },
      "source": [
        "index.getLexicon()[\"chemic\"].getDocumentFrequency()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "20"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_61z-MQQ-48S",
        "colab_type": "text"
      },
      "source": [
        "### What is the un-smoothed probability of term Y occurring in the collection?\n",
        "\n",
        "Here, we again use the [Lexicon](http://terrier.org/docs/current/javadoc/org/terrier/structures/Lexicon.html) of the underlying Terrier index. We check that the term occurs in the lexicon (to prevent a KeyError). The Lexicon returns a [LexiconEntry](http://terrier.org/docs/current/javadoc/org/terrier/structures/LexiconEntry.html), which allows us access to the number of occurrences of the term in the index.\n",
        "\n",
        "Finally, we use the [CollectionStatistics](http://terrier.org/docs/current/javadoc/org/terrier/structures/CollectionStatistics.html) object to determine the total number of occurrences of all terms in the index."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7jEPzyru-tCF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b0f0b461-be41-4382-cfa2-22de5cac3f1c"
      },
      "source": [
        "index.getLexicon()[\"chemic\"].getFrequency() / index.getCollectionStatistics().getNumberOfTokens() if \"chemic\" in index.getLexicon() else 0"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7.732499696223226e-05"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4O5oTE5m_72-",
        "colab_type": "text"
      },
      "source": [
        "### What terms occur in the 11th document?\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DS2aAdbE_OqA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "outputId": "1c0abbad-0664-423a-8c51-b41d0f22124b"
      },
      "source": [
        "di = index.getDirectIndex()\n",
        "doi = index.getDocumentIndex()\n",
        "lex = index.getLexicon()\n",
        "docid = 10 #docids are 0-based\n",
        "#NB: postings will be null if the document is empty\n",
        "for posting in  di.getPostings(doi.getDocumentEntry(docid)):\n",
        "  termid = posting.getId()\n",
        "  lee = lex.getLexiconEntry(termid)\n",
        "  print(\"%s with frequency %d\" % (lee.getKey(),posting.getFrequency()))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "circuit with frequency 3\n",
            "transistor with frequency 1\n",
            "us with frequency 1\n",
            "obtain with frequency 1\n",
            "switch with frequency 2\n",
            "design with frequency 1\n",
            "affect with frequency 1\n",
            "plot with frequency 1\n",
            "junction with frequency 1\n",
            "characterist with frequency 1\n",
            "paramet with frequency 1\n",
            "relat with frequency 1\n",
            "theoret with frequency 1\n",
            "load with frequency 1\n",
            "bistabl with frequency 1\n",
            "curv with frequency 1\n",
            "mai with frequency 1\n",
            "diagram with frequency 1\n",
            "line with frequency 1\n",
            "static with frequency 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pOMptWqtAmUP",
        "colab_type": "text"
      },
      "source": [
        "### What documents does term \"Z\" occur in?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CfDga790AZ26",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "outputId": "ab52b562-cc5b-4733-a9fc-934ca7fc71f5"
      },
      "source": [
        "meta = index.getMetaIndex()\n",
        "inv = index.getInvertedIndex()\n",
        "\n",
        "le = lex.getLexiconEntry( \"chemic\" )\n",
        "# the lexicon entry is also our pointer to access the inverted index posting list\n",
        "for posting in inv.getPostings( le ): \n",
        "\tdocno = meta.getItem(\"docno\", posting.getId())\n",
        "\tprint(\"%s with frequency %d \" % (docno, posting.getFrequency()))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1056 with frequency 1 \n",
            "1140 with frequency 1 \n",
            "2050 with frequency 1 \n",
            "2417 with frequency 1 \n",
            "2520 with frequency 1 \n",
            "2558 with frequency 1 \n",
            "3320 with frequency 1 \n",
            "4054 with frequency 1 \n",
            "4687 with frequency 1 \n",
            "4886 with frequency 1 \n",
            "4912 with frequency 1 \n",
            "6129 with frequency 1 \n",
            "6279 with frequency 2 \n",
            "7049 with frequency 1 \n",
            "8416 with frequency 1 \n",
            "8766 with frequency 1 \n",
            "9374 with frequency 1 \n",
            "10139 with frequency 1 \n",
            "10445 with frequency 1 \n",
            "10703 with frequency 1 \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pWdJTL58BLge",
        "colab_type": "text"
      },
      "source": [
        "Our index does not have position information, but *if it did*, the above loop would look like:\n",
        "\n",
        "```python\n",
        "for posting in inv.getPostings( le ): \n",
        "  docno = meta.getItem(\"docno\", posting.getId())\n",
        "  # unlike in Java, we dont need to cast posting to be a BlockPosting\n",
        "  positions = postings.getPositions()\n",
        "  print(\"%s with frequency %d and positions %s\" % (docno, posting.getFrequency(), str(positions))\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yb1tfXz0B5EY",
        "colab_type": "text"
      },
      "source": [
        "### What are the PL2 weighting model scores of documents that \"Y\" occurs in?\n",
        "\n",
        "Use of a WeightingModel class needs some setup, namely the [EntryStatistics](http://terrier.org/docs/current/javadoc/org/terrier/structures/EntryStatistics.html) of the term (obtained from the Lexicon, in the form of the LexiconEntry), as well as the CollectionStatistics (obtained from the index)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JVO3BjLTBBXb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "outputId": "c5d8e8d9-266a-4480-bc28-e08164af2bcc"
      },
      "source": [
        "inv = index.getInvertedIndex()\n",
        "meta = index.getMetaIndex()\n",
        "lex = index.getLexicon()\n",
        "le = lex.getLexiconEntry( \"chemic\" )\n",
        "wmodel = pt.autoclass(\"org.terrier.matching.models.PL2\")()\n",
        "wmodel.setCollectionStatistics(index.getCollectionStatistics())\n",
        "wmodel.setEntryStatistics(le);\n",
        "wmodel.setKeyFrequency(1)\n",
        "wmodel.prepare()\n",
        "for posting in inv.getPostings(le):\n",
        "  docno = meta.getItem(\"docno\", posting.getId())\n",
        "  score = wmodel.score(posting)\n",
        "  print(\"%s with score %0.4f\"  % (docno, score))\n"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1056 with score 6.3584\n",
            "1140 with score 5.3378\n",
            "2050 with score 4.5494\n",
            "2417 with score 4.5494\n",
            "2520 with score 5.1136\n",
            "2558 with score 5.1136\n",
            "3320 with score 1.5902\n",
            "4054 with score 2.1297\n",
            "4687 with score 5.0092\n",
            "4886 with score 6.1814\n",
            "4912 with score 4.2399\n",
            "6129 with score 3.0708\n",
            "6279 with score 5.6394\n",
            "7049 with score 4.3891\n",
            "8416 with score 1.9834\n",
            "8766 with score 5.3378\n",
            "9374 with score 4.4678\n",
            "10139 with score 5.2230\n",
            "10445 with score 3.6754\n",
            "10703 with score 6.9992\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j7HZ9-cPCkEU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [],
      "execution_count": 0,
      "outputs": []
    }
  ]
}