elasticsearch_overview.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Introduction to Elasticsearch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "First, let's initialize a connection to Elasticsearch. We'll call it `es`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from elasticsearch import Elasticsearch\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "# by default we connect to localhost:9200\n",
    "es = Elasticsearch()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Next, let's create an index called `test-index`. We can use this for storing some sample documents, which we will later visualize in Kibana."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acknowledged': True}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.delete(index='speeches')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acknowledged': True, 'index': 'speeches', 'shards_acknowledged': True}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.create(index='speeches')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now that we have an index, we can put some documents into the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "speeches = pd.read_csv('data/SecDef_Speeches.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "months = {'Jan': 'January',\n",
    "          'Feb': 'February',\n",
    "          'Mar': 'March',\n",
    "          'Apr': 'April',\n",
    "          'May': 'May',\n",
    "          'Jun': 'June',\n",
    "          'Jul': 'July',\n",
    "          'Aug': 'August',\n",
    "          'Sep': 'September',\n",
    "          'Sept': 'September',\n",
    "          'Oct': 'October',\n",
    "          'Nov': 'November',\n",
    "          'Dec': 'December',\n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def format_date(date_str, months):\n",
    "    if '.' in date_str:\n",
    "        month = months[date_str.split('.')[0]]\n",
    "        date_str = month + date_str.split('.')[1]\n",
    "    date = datetime.strptime(date_str, '%B %d, %Y')\n",
    "    return date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "count = 1\n",
    "for kk, vv in speeches.iterrows():\n",
    "    doc = {\n",
    "        \"title\": vv.title, \n",
    "        \"document\": vv.speech, \n",
    "        \"timestamp\": format_date(vv.date, months),\n",
    "    }\n",
    "    es.index(index=\"speeches\", doc_type=\"secdef\", id=count, body=doc)\n",
    "    count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date                                         March 17, 2016\n",
       "speech    I. PURPOSE OF THIS TESTIMONY Chairman McCain, ...\n",
       "title     Submitted Statement -- Senate Armed Services C...\n",
       "url       https://www.defense.gov/News/Speeches/Speech-V...\n",
       "Name: 203, dtype: object"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "    doc = {\n",
    "        \"title\": vv.title, \n",
    "        \"document\": vv.speech, \n",
    "        \"my_random_field\": \"shouldn't this puke!!?!\",\n",
    "        \"timestamp\": format_date(vv.date, months),\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'_id': '205',\n",
       " '_index': 'speeches',\n",
       " '_primary_term': 1,\n",
       " '_seq_no': 47,\n",
       " '_shards': {'failed': 0, 'successful': 1, 'total': 2},\n",
       " '_type': 'secdef',\n",
       " '_version': 1,\n",
       " 'result': 'created'}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.index(index=\"speeches\", doc_type=\"secdef\", id=count, body=doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "query = {\n",
    "    \"query\": {\n",
    "        \"match\" : {\n",
    "            \"my_random_field\": \"puke\"\n",
    "            }\n",
    "        }\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "To actually run the search try:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results = es.search(body=query, index='speeches', doc_type='secdef')['hits']['hits']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Submitted Statement -- Senate Armed Services Committee (FY 2017 Budget Request)\n"
     ]
    }
   ],
   "source": [
    "for i in results:\n",
    "    print(i['_source']['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}