Wikipedia Parser Test-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
<<<<<<< HEAD
   "metadata": {
    "collapsed": true
   },
=======
   "metadata": {},
>>>>>>> master
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as etree\n",
    "import codecs\n",
    "import csv\n",
    "import time\n",
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
<<<<<<< HEAD
   "metadata": {
    "collapsed": true
   },
=======
   "metadata": {},
>>>>>>> master
   "outputs": [],
   "source": [
    "# Nicely formatted time string\n",
    "def hms_string(sec_elapsed):\n",
    "    h = int(sec_elapsed / (60 * 60))\n",
    "    m = int((sec_elapsed % (60 * 60)) / 60)\n",
    "    s = sec_elapsed % 60\n",
    "    return \"{}:{:>02}:{:>05.2f}\".format(h, m, s)\n",
    "def strip_tag_name(t):\n",
    "    t = elem.tag\n",
    "    idx = k = t.rfind(\"}\")\n",
    "    if idx != -1:\n",
    "        t = t[idx + 1:]\n",
    "    return t\n",
    "def get_drugbox(s):\n",
    "    beg = (s.rfind('{{Drugbox'))\n",
    "    end  =(s.rfind('\\n}}'))\n",
    "    if( end == -1):\n",
    "        end = end =(s.rfind('}}\\n'))\n",
    "    if( end == -1):\n",
    "        end = end =(s.rfind('}}\\n<!--'))\n",
    "    if( end == -1):\n",
    "        end = end =(s.rfind('}}\\n=='))\n",
    "    if( end == -1):\n",
    "        end = end =(s.rfind('}}\\n\\d'))\n",
    "    s = s[beg: end+2]\n",
    "    return s\n",
    "def get_med_cond(s):\n",
    "    beg = (s.rfind('{{Infobox medical condition'))\n",
    "    end  =(s.rfind('\\n}}'))\n",
    "    if( end == -1):\n",
    "        end = end =(s.rfind('}}\\n'))\n",
    "    if( end == -1):\n",
    "        end = end =(s.rfind('}}\\n<!--'))\n",
    "    if( end == -1):\n",
    "        end = end =(s.rfind('}}\\n=='))\n",
    "    if( end == -1):\n",
    "        end = end =(s.rfind('}}\\n\\d'))\n",
    "    s = s[beg: end+2]\n",
    "    return s\n",
    "def get_medical(s):\n",
    "    beg = (s.rfind('{{Medical resources'))\n",
    "    end  =(s.rfind('\\n}}'))\n",
    "    s = s[beg: end+3]\n",
    "    return s\n",
    "def find_unii(s):\n",
    "    s = re.findall(r'UNII\\s*?=\\s?.*',s)\n",
    "    if(len(s)>0):   \n",
    "        s = s[0]\n",
    "        equal = s.rfind('=')\n",
    "        #if there is a space after the equal remove it\n",
    "        if(s[equal+1]==' '):\n",
    "            s = s[equal+2:]\n",
    "        else: \n",
    "            s = s[equal+1:]\n",
    "    else:\n",
    "        s = ''\n",
    "    return s\n",
    "def find_icd10(s):\n",
    "    s = s.replace('|','')\n",
    "    s = s.replace('{{ICD10',\"\")\n",
    "    icd10 = re.findall('\\w{1}\\d{2,6}',s)\n",
    "    return icd10\n",
    "\n",
    "def find_medication(s):\n",
    "    s = re.findall(r'medication\\s*?=.*<',s)\n",
    "    if(len(s)== 0 ):\n",
    "        s = []\n",
    "    else:\n",
    "        s = s[0]\n",
    "        s = s[:-1]\n",
    "        s = s[s.find('=')+2:]\n",
    "        s = s.replace('[[','')\n",
    "        s = s.replace(']]','')\n",
    "        s = s.split(',')\n",
    "        s\n",
    "    return s\n",
    "def pad_array(arr,length):\n",
    "    for i in range(0,length-len(arr)):\n",
    "        arr.append(np.NaN)\n",
    "    return arr\n",
    "def find_max_medications(df):\n",
    "    return max([len(x) for x in df[df.medications.map(len) > 0].medications.values])\n",
    "def get_row_with_medications(df):\n",
    "    df =  df[df.medications.map(len) > 0]\n",
    "    df.reset_index(drop=True,inplace=True)\n",
    "    return df\n",
    "def set_meds_columns(df,max_col):\n",
    "    current_meds = df.medications.values\n",
    "    padded_meds = [pad_array(el,max_col) for el in current_meds]\n",
    "    cols = [f\"medication_{i}\" for i in range(max_col)]\n",
    "    col_df = pd.DataFrame(padded_meds, columns=cols)\n",
    "    final = pd.concat([df,col_df], axis=1)\n",
    "    return final\n",
    "def clean_row(test):\n",
    "    for i in range (len(test)):\n",
    "        el = test[i]\n",
    "        end = el.rfind('<')\n",
    "        if end !=-1:\n",
    "            new_el = el[:end]\n",
    "            test[i]=new_el\n",
    "        if i>0:\n",
    "            test[i]= test[i][1:]\n",
    "    return test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "totalCount = 0\n",
    "title = None\n",
    "icd9=[]\n",
    "icd10=[]\n",
    "#-----------arrays for drugs_df----------------DO NOT CHANGE NAMES\n",
    "drugs_name = []\n",
    "drugs_id = []\n",
    "drugs_unii = []\n",
    "#----arrays for diseases_df---------------------DO NOT CHANGE NAMES\n",
    "diseases_name = []\n",
    "diseases_id = []\n",
    "diseases_icd9 = []\n",
    "diseases_icd10 = []\n",
    "diseases_meds = []\n",
    "#-------------------------------------------\n",
    "start_time = time.time()\n",
    "for event, elem in etree.iterparse('enwiki-latest-pages-articles.xml', events=('start', 'end')):\n",
    "    tname = strip_tag_name(elem.tag)\n",
    "\n",
    "    if event == 'start':\n",
    "        if tname == 'page':\n",
    "            title = ''\n",
    "            id = -1\n",
    "            redirect = ''\n",
    "            inrevision = False\n",
    "            ns = 0\n",
    "        elif tname == 'revision':\n",
    "            # Do not pick up on revision id's\n",
    "            inrevision = True\n",
    "    else:\n",
    "        if tname == 'title':\n",
    "            title = elem.text\n",
    "        elif tname == 'id' and not inrevision:\n",
    "            id = int(elem.text)\n",
    "        elif tname == 'redirect':\n",
    "            redirect = elem.attrib['title']\n",
    "        elif tname == 'ns':\n",
    "            ns = int(elem.text)\n",
    "        elif tname == 'page':\n",
    "            totalCount += 1\n",
    "        elif(tname =='text'):\n",
    "            cont = elem.text\n",
    "            #check if cont exists\n",
    "            if(type(cont)==type('alex')):\n",
    "                #check if the drugbox template exists \n",
    "                if (cont.find('Drugbox')>-1 ):\n",
    "                    #get the drugbox template\n",
    "                    result = get_drugbox(cont)\n",
    "                    #get the CAS number\n",
    "                    unii = find_unii(result)\n",
    "                    #drugsWriter.writerow([id, title, cas])\n",
    "                    drugs_name.append(title)\n",
    "                    drugs_id.append(id)\n",
    "                    drugs_unii.append(unii)\n",
    "        if (id%100000 == 0):\n",
    "            \n",
    "\n",
    "elem.clear()\n",
    "elapsed_time = time.time() - start_time\n",
    "print(\"Elapsed time: {}\".format(hms_string(elapsed_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "diseases_with_meds = get_row_with_medications(diseases_df)\n",
    "diseases_with_meds.medications.apply(clean_row)\n",
    "max_meds = find_max_medications(diseases_with_meds)\n",
    "test = set_meds_columns(diseases_with_meds, max_meds)\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>unii</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1912</td>\n",
       "      <td>Ampicillin</td>\n",
       "      <td>7C782967RD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>4191</td>\n",
       "      <td>BCG vaccine</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>6346</td>\n",
       "      <td>Chloramphenicol</td>\n",
       "      <td>66974FR9Q1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>10024</td>\n",
       "      <td>MDMA</td>\n",
       "      <td>KE1SEN21RM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>11725</td>\n",
       "      <td>Flunitrazepam</td>\n",
       "      <td>620X0222FQ</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0     id             name        unii\n",
       "0           0   1912       Ampicillin  7C782967RD\n",
       "1           1   4191      BCG vaccine         NaN\n",
       "2           2   6346  Chloramphenicol  66974FR9Q1\n",
       "3           3  10024             MDMA  KE1SEN21RM\n",
       "4           4  11725    Flunitrazepam  620X0222FQ"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drugs = pd.read_csv('new_drugs_with_unii.csv')\n",
    "drugs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2138 entries, 0 to 2137\n",
      "Data columns (total 6 columns):\n",
      "Unnamed: 0     2138 non-null int64\n",
      "id             2138 non-null int64\n",
      "name           2138 non-null object\n",
      "icd9           2138 non-null object\n",
      "icd10          2138 non-null object\n",
      "medications    2138 non-null object\n",
      "dtypes: int64(2), object(4)\n",
      "memory usage: 100.3+ KB\n"
     ]
    }
   ],
   "source": [
    "diseases_df = pd.read_csv('new_diseases_with_med.csv')\n",
    "diseases_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'str' object does not support item assignment",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-407e6b8eea2c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mdiseases_with_meds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_row_with_medications\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdiseases_df\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdiseases_with_meds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmedications\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclean_row\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mmax_meds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfind_max_medications\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdiseases_with_meds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0malex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mset_meds_columns\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdiseases_with_meds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_meds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[0;32m   3190\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3191\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3192\u001b[1;33m                 \u001b[0mmapped\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3193\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3194\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/src\\inference.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32m<ipython-input-2-db97bd69d53e>\u001b[0m in \u001b[0;36mclean_row\u001b[1;34m(test)\u001b[0m\n\u001b[0;32m     99\u001b[0m             \u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnew_el\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    100\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m>\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 101\u001b[1;33m             \u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    102\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: 'str' object does not support item assignment"
     ]
    }
   ],
   "source": [
    "diseases_with_meds = get_row_with_medications(diseases_df)\n",
    "diseases_with_meds.medications.apply(clean_row)\n",
    "max_meds = find_max_medications(diseases_with_meds)\n",
    "alex = set_meds_columns(diseases_with_meds, max_meds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}