genelogy_award.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from dateparser.search import search_dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "people=pd.read_csv('./neurotree/people_cs.tsv', sep='\\t', encoding='utf-8',  escapechar='\\\\')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['pid', 'firstname', 'middlename', 'lastname', 'degrees', 'location',\n",
       "       'locid', 'area', 'majorarea', 'award', 'orcid_id', 'homepage',\n",
       "       'addedby', 'dateadded'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "people_inst=people[['pid','locid']]\n",
    "people_inst.to_csv('./neurotree/people_inst_cs.tsv',sep='\\t',index=False, encoding='utf-8',escapechar='\\\\')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "people=people[['pid','award']]\n",
    "people['award']=people['award'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people['pid'].nunique()==len(people['pid'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pid=[]\n",
    "awrd=[]\n",
    "for row in people.values:\n",
    "    text=str(row[1])\n",
    "    awrds=text.split(',')\n",
    "    for a in awrds:\n",
    "        pid.append(row[0])\n",
    "        awrd.append(a)\n",
    "people=pd.DataFrame(list(zip(pid, awrd)), \n",
    "               columns =['pid', 'award']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(697744, 3)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "93"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people['award'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_date(text):\n",
    "    date='0000'\n",
    "    text=str(text)\n",
    "    if text=='nan' or text=='n':\n",
    "         return date\n",
    "    else:\n",
    "        r=search_dates(text)\n",
    "        if r:\n",
    "            return str(r[0][1]).split('-')[0]\n",
    "    return date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "people['year']=people['award'].apply(extract_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "people['award']=people['award'].str.replace('\\d+', '')\n",
    "people['award']=people['award'].str.replace(\"\\(\\)\", '')\n",
    "people['award']=people['award'].str.replace(\"prize\", '')\n",
    "people['award']=people['award'].apply(lambda x: x.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "awards=pd.unique(people['award'].values.ravel('K'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "awards=pd.DataFrame({'award':awards[:]})\n",
    "awards['id']=awards.index\n",
    "awards=awards[['id','award']]\n",
    "awards.to_csv('./neurotree/awards.csv',sep='\\t',index=False, encoding='utf-8',escapechar='\\\\')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "people['award']=people['award'].map(awards.set_index('award')['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "people.to_csv('./neurotree/award_people.csv',sep='\\t',index=False, encoding='utf-8',escapechar='\\\\')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r=search_dates('nas 1954, national medal of science 1974')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "award['year']=award.apply(lambda x : x + 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "awards.iloc[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "remove=[\"\\d+\",\"?\\(\\)\"]\n",
    "import re\n",
    "a=re.search(\"\\d+\",'nas 1954, national medal of science 1974')\n",
    "a=a.group()\n",
    "re.sub(r\"\\d+\", \" \", 'bancroft (2012)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "people['award'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "items = [\"example ()\", \"w3resource\", \"github (.com)\", \"stackoverflow (.com)\"]\n",
    "for item in items:\n",
    "    print(re.sub(r\" ?\\(\\)\", \"\", item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "people['name']=people['firstname'].fillna('')+\" \"+people['middlename'].fillna('')+\" \"+people['lastname'].fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "people['name']=people['name'].str.lower().replace(' +',value=' ',regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    edward h. adelson\n",
       "1    stephen e. palmer\n",
       "2    michael i. jordan\n",
       "3        reza shadmehr\n",
       "4         john jonides\n",
       "Name: name, dtype: object"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people['name'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}